Major code import.
This commit is contained in:
parent
13fb5dac1c
commit
43d40f7fce
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"site_root":"https://example.com",
|
||||||
|
"title":"Test Metadata",
|
||||||
|
"author": "Test User",
|
||||||
|
"author_email": "test_user@example.com",
|
||||||
|
"uuid_oid_root": "pixywerk-demo"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
|
||||||
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||||
|
|
||||||
|
<title>{{ metadata.title }}</title>
|
||||||
|
<subtitle>{{ metadata.subtitle }}</subtitle>
|
||||||
|
<link href="{{ metadata.site_root }}/{{ metadata.file_name }}" rel="self" />
|
||||||
|
<link href="{{ metadata.site_root }}" />
|
||||||
|
<id>urn:uuid:{{ metadata.uuid }}</id>
|
||||||
|
<updated>{{ get_time_iso8601(metadata['build-time']) }}</updated>
|
||||||
|
|
||||||
|
{% set posts = get_file_list('blog_posts/*.cont') %}
|
||||||
|
{% for post in posts %}
|
||||||
|
{% set post_meta = get_file_metadata(post['file_path']) %}
|
||||||
|
<entry>
|
||||||
|
<title>{{ post_meta.title }}</title>
|
||||||
|
<link href="{{ metadata.site_root }}/{{post_meta.file_path}}" />
|
||||||
|
<id>urn:uuid:{{ post_meta.uuid }}</id>
|
||||||
|
<updated>{{ get_time_iso8601(post_meta.stat.mtime) }}</updated>
|
||||||
|
<summary>{{post_meta.summary }}</summary>
|
||||||
|
<!-- this would be the snippet, more than summary chunk -->
|
||||||
|
<!-- <content type="xhtml"> -->
|
||||||
|
<!-- <div xmlns="http://www.w3.org/1999/xhtml"> -->
|
||||||
|
<!-- <p>{{ post_meta.summary }}</p> -->
|
||||||
|
<!-- </div> -->
|
||||||
|
<!-- </content> -->
|
||||||
|
<author>
|
||||||
|
<name>{{ post_meta.author }}</name>
|
||||||
|
<email>{{ post_meta.author_email }}</email>
|
||||||
|
</author>
|
||||||
|
</entry>
|
||||||
|
{% endfor %}
|
||||||
|
</feed>
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
{
|
||||||
|
"type": "templatable",
|
||||||
|
"title": "Test RSS Feed",
|
||||||
|
"subtitle": "Some Subtitle"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
Some more post
|
||||||
|
|
||||||
|
|
||||||
|
la la la
|
||||||
|
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
{
|
||||||
|
"title":"Another Post(tm)",
|
||||||
|
"summary":"Yet another post"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Some content.
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
{
|
||||||
|
"title":"Test.cont",
|
||||||
|
"summary":"Some empty test content"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
yo fresh
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
{
|
||||||
|
"foo":"bar",
|
||||||
|
"title":"A title",
|
||||||
|
"summary":"Just a post."
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
<h1>Index of all content</h1>
|
||||||
|
{% for f in get_file_list('*', sort_order='file_name') %}
|
||||||
|
<a href="{{ get_file_name(f['file_name']) }}">{{get_file_name(f['file_name'])}}</a>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
<p>Including foo.cont.meta:
|
||||||
|
<pre>
|
||||||
|
{{ get_file_content('foo.cont.meta') }}
|
||||||
|
</pre>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<h1>Metadata</h1>
|
||||||
|
<table class="metadata">
|
||||||
|
<tr><th>key</th><th>value</th></tr>
|
||||||
|
{% set metadata = get_file_metadata('foo.cont') %}
|
||||||
|
{% for k in metadata.keys() %}
|
||||||
|
<tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
# README #
|
||||||
|
|
||||||
|
This is a test of the emergency compiled HTML system. This is only a *test*.
|
||||||
|
|
||||||
|
[Foo!](foo.html)
|
||||||
|
|
||||||
|
{% for i in range(100) %}
|
||||||
|
* {{ i }}
|
||||||
|
{% endfor %}
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
"pragma":["no-proc"]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
# README #
|
||||||
|
|
||||||
|
This is a test of the emergency compiled HTML system. This is only a *test*.
|
||||||
|
|
||||||
|
[Foo!](foo.html)
|
||||||
|
|
||||||
|
{% for i in range(100) %}
|
||||||
|
* {{ i }}
|
||||||
|
{% endfor %}
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
"title":"Yo, markdown"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<head>
|
||||||
|
<title>Debug for {{path}}</title>
|
||||||
|
<style type="text/css">
|
||||||
|
table { border: 1px solid black; }
|
||||||
|
div { border: 1px solid black; }
|
||||||
|
td { border: 1px solid black; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<p>{{path}}</p>
|
||||||
|
<h1>Content</h1>
|
||||||
|
<div class="content">
|
||||||
|
{{content}}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h1>Environment</h1>
|
||||||
|
<table class="environment">
|
||||||
|
<tr><th>key</th><th>value</th></tr>
|
||||||
|
{% for k in environ.keys() %}
|
||||||
|
<tr><td>{{k}}</td><td>{{environ[k]}}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<h1>Metadata</h1>
|
||||||
|
<table class="metadata">
|
||||||
|
<tr><th>key</th><th>value</th></tr>
|
||||||
|
{% for k in metadata.keys() %}
|
||||||
|
<tr><td>{{k}}</td><td>{{metadata[k]}}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
</body>
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
<table class="werk-file-list">
|
||||||
|
<tr class="werk-file-list-head"><th>file</th><th>type</th><th>size</th><th>last change</th></tr>
|
||||||
|
{% for f in files.keys() %}
|
||||||
|
<tr class="werk-file-list-item"><td><a href="/{{files[f].relpath}}">{{f}}</a></td><td>{{files[f].type}}</td><td>{{files[f].size}}</td><td>{{files[f].ctime | date}}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<head>
|
||||||
|
<title>{{metadata.title}}</title>
|
||||||
|
<style type="text/css">
|
||||||
|
table { border: 1px solid black; }
|
||||||
|
div { border: 1px solid black; }
|
||||||
|
td { border: 1px solid black; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
{{content}}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
# iterate source tree
|
||||||
|
# create directors in target tree
|
||||||
|
# for each item:
|
||||||
|
# run processor(s) on item, each processor could be in a chain or a branch
|
||||||
|
# Processors also provide filename munging
|
||||||
|
# output target based on processor output
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
from typing import Dict, List, cast
|
||||||
|
|
||||||
|
from .processchain import ProcessorChains
|
||||||
|
from .metadata import MetaTree
|
||||||
|
from .template_tools import file_list, file_name, file_content, file_metadata, time_iso8601
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
|
def setup_logging(verbose: bool = False) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_args(args: List[str]) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser("Compile a Pixywerk directory into an output directory.")
|
||||||
|
|
||||||
|
parser.add_argument("root", help="The root of the pixywerk directory to process.")
|
||||||
|
parser.add_argument("output", help="The output directory to export post-compiled files to.")
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"-c", "--clean", help="Remove the target tree before proceeding (by renaming to .bak).", action="store_true"
|
||||||
|
)
|
||||||
|
parser.add_argument("-s", "--safe", help="Abort if the target directory already exists.", action="store_true")
|
||||||
|
parser.add_argument("-t", "--template", help="The template directory (default: root/templates)", default=None)
|
||||||
|
parser.add_argument("-d", "--dry-run", help="Perform a dry-run.", action="store_true")
|
||||||
|
parser.add_argument("-v", "--verbose", help="Output verbosely.", action="store_true")
|
||||||
|
parser.add_argument("--processors", help="Specify a path to a processor configuration file.", default=None)
|
||||||
|
|
||||||
|
result = parser.parse_args(args)
|
||||||
|
|
||||||
|
# validate arguments
|
||||||
|
if not os.path.isdir(result.root):
|
||||||
|
raise FileNotFoundError("can't find root folder {}".format(result.root))
|
||||||
|
|
||||||
|
if not result.template:
|
||||||
|
result.template = os.path.join(result.root, "templates")
|
||||||
|
result.excludes = [result.template]
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
args = get_args(sys.argv[1:])
|
||||||
|
except FileNotFoundError as ex:
|
||||||
|
print("error finding arguments: {}".format(ex))
|
||||||
|
return 1
|
||||||
|
setup_logging(args.verbose)
|
||||||
|
if os.path.exists(args.output) and args.clean:
|
||||||
|
bak = "{}.bak-{}".format(args.output, int(time.time()))
|
||||||
|
print("cleaning target {} -> {}".format(args.output, bak))
|
||||||
|
os.rename(args.output, bak)
|
||||||
|
|
||||||
|
process_chains = ProcessorChains(args.processors)
|
||||||
|
|
||||||
|
default_metadata = {
|
||||||
|
"templates": args.template,
|
||||||
|
"template": "default.jinja2",
|
||||||
|
"dir-template": "default-dir.jinja2",
|
||||||
|
"filters": {},
|
||||||
|
"build-time": time.time(),
|
||||||
|
"build-datetime": time.ctime(),
|
||||||
|
"uuid-oid-root": "pixywerk",
|
||||||
|
}
|
||||||
|
meta_tree = MetaTree(args.root, default_metadata)
|
||||||
|
file_list_cache = cast(Dict, {})
|
||||||
|
file_cont_cache = cast(Dict, {})
|
||||||
|
file_name_cache = cast(Dict, {})
|
||||||
|
default_metadata["globals"] = {
|
||||||
|
"get_file_list": file_list(args.root, file_list_cache),
|
||||||
|
"get_file_name": file_name(args.root, meta_tree, process_chains, file_name_cache),
|
||||||
|
"get_file_content": file_content(args.root, meta_tree, process_chains, file_cont_cache),
|
||||||
|
"get_file_metadata": file_metadata(meta_tree),
|
||||||
|
"get_time_iso8601": time_iso8601("UTC"),
|
||||||
|
}
|
||||||
|
|
||||||
|
for root, _, files in os.walk(args.root):
|
||||||
|
workroot = os.path.relpath(root, args.root)
|
||||||
|
if workroot == ".":
|
||||||
|
workroot = ""
|
||||||
|
target_dir = os.path.join(args.output, workroot)
|
||||||
|
print("mkdir -> {}".format(target_dir))
|
||||||
|
if not args.dry_run:
|
||||||
|
try:
|
||||||
|
os.mkdir(target_dir)
|
||||||
|
except FileExistsError:
|
||||||
|
if args.safe:
|
||||||
|
print("error, target directory exists, aborting")
|
||||||
|
return 1
|
||||||
|
for f in files:
|
||||||
|
# fixme global generic filters
|
||||||
|
if f.endswith(".meta") or f.endswith("~"):
|
||||||
|
continue
|
||||||
|
metadata = meta_tree.get_metadata(os.path.join(workroot, f))
|
||||||
|
chain = process_chains.get_chain_for_filename(os.path.join(root, f), ctx=metadata)
|
||||||
|
print("process {} -> {}".format(os.path.join(root, f), os.path.join(target_dir, chain.output_filename)))
|
||||||
|
if not args.dry_run:
|
||||||
|
with open(os.path.join(target_dir, chain.output_filename), "w") as outfile:
|
||||||
|
for line in chain.output:
|
||||||
|
outfile.write(line)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Default: output == input
|
||||||
|
default:
|
||||||
|
extension: default
|
||||||
|
chain:
|
||||||
|
- passthrough
|
||||||
|
|
||||||
|
# Any object that needs jinja scripts but no other explicit processing
|
||||||
|
templatable:
|
||||||
|
extension: null
|
||||||
|
chain:
|
||||||
|
- jinja2
|
||||||
|
|
||||||
|
# Markdown, BBCode and RST are first run through the templater, and then
|
||||||
|
# they are processed into HTML, and finally embedded in a page template.
|
||||||
|
markdown:
|
||||||
|
extension:
|
||||||
|
- md
|
||||||
|
chain:
|
||||||
|
- jinja2
|
||||||
|
- process_md
|
||||||
|
- jinja2_page_embed
|
||||||
|
bbcode:
|
||||||
|
extension:
|
||||||
|
- bb
|
||||||
|
- pp
|
||||||
|
chain:
|
||||||
|
- jinja2
|
||||||
|
- process_pp
|
||||||
|
- jinja2_page_embed
|
||||||
|
# FIXME implement RST processor
|
||||||
|
# restructured:
|
||||||
|
# extension:
|
||||||
|
# - rst
|
||||||
|
# chain:
|
||||||
|
# - jinja2
|
||||||
|
# - process_rst
|
||||||
|
# - jinja2_page_embed
|
||||||
|
|
||||||
|
# # JSON and YAML are split, passed through a pretty printer, and then output
|
||||||
|
# FIXME implement split chain processor, implement processor arguments
|
||||||
|
# json:
|
||||||
|
# extension:
|
||||||
|
# - json
|
||||||
|
# chain:
|
||||||
|
# - split (passthrough)
|
||||||
|
# - pp_json
|
||||||
|
# yaml:
|
||||||
|
# extension:
|
||||||
|
# - yml
|
||||||
|
# - yaml
|
||||||
|
# chain:
|
||||||
|
# - split (passthrough)
|
||||||
|
# - pp_yaml
|
||||||
|
|
||||||
|
# Template-html is first passed through the templater, and then embedded
|
||||||
|
# in a page template
|
||||||
|
template-html:
|
||||||
|
extension:
|
||||||
|
- thtml
|
||||||
|
- cont
|
||||||
|
chain:
|
||||||
|
- jinja2
|
||||||
|
- jinja2_page_embed
|
||||||
|
|
||||||
|
# Smart CSS are simply converted to CSS.
|
||||||
|
sass:
|
||||||
|
extension:
|
||||||
|
- sass
|
||||||
|
- scss
|
||||||
|
chain:
|
||||||
|
- process_sass
|
||||||
|
less:
|
||||||
|
extension:
|
||||||
|
- less
|
||||||
|
chain:
|
||||||
|
- process_less
|
||||||
|
|
||||||
|
stylus:
|
||||||
|
extension:
|
||||||
|
- styl
|
||||||
|
chain:
|
||||||
|
- process_styl
|
||||||
|
|
||||||
|
# # Images are processed into thumbnails and sized in addition to being retained as their original
|
||||||
|
# FIXME implement split chain processor, implement processor arguments,
|
||||||
|
# image:
|
||||||
|
# extension:
|
||||||
|
# - jpg
|
||||||
|
# - jpeg
|
||||||
|
# - png
|
||||||
|
# chain:
|
||||||
|
# - split (image_bigthumb)
|
||||||
|
# - split (image_smallthumb)
|
||||||
|
# - passthrough
|
||||||
|
|
||||||
|
# image_bigthumb:
|
||||||
|
# extension:
|
||||||
|
# chain:
|
||||||
|
# - smart_resize (big)
|
||||||
|
|
||||||
|
# image_smallthumb:
|
||||||
|
# extension:
|
||||||
|
# chain:
|
||||||
|
# - smart_resize (small)
|
||||||
|
|
@ -0,0 +1,145 @@
|
||||||
|
"""Constructs a tree-like object containing the metadata for a given path, and caches said metadata."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import mimetypes
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from typing import Dict, Optional, Union, List, Tuple, Any, cast
|
||||||
|
|
||||||
|
import jstyleson
|
||||||
|
|
||||||
|
from .utils import guess_mime
|
||||||
|
|
||||||
|
# setup mimetypes with some extra ones
|
||||||
|
mimetypes.init()
|
||||||
|
mimetypes.add_type("text/html", "thtml")
|
||||||
|
mimetypes.add_type("text/html", "cont")
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class MetaCacheMiss(Exception):
|
||||||
|
"""Raised on cache miss."""
|
||||||
|
|
||||||
|
|
||||||
|
class MetaCache:
|
||||||
|
"""This class provides an in-memory cache for metadata tree."""
|
||||||
|
|
||||||
|
def __init__(self, max_age: float = 200.0):
|
||||||
|
"""Initialize the cache.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
max_age (int): the number of seconds to age-out cache items
|
||||||
|
|
||||||
|
"""
|
||||||
|
self._max_age = max_age
|
||||||
|
self._cache: Dict[str, Tuple[float, Any]] = {}
|
||||||
|
|
||||||
|
def get(self, key: str, new_time_stamp: float) -> Any:
|
||||||
|
"""Get an item from the cache.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
key (str): the cache key to retieve
|
||||||
|
new_time_stamp (int): The time to use to compare the stored time with
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
:obj:misc: The previously stored value.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
MetaCacheMiss: on missing key, or on aged out
|
||||||
|
|
||||||
|
"""
|
||||||
|
if key not in self._cache:
|
||||||
|
raise MetaCacheMiss("no item for key {}".format(key))
|
||||||
|
|
||||||
|
if self._cache[key][0] + self._max_age <= new_time_stamp:
|
||||||
|
return self._cache[key][1]
|
||||||
|
|
||||||
|
raise MetaCacheMiss("cache expired for key {}".format(key))
|
||||||
|
|
||||||
|
def put(self, key: str, value: Union[Dict, List, int, str, object], time_stamp: float) -> None:
|
||||||
|
"""Put an item into the cache.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
key (str): the key to store the cache item under
|
||||||
|
value (:obj:misc): the value to store
|
||||||
|
time_stamp (float): the time stamp to store the item under
|
||||||
|
|
||||||
|
"""
|
||||||
|
self._cache[key] = (time_stamp, value)
|
||||||
|
|
||||||
|
|
||||||
|
class MetaTree:
|
||||||
|
"""This provides an interface to loading and caching tree metadata for a given directory tree."""
|
||||||
|
|
||||||
|
def __init__(self, root: str, default_metadata: Optional[Dict] = None):
|
||||||
|
"""Initialize the metadata tree object.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
root (str): The path to the root of the file tree to operate on.
|
||||||
|
default_metadata (dict, optional): The default metadata to apply to the tree
|
||||||
|
|
||||||
|
"""
|
||||||
|
self._cache = MetaCache()
|
||||||
|
if default_metadata is None:
|
||||||
|
default_metadata = {}
|
||||||
|
self._default_metadata = default_metadata
|
||||||
|
if root[-1] != "/":
|
||||||
|
root += "/"
|
||||||
|
self._root = root
|
||||||
|
|
||||||
|
def get_metadata(self, rel_path: str) -> Dict:
|
||||||
|
"""Retrieve the metadata for a given path
|
||||||
|
|
||||||
|
The general procedure is to iterate the tree, at each level
|
||||||
|
m load .meta (JSON formatted dictionary) for that level, and
|
||||||
|
then finally load the path.meta, and merge these dictionaries
|
||||||
|
in descendant order.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
rel_path (str): The path to retrieve the metadata for (relative to root)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict: A dictionary of metadata for that path tree.
|
||||||
|
|
||||||
|
"""
|
||||||
|
metablob = dict(self._default_metadata)
|
||||||
|
# iterate path components from root to target path
|
||||||
|
comps = [self._root] + rel_path.split("/")
|
||||||
|
fullpath = ""
|
||||||
|
for pth in comps:
|
||||||
|
fullpath = os.path.join(fullpath, pth)
|
||||||
|
st = os.stat(fullpath)
|
||||||
|
|
||||||
|
cachekey = fullpath + ".meta"
|
||||||
|
meta = cast(Dict, {})
|
||||||
|
try:
|
||||||
|
st_meta = os.stat(cachekey)
|
||||||
|
meta = self._cache.get(cachekey, st_meta.st_mtime)
|
||||||
|
except FileNotFoundError:
|
||||||
|
st_meta = None # type: ignore
|
||||||
|
except MetaCacheMiss:
|
||||||
|
meta = {}
|
||||||
|
|
||||||
|
if not meta and st_meta:
|
||||||
|
meta = jstyleson.load(open(cachekey, "r"))
|
||||||
|
self._cache.put(cachekey, meta, st_meta.st_mtime)
|
||||||
|
|
||||||
|
metablob.update(meta)
|
||||||
|
|
||||||
|
# return final dict
|
||||||
|
metablob["dir"], metablob["file_name"] = os.path.split(rel_path)
|
||||||
|
metablob["file_path"] = rel_path
|
||||||
|
metablob["uuid"] = uuid.uuid3(
|
||||||
|
uuid.NAMESPACE_OID, metablob["uuid-oid-root"] + os.path.join(self._root, rel_path)
|
||||||
|
)
|
||||||
|
metablob["os-path"], _ = os.path.split(fullpath)
|
||||||
|
metablob["guessed-type"] = guess_mime(os.path.join(self._root, rel_path))
|
||||||
|
if "mime-type" not in metablob:
|
||||||
|
metablob["mime-type"] = metablob["guessed-type"]
|
||||||
|
metablob["stat"] = {}
|
||||||
|
for stk in ("st_mtime", "st_ctime", "st_atime", "st_mode", "st_size", "st_ino"):
|
||||||
|
metablob["stat"][stk.replace("st_", "")] = getattr(st, stk)
|
||||||
|
|
||||||
|
return metablob
|
||||||
|
|
@ -0,0 +1,180 @@
|
||||||
|
"""Interface for chains of processors"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import os.path
|
||||||
|
import random
|
||||||
|
|
||||||
|
from typing import List, Iterable, Optional, Any, Dict, Type, cast
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from .processors.processors import Processor
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessorChain:
|
||||||
|
"""This implements a wrapper for an arbitrary set of processors and an associated file stream."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
processors: List[Processor],
|
||||||
|
file_name: str,
|
||||||
|
file_data: Iterable[str],
|
||||||
|
file_type: str,
|
||||||
|
ctx: Optional[Dict] = None,
|
||||||
|
):
|
||||||
|
"""Initialize the processing stream.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
processors (list): A list of processor objects.
|
||||||
|
file_data (Iterable): An iterable from which to retrieve the input
|
||||||
|
file_type (str): the specified file type for consumer information.
|
||||||
|
|
||||||
|
"""
|
||||||
|
self._processors = processors
|
||||||
|
self._file_data = file_data
|
||||||
|
self._file_type = file_type
|
||||||
|
self._file_name = file_name
|
||||||
|
self._ctx: Dict = {}
|
||||||
|
if ctx is not None:
|
||||||
|
self._ctx = cast(Dict, ctx)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output(self) -> Iterable:
|
||||||
|
"""Return an iterable for the output of the process chain
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
:obj:'iterable': the iterable
|
||||||
|
|
||||||
|
"""
|
||||||
|
prev = self._file_data
|
||||||
|
for processor in self._processors:
|
||||||
|
if processor:
|
||||||
|
prev = processor.process(prev, self._ctx)
|
||||||
|
|
||||||
|
return prev
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_mime(self) -> str:
|
||||||
|
"""Return the post-processed MIME value from the processing chain
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the mime type
|
||||||
|
|
||||||
|
"""
|
||||||
|
fname = self._file_name
|
||||||
|
for processor in self._processors:
|
||||||
|
fname = processor.mime_type(fname, self._ctx)
|
||||||
|
return fname
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_ext(self) -> str:
|
||||||
|
"""Return the post-processed extension from the processing chain
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the extension
|
||||||
|
"""
|
||||||
|
fname = self._file_name
|
||||||
|
for processor in self._processors:
|
||||||
|
fname = processor.extension(fname, self._ctx)
|
||||||
|
return fname
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_filename(self) -> str:
|
||||||
|
"""Return the post-processed filename from the processing chain
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new filename
|
||||||
|
|
||||||
|
"""
|
||||||
|
fname = os.path.basename(self._file_name)
|
||||||
|
for processor in self._processors:
|
||||||
|
fname = processor.filename(fname, self._ctx)
|
||||||
|
return fname
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessorChains:
|
||||||
|
"""Load a configuration for processor chains, and provide ability to process the chains given a particular input
|
||||||
|
file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, config: Optional[str] = None):
|
||||||
|
"""Initialize, with a specified configuration file
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
config (str, optional): The path to a yaml formatted configuration file.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if config is None: # pragma: no coverage
|
||||||
|
config = os.path.join(os.path.dirname(__file__), "defaults", "chains.yaml")
|
||||||
|
|
||||||
|
self.chainconfig = yaml.load(open(config, "r"))
|
||||||
|
self.extensionmap: Dict[str, Any] = {}
|
||||||
|
self.processors: Dict[str, Type[Processor]] = {}
|
||||||
|
for ch, conf in self.chainconfig.items():
|
||||||
|
if conf["extension"] == "default":
|
||||||
|
self.default = ch
|
||||||
|
else:
|
||||||
|
if conf["extension"]:
|
||||||
|
for ex in conf["extension"]:
|
||||||
|
if ex in self.extensionmap or ex is None:
|
||||||
|
# log an error or except or something we'll just override for now.
|
||||||
|
pass
|
||||||
|
self.extensionmap[ex] = ch
|
||||||
|
for pr in conf["chain"]:
|
||||||
|
if pr in self.processors:
|
||||||
|
continue
|
||||||
|
processor_module = __import__("processors", globals(), locals(), [pr], 1)
|
||||||
|
self.processors[pr] = processor_module.__dict__[pr].processor
|
||||||
|
|
||||||
|
def get_chain_for_filename(self, filename: str, ctx: Optional[Dict] = None) -> ProcessorChain:
|
||||||
|
"""Get the ProcessorChain, as configured for a given file by extension.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
filename (str): The name of the file to get a chain for.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ProcessorChain: the constructed processor chain.
|
||||||
|
"""
|
||||||
|
r = filename.rsplit(".", 1)
|
||||||
|
ftype = "default"
|
||||||
|
if r:
|
||||||
|
ftype = r[-1]
|
||||||
|
if ctx and "pragma" in ctx:
|
||||||
|
if "no-proc" in ctx["pragma"]:
|
||||||
|
ftype = "default"
|
||||||
|
|
||||||
|
if ctx and "type" in ctx:
|
||||||
|
ftype = ctx["type"]
|
||||||
|
return self.get_chain_for_file(open(filename, "r"), ftype, filename, ctx)
|
||||||
|
|
||||||
|
def get_chain_for_file(
|
||||||
|
self, file_obj: Iterable, file_ext: str, file_name: Optional[str] = None, ctx: Optional[Dict] = None
|
||||||
|
) -> ProcessorChain:
|
||||||
|
"""Get the ProcessorChain for a given iterable object based on the specified file type
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
file_obj (:obj:`iterable`): The input file stream
|
||||||
|
file_ext (str): The type (extension) of the input stream
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ProcessorChain: the constructed processor chain.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if file_ext not in self.extensionmap or not self.extensionmap[file_ext]:
|
||||||
|
if file_ext in self.chainconfig:
|
||||||
|
file_type = file_ext
|
||||||
|
else:
|
||||||
|
file_type = "default"
|
||||||
|
else:
|
||||||
|
file_type = self.extensionmap[file_ext]
|
||||||
|
|
||||||
|
if not (bool(file_name)):
|
||||||
|
file_name = hex(random.randint(0, 65536))
|
||||||
|
|
||||||
|
return ProcessorChain(
|
||||||
|
[self.processors[x]() for x in self.chainconfig[file_type]["chain"]],
|
||||||
|
cast(str, file_name),
|
||||||
|
file_obj,
|
||||||
|
file_type,
|
||||||
|
ctx,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
# processors metadata here
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
"""Define a Jinja2 Processor which applies programmable templating to the input stream."""
|
||||||
|
|
||||||
|
from typing import Iterable, Optional, Dict, cast
|
||||||
|
|
||||||
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
|
||||||
|
from .passthrough import PassThrough
|
||||||
|
|
||||||
|
|
||||||
|
class Jinja2(PassThrough):
|
||||||
|
"""Pass the input stream through Jinja2 for scritable templating."""
|
||||||
|
|
||||||
|
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
|
||||||
|
"""Return an iterable object of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
input_file (iterable): An input stream
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
iterable: The post-processed output stream
|
||||||
|
"""
|
||||||
|
ctx = cast(Dict, ctx)
|
||||||
|
template_env = Environment(loader=FileSystemLoader(ctx["templates"]))
|
||||||
|
template_env.globals.update(ctx["globals"])
|
||||||
|
template_env.filters.update(ctx["filters"])
|
||||||
|
tmpl = template_env.from_string("".join([x for x in input_file]))
|
||||||
|
return tmpl.render(metadata=ctx)
|
||||||
|
|
||||||
|
|
||||||
|
processor = Jinja2
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
"""Define a Jinja2 processor which embeds the (presumably HTML) input stream into a Page Template
|
||||||
|
as defined in the ctx metadata (the ``content`` variable is assigned to the input stream and
|
||||||
|
the target template is rendered)."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from typing import Iterable, Optional, Dict, cast
|
||||||
|
|
||||||
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
|
||||||
|
from .processors import Processor
|
||||||
|
|
||||||
|
|
||||||
|
class Jinja2PageEmbed(Processor):
|
||||||
|
"""Embed input stream as ``content`` variable in page template defined in context key ``template``."""
|
||||||
|
|
||||||
|
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the filename of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the previous name for the file.
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new name for the file
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
return os.path.splitext(oldname)[0] + ".html"
|
||||||
|
|
||||||
|
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new mimetype of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
return "text/html"
|
||||||
|
|
||||||
|
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
|
||||||
|
"""Return an iterable object of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
input_file (iterable): An input stream
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
iterable: The post-processed output stream
|
||||||
|
"""
|
||||||
|
ctx = cast(Dict, ctx)
|
||||||
|
template_env = Environment(loader=FileSystemLoader(ctx["templates"]))
|
||||||
|
template_env.globals.update(ctx["globals"])
|
||||||
|
template_env.filters.update(ctx["filters"])
|
||||||
|
tmpl = template_env.get_template(ctx["template"])
|
||||||
|
content = "".join([x for x in input_file])
|
||||||
|
return tmpl.render(content=content, metadata=ctx)
|
||||||
|
|
||||||
|
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new extension of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
return "html"
|
||||||
|
|
||||||
|
|
||||||
|
processor = Jinja2PageEmbed
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
"""Passthrough progcessor which takes input and returns it."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from .processors import Processor
|
||||||
|
from ..utils import guess_mime
|
||||||
|
from typing import Iterable, Optional, Dict, cast
|
||||||
|
|
||||||
|
|
||||||
|
class PassThrough(Processor):
|
||||||
|
"""A simple passthrough processor that takes input and sends it to output."""
|
||||||
|
|
||||||
|
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the filename of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the previous name for the file.
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new name for the file
|
||||||
|
|
||||||
|
"""
|
||||||
|
return oldname
|
||||||
|
|
||||||
|
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new mimetype of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
result = cast(str, guess_mime(oldname))
|
||||||
|
if result == "directory":
|
||||||
|
result = "DIR"
|
||||||
|
return result
|
||||||
|
|
||||||
|
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
|
||||||
|
"""Return an iterable object of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
input_file (iterable): An input stream
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
iterable: The post-processed output stream
|
||||||
|
"""
|
||||||
|
return input_file
|
||||||
|
|
||||||
|
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new extension of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
return os.path.splitext(oldname)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
processor = PassThrough
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
processor = None
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
"""Convert an MD stream into an HTML stream"""
|
||||||
|
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
|
||||||
|
from typing import Iterable, Optional, Dict
|
||||||
|
|
||||||
|
import markdown
|
||||||
|
|
||||||
|
from .processors import Processor
|
||||||
|
|
||||||
|
|
||||||
|
class MarkdownProcessor(Processor):
|
||||||
|
"""Convert an MD stream into an HTML stream"""
|
||||||
|
|
||||||
|
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the filename of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the previous name for the file.
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new name for the file
|
||||||
|
|
||||||
|
"""
|
||||||
|
return os.path.splitext(oldname)[0] + ".html"
|
||||||
|
|
||||||
|
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new mimetype of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
return "text/html"
|
||||||
|
|
||||||
|
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new extension of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
return "html"
|
||||||
|
|
||||||
|
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
|
||||||
|
"""Return an iterable object of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
input_file (iterable): An input stream
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
iterable: The post-processed output stream
|
||||||
|
"""
|
||||||
|
md = u"".join([x for x in input_file])
|
||||||
|
return io.StringIO(markdown.markdown(md, extensions=["extra", "admonition", "wikilinks"]))
|
||||||
|
|
||||||
|
|
||||||
|
processor = MarkdownProcessor # pylint: disable=invalid-name
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
processor = None
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
processor = None
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
processor = None
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
import abc
|
||||||
|
|
||||||
|
from typing import Iterable, Optional, Dict
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessorException(Exception): # pragma: no cover
|
||||||
|
"""A base exception class to be used by processor objects."""
|
||||||
|
|
||||||
|
|
||||||
|
class Processor(abc.ABC): # pragma: no cover
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
"""Initialize the class."""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def filename(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the filename of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the previous name for the file.
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new name for the file
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def mime_type(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new mimetype of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def extension(self, oldname: str, ctx: Optional[Dict] = None) -> str:
|
||||||
|
"""Return the mimetype of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
oldname (str): the input filename
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the new extension of the file after processing
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def process(self, input_file: Iterable, ctx: Optional[Dict] = None) -> Iterable:
|
||||||
|
"""Return an iterable object of the post-processed file.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
input_file (iterable): An input stream
|
||||||
|
ctx (dict, optional): A context object generated from the processor configuration
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
iterable: The post-processed output stream
|
||||||
|
"""
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
import datetime
|
||||||
|
import glob
|
||||||
|
import itertools
|
||||||
|
import os
|
||||||
|
import pytz
|
||||||
|
from typing import Callable, Dict, List, Iterable, Union, cast
|
||||||
|
|
||||||
|
from .metadata import MetaTree
|
||||||
|
from .processchain import ProcessorChains
|
||||||
|
|
||||||
|
|
||||||
|
def file_list(root: str, listcache: Dict) -> Callable:
|
||||||
|
def get_file_list(path_glob: str, *, sort_order: str = "ctime", reverse: bool = False, limit: int = 0) -> Iterable:
|
||||||
|
stattable = cast(List, [])
|
||||||
|
if path_glob in listcache:
|
||||||
|
stattable = listcache[path_glob]
|
||||||
|
else:
|
||||||
|
for fil in glob.glob(os.path.join(root, path_glob)):
|
||||||
|
if os.path.isdir(fil):
|
||||||
|
continue
|
||||||
|
if fil.endswith(".meta") or fil.endswith("~"):
|
||||||
|
continue
|
||||||
|
st = os.stat(fil)
|
||||||
|
stattable.append(
|
||||||
|
{
|
||||||
|
"file_path": os.path.relpath(fil, root),
|
||||||
|
"file_name": os.path.split(fil)[-1],
|
||||||
|
"mtime": st.st_mtime,
|
||||||
|
"ctime": st.st_ctime,
|
||||||
|
"size": st.st_size,
|
||||||
|
"ext": os.path.splitext(fil)[1],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
listcache[path_glob] = stattable
|
||||||
|
ret = sorted(stattable, key=lambda x: x[sort_order], reverse=reverse)
|
||||||
|
if limit > 0:
|
||||||
|
return itertools.islice(ret, limit)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
return get_file_list
|
||||||
|
|
||||||
|
|
||||||
|
def file_name(root: str, metatree: MetaTree, processor_chains: ProcessorChains, namecache: Dict) -> Callable:
|
||||||
|
def get_file_name(file_name: str) -> Dict:
|
||||||
|
if file_name in namecache:
|
||||||
|
return namecache[file_name]
|
||||||
|
metadata = metatree.get_metadata(file_name)
|
||||||
|
chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata)
|
||||||
|
namecache[file_name] = chain.output_filename
|
||||||
|
return namecache[file_name]
|
||||||
|
|
||||||
|
return get_file_name
|
||||||
|
|
||||||
|
|
||||||
|
def file_content(root: str, metatree: MetaTree, processor_chains: ProcessorChains, contcache: Dict) -> Callable:
|
||||||
|
def get_file_content(file_name: str) -> Iterable:
|
||||||
|
if file_name in contcache:
|
||||||
|
return contcache[file_name]
|
||||||
|
metadata = metatree.get_metadata(file_name)
|
||||||
|
chain = processor_chains.get_chain_for_filename(os.path.join(root, file_name), ctx=metadata)
|
||||||
|
contcache[file_name] = chain.output
|
||||||
|
return chain.output
|
||||||
|
|
||||||
|
return get_file_content
|
||||||
|
|
||||||
|
|
||||||
|
def file_metadata(metatree: MetaTree) -> Callable:
|
||||||
|
def get_file_metadata(file_name: str) -> Dict:
|
||||||
|
return metatree.get_metadata(file_name)
|
||||||
|
|
||||||
|
return get_file_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def time_iso8601(timezone: str) -> Callable:
|
||||||
|
tz = pytz.timezone(timezone)
|
||||||
|
|
||||||
|
def get_time_iso8601(time_t: Union[int, float]) -> str:
|
||||||
|
return datetime.datetime.fromtimestamp(time_t, tz).isoformat("T")
|
||||||
|
|
||||||
|
return get_time_iso8601
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
class TestProcessChain:
|
||||||
|
def test_process_chain(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_processor_chain(self):
|
||||||
|
pass
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
import mimetypes
|
||||||
|
import os
|
||||||
|
|
||||||
|
from typing import Dict, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def merge_dicts(dict_a: Dict, dict_b: Dict) -> Dict:
|
||||||
|
"""Merge two dictionaries.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
dict_a (dict): The dictionary to use as the base.
|
||||||
|
dict_b (dict): The dictionary to update the values with.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict: A new merged dictionary.
|
||||||
|
|
||||||
|
"""
|
||||||
|
dict_z = dict_a.copy()
|
||||||
|
dict_z.update(dict_b)
|
||||||
|
return dict_z
|
||||||
|
|
||||||
|
|
||||||
|
def guess_mime(path: str) -> Optional[str]:
|
||||||
|
"""Guess the mime type for a given path.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
root (str): the root path of the file tree
|
||||||
|
path (str): the sub-path within the file tree
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: the guessed mime-type
|
||||||
|
|
||||||
|
"""
|
||||||
|
mtypes = mimetypes.guess_type(path)
|
||||||
|
ftype = None
|
||||||
|
if os.path.isdir(path):
|
||||||
|
ftype = "directory"
|
||||||
|
elif os.access(path, os.F_OK) and mtypes[0]:
|
||||||
|
ftype = mtypes[0]
|
||||||
|
else:
|
||||||
|
ftype = "application/octet-stream"
|
||||||
|
return ftype
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
"""Package configuration."""
|
||||||
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
|
LONG_DESCRIPTION = """Pixywerk 2 is a DWIM filesystem based static site generator."""
|
||||||
|
|
||||||
|
INSTALL_REQUIRES = ["yaml-1.3", "markdown", "jstyleson", "jinja2"]
|
||||||
|
|
||||||
|
# Extra dependencies
|
||||||
|
EXTRAS_REQUIRE = {
|
||||||
|
# Test dependencies
|
||||||
|
"tests": [
|
||||||
|
"black",
|
||||||
|
"bandit>=1.1.0",
|
||||||
|
"flake8>=3.2.1",
|
||||||
|
"mypy>=0.470",
|
||||||
|
"prospector[with_everything]>=0.12.4",
|
||||||
|
"pytest-cov>=1.8.0",
|
||||||
|
"pytest-xdist>=1.15.0",
|
||||||
|
"pytest>=3.0.3",
|
||||||
|
"sphinx_rtd_theme>=0.1.6",
|
||||||
|
"sphinx-argparse>=0.1.15",
|
||||||
|
"Sphinx>=1.4.9",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
SETUP_REQUIRES = ["pytest-runner>=2.7.1", "setuptools_scm>=1.15.0"]
|
||||||
|
setup(
|
||||||
|
author="Cassowary Rusnov",
|
||||||
|
author_email="rusnovn@gmail.com",
|
||||||
|
classifiers=[
|
||||||
|
"Development Status :: 1 - Pre-alpha",
|
||||||
|
"Environment :: Console",
|
||||||
|
"License :: OSI Approved :: MIT",
|
||||||
|
"Operating System :: POSIX :: Linux",
|
||||||
|
"Programming Language :: Python :: 3.6",
|
||||||
|
"Programming Language :: Python :: 3.7",
|
||||||
|
"Programming Language :: Python :: 3 :: Only",
|
||||||
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||||
|
],
|
||||||
|
description="A filesystem-based DWIM website generator / CMS",
|
||||||
|
# entry_points={
|
||||||
|
# 'console_scripts': [
|
||||||
|
# 'cookbook = spicerack.cookbook:main',
|
||||||
|
# ],
|
||||||
|
# },
|
||||||
|
extras_require=EXTRAS_REQUIRE,
|
||||||
|
install_requires=INSTALL_REQUIRES,
|
||||||
|
keywords=["wmf", "automation", "orchestration"],
|
||||||
|
license="MIT",
|
||||||
|
long_description=LONG_DESCRIPTION,
|
||||||
|
name="pixywerk2", # Must be the same used for __version__ in __init__.py
|
||||||
|
packages=find_packages(exclude=["*.tests", "*.tests.*"]),
|
||||||
|
platforms=["GNU/Linux"],
|
||||||
|
setup_requires=SETUP_REQUIRES,
|
||||||
|
use_scm_version=True,
|
||||||
|
url="https://git.antpanethon.com/cas/pixywerk2",
|
||||||
|
zip_safe=False,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
[tox]
|
||||||
|
envlist=py{36,37}-{code-quality, unit} #, py37-sphinx
|
||||||
|
skipsdist = true
|
||||||
|
|
||||||
|
[testenv]
|
||||||
|
setenv =
|
||||||
|
LANG = en_US.UTF-8
|
||||||
|
deps = .[tests]
|
||||||
|
commands =
|
||||||
|
unit: py.test --strict --cov-report=term-missing --cov=pixywerk2 pixywerk2/tests/unit {posargs}
|
||||||
|
code-quality: flake8 pixywerk2
|
||||||
|
code-quality: black -l 120 --check pixywerk2
|
||||||
|
code-quality: - prospector -A
|
||||||
|
code-quality: - mypy --ignore-missing-imports pixywerk2
|
||||||
|
# sphinx: python setup.py build_sphinx -b html
|
||||||
|
# sphinx: python setup.py build_sphinx -b man
|
||||||
|
basepython =
|
||||||
|
py36: python3.6
|
||||||
|
py37: python3.7
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
max-line-length = 120
|
||||||
Loading…
Reference in New Issue
