Updates to support HECKformat documents, and minor changes.

- Update copyright.
- Remove manifest.in since we're switching to PDM (and defaults moved
  to the module).
- Remove setup.py since we're switching to PDM.
- Remove chains.yaml, move data to the processor module.
- Fix passthrough in __main__
- Move main function to separate function to support PDM entrypoint.
- metadata.py: Extensive rework
  * Add heck support (lots of little changes to support it). (.heck files can
    replace .meta files)
  * Add yaml metadata support (.meta files can be yaml)
  * Some formatting changes.
  * Make metatree be a little easier to read by separating out
    functionality into extra functions
- processchain.py: Move chains.yaml to a structure internal.
- Add processors/process_heck.py to support the document side of HECKformat
- add pyproject.toml and embrace PDM.

heckweasel/metadata.py

@@ -6,15 +6,19 @@ import mimetypes
 import os
 import uuid
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
+import yaml
 
 import jstyleson
 
+import heckformat.parse
+
 from .utils import guess_mime
 
 # setup mimetypes with some extra ones
 mimetypes.init()
 mimetypes.add_type("text/html", "thtml")
 mimetypes.add_type("text/html", "cont")
+mimetypes.add_type("text/x-heckformat", "heck")
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +26,8 @@ logger = logging.getLogger(__name__)
 class MetaCacheMiss(Exception):
     """Raised on cache miss."""
 
+class MetaLoadError(Exception):
+    "Raised when metadata fails to load."
 
 class MetaCache:
     """This class provides an in-memory cache for metadata tree."""
@@ -50,10 +56,10 @@ class MetaCache:
             MetaCacheMiss: on missing key, or on aged out
 
         """
-        if key not in self._cache:
+        if (key not in self._cache):
             raise MetaCacheMiss("no item for key {}".format(key))
 
-        if self._cache[key][0] + self._max_age <= new_time_stamp:
+        if ((self._cache[key][0] + self._max_age) <= new_time_stamp):
             return self._cache[key][1]
 
         raise MetaCacheMiss("cache expired for key {}".format(key))
@@ -82,13 +88,47 @@ class MetaTree:
 
         """
         self._cache = MetaCache()
-        if default_metadata is None:
+        if (default_metadata is None):
             default_metadata = {}
         self._default_metadata = default_metadata
-        if root[-1] != "/":
+        if (root[-1] != "/"):
             root += "/"
         self._root = root
 
+    def _get_cache_key(self, fullpath: str):
+        cachekey = fullpath + '.meta'
+        if fullpath.endswith(".heck"):
+            cachekey = fullpath
+        elif os.path.isdir(fullpath):
+            cachekey = os.path.join(fullpath, ".meta")
+            if (not os.path.exists(cachekey)):
+                cachekey = os.path.join(fullpath, ".heck")
+
+        return cachekey
+
+    def _load_metadata(self, cachekey: str) -> Dict:
+        meta = {}
+        with open(cachekey, "r") as inf:
+            if cachekey.endswith(".heck"):
+                # raise NotImplemented("We don't yet support HECKformat")
+                with open(cachekey) as cachefile:
+                    h = heckformat.parse.load(cachefile)
+                    meta = h.flatten_replace()
+            else:
+                try:
+                    # try json load
+                    meta = jstyleson.load(inf)
+                except jstyleson.JSONDecodeError as exc:
+                    # try yaml load
+                    try:
+                        meta = yaml.load(inf)
+                    except yaml.parser.ParserError as exc2:
+                        # else either the yaml or json has an error
+                        me = MetaLoadError()
+                        exc2.__context__ = exc
+                        raise me from exc2
+            return meta
+
     def get_metadata(self, rel_path: str) -> Dict:
         """Retrieve the metadata for a given path
 
@@ -113,11 +153,10 @@ class MetaTree:
             fullpath = os.path.join(fullpath, pth)
             st = os.stat(fullpath)
 
-            if os.path.isdir(fullpath):
-                cachekey = os.path.join(fullpath, ".meta")
-            else:
-                cachekey = fullpath + ".meta"
+            cachekey = self._get_cache_key(fullpath)
+
             meta = cast(Dict, {})
+
             try:
                 st_meta = os.stat(cachekey)
                 meta = self._cache.get(cachekey, st_meta.st_mtime)
@@ -126,28 +165,40 @@ class MetaTree:
             except MetaCacheMiss:
                 meta = {}
 
-            if not meta and st_meta:
-                meta = jstyleson.load(open(cachekey, "r"))
+            # if we didn't get any meta from the cache, but the metafile exists, try loading it
+            if ((not meta) and st_meta):
+                meta = self._load_metadata(cachekey)
                 self._cache.put(cachekey, meta, st_meta.st_mtime)
 
-            if fullpath == ospath and "wildcard_metadata" in metablob:
+            # add whatever is in the metablob as 'wildcard_metadata' to the metadata if the filename
+            # matches the wildcards
+            if ((fullpath == ospath) and ("wildcard_metadata" in metablob)):
                 for wild in metablob["wildcard_metadata"]:
                     if fnmatch.fnmatch(pth, wild[0]):
                         metablob.update(wild[1])
 
             metablob.update(meta)
 
-        # return final dict
+        ### fill in all objective metadata
+        # containing directory and filename
         metablob["dir"], metablob["file_name"] = os.path.split(rel_path)
+        # path within the source tree
         metablob["file_path"] = rel_path
+        # the path relative to the output tree
         metablob["relpath"] = os.path.relpath("/", "/" + metablob["dir"])
+        # the UUID for this file
         metablob["uuid"] = uuid.uuid3(uuid.NAMESPACE_OID, metablob["uuid-oid-root"] + ospath)
+        # the pre-split components of the full path
         metablob["os-path"], _ = os.path.split(fullpath)
+        # the mime type we guessed for this file
         metablob["guessed-type"] = guess_mime(ospath)
+        # if the mime-type isn't overriden in the explicit metadata, we make it equal to the guessed type
         if "mime-type" not in metablob:
             metablob["mime-type"] = metablob["guessed-type"]
+        # the `stat` components
         metablob["stat"] = {}
         for stk in ("st_mtime", "st_ctime", "st_atime", "st_mode", "st_size", "st_ino"):
             metablob["stat"][stk.replace("st_", "")] = getattr(st, stk)
 
+        # return final dict
         return metablob