71 lines
1.7 KiB
Python
71 lines
1.7 KiB
Python
import logging
|
|
import urllib.parse
|
|
|
|
import tqdm # type: ignore
|
|
|
|
from typing import List, Set
|
|
from pathlib import Path
|
|
|
|
"""
|
|
Miscellaneous utilities and constants.
|
|
"""
|
|
|
|
|
|
PROBABLY_BINARY_EXTENSIONS: List[str] = [
|
|
".ico",
|
|
".ppm",
|
|
".pnm",
|
|
".png",
|
|
".gz",
|
|
".svg",
|
|
".pgm",
|
|
".gih",
|
|
".gbr",
|
|
]
|
|
|
|
|
|
class TqdmLoggingHandler(logging.Handler):
|
|
"""
|
|
A simple logging wrapper that won't clobber TQDM's progress bar.
|
|
"""
|
|
def __init__(self, level=logging.NOTSET):
|
|
super().__init__(level)
|
|
|
|
def emit(self, record):
|
|
try:
|
|
msg = self.format(record)
|
|
tqdm.tqdm.write(msg)
|
|
self.flush()
|
|
except Exception:
|
|
self.handleError(record)
|
|
|
|
|
|
## Utility Functions
|
|
def is_binary(path: Path, amount: int = 256) -> bool:
|
|
"""
|
|
A pretty sloppy way to determine if a file is binary or not. Takes a path and returns true/false. Bigger
|
|
amount means more accurate (more of the file read).
|
|
"""
|
|
try:
|
|
b = path.open().read(amount)
|
|
return False
|
|
except UnicodeDecodeError:
|
|
return True
|
|
|
|
def reverse_domain(url: str) -> str:
|
|
"""
|
|
Create a 'reverse domain' used for labeling objects and whatnot.
|
|
"""
|
|
p = urllib.parse.urlparse(url)
|
|
domain = p.netloc
|
|
return '.'.join(reversed(domain.split('.')))
|
|
|
|
def find_file(fname: str, searchpaths: Set[Path]) -> Path:
|
|
"""
|
|
Return a path object for the first file we find in the search paths with the filename or raise FileNotFound error.
|
|
"""
|
|
for path in searchpaths:
|
|
if (path / fname).exists():
|
|
return (path / fname)
|
|
raise FileNotFoundError(f"can't find {fname} in searchpaths")
|
