first

node_modules/@11ty/eleventy-fetch/src/AssetCache.js

@@ -0,0 +1,270 @@
+const fs = require("node:fs");
+const path = require("node:path");
+const { DateCompare, createHashHexSync } = require("@11ty/eleventy-utils");
+
+const FileCache = require("./FileCache.js");
+const Sources = require("./Sources.js");
+
+const debugUtil = require("debug");
+const debug = debugUtil("Eleventy:Fetch");
+
+class AssetCache {
+	#source;
+	#hash;
+	#customFilename;
+	#cache;
+	#cacheDirectory;
+	#cacheLocationDirty = false;
+	#directoryManager;
+
+	constructor(source, cacheDirectory, options = {}) {
+		if(!Sources.isValidSource(source)) {
+			throw Sources.getInvalidSourceError(source);
+		}
+
+		let uniqueKey = AssetCache.getCacheKey(source, options);
+		this.uniqueKey = uniqueKey;
+		this.hash = AssetCache.getHash(uniqueKey, options.hashLength);
+
+		this.cacheDirectory = cacheDirectory || ".cache";
+		this.options = options;
+
+		this.defaultDuration = "1d";
+		this.duration = options.duration || this.defaultDuration;
+
+		// Compute the filename only once
+		if (typeof this.options.filenameFormat === "function") {
+			this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash));
+
+			if (typeof this.#customFilename !== "string" || this.#customFilename.length === 0) {
+				throw new Error(`The provided filenameFormat callback function needs to return valid filename characters.`);
+			}
+		}
+	}
+
+	log(message) {
+		if (this.options.verbose) {
+			console.log(`[11ty/eleventy-fetch] ${message}`);
+		} else {
+			debug(message);
+		}
+	}
+
+	static cleanFilename(filename) {
+		// Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk)
+		if (filename.match(/([\/\\<>:"|?*]+?)/)) {
+			let sanitizedFilename = filename.replace(/[\/\\<>:"|?*]+/g, "");
+			debug(
+				`[@11ty/eleventy-fetch] Some illegal characters were removed from the cache filename: ${filename} will be cached as ${sanitizedFilename}.`,
+			);
+			return sanitizedFilename;
+		}
+
+		return filename;
+	}
+
+	static getCacheKey(source, options) {
+		// RemoteAssetCache passes in a string here, which skips this check (requestId is already used upstream)
+		if (Sources.isValidComplexSource(source)) {
+			if(options.requestId) {
+				return options.requestId;
+			}
+
+			if(typeof source.toString === "function") {
+				// 	return source.toString();
+				let toStr = source.toString();
+				if(toStr !== "function() {}" && toStr !== "[object Object]") {
+					return toStr;
+				}
+			}
+
+			throw Sources.getInvalidSourceError(source);
+		}
+
+		return source;
+	}
+
+	// Defult hashLength also set in global options, duplicated here for tests
+	// v5.0+ key can be Array or literal
+	static getHash(key, hashLength = 30) {
+		if (!Array.isArray(key)) {
+			key = [key];
+		}
+
+		let result = createHashHexSync(...key);
+		return result.slice(0, hashLength);
+	}
+
+	get source() {
+		return this.#source;
+	}
+
+	set source(source) {
+		this.#source = source;
+	}
+
+	get hash() {
+		return this.#hash;
+	}
+
+	set hash(value) {
+		if (value !== this.#hash) {
+			this.#cacheLocationDirty = true;
+		}
+
+		this.#hash = value;
+	}
+
+	get cacheDirectory() {
+		return this.#cacheDirectory;
+	}
+
+	set cacheDirectory(dir) {
+		if (dir !== this.#cacheDirectory) {
+			this.#cacheLocationDirty = true;
+		}
+
+		this.#cacheDirectory = dir;
+	}
+
+	get cacheFilename() {
+		if (typeof this.#customFilename === "string" && this.#customFilename.length > 0) {
+			return this.#customFilename;
+		}
+
+		return `eleventy-fetch-${this.hash}`;
+	}
+
+	get rootDir() {
+		// Work in an AWS Lambda (serverless)
+		// https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html
+
+		// Bad: LAMBDA_TASK_ROOT is /var/task/ on AWS so we must use ELEVENTY_ROOT
+		// When using ELEVENTY_ROOT, cacheDirectory must be relative
+		// (we are bundling the cache files into the serverless function)
+		if (
+			process.env.LAMBDA_TASK_ROOT &&
+			process.env.ELEVENTY_ROOT &&
+			!this.cacheDirectory.startsWith("/")
+		) {
+			return path.resolve(process.env.ELEVENTY_ROOT, this.cacheDirectory);
+		}
+
+		// otherwise, it is recommended to use somewhere in /tmp/ for serverless (otherwise it won’t write)
+		return path.resolve(this.cacheDirectory);
+	}
+
+	get cachePath() {
+		return path.join(this.rootDir, this.cacheFilename);
+	}
+
+	get cache() {
+		if (!this.#cache || this.#cacheLocationDirty) {
+			let cache = new FileCache(this.cacheFilename, {
+				dir: this.rootDir,
+				source: this.source,
+			});
+			cache.setDefaultType(this.options.type);
+			cache.setDryRun(this.options.dryRun);
+			cache.setDirectoryManager(this.#directoryManager);
+
+			this.#cache = cache;
+			this.#cacheLocationDirty = false;
+		}
+		return this.#cache;
+	}
+
+	getDurationMs(duration = "0s") {
+		return DateCompare.getDurationMs(duration);
+	}
+
+	setDirectoryManager(manager) {
+		this.#directoryManager = manager;
+	}
+
+	async save(contents, type = "buffer", metadata = {}) {
+		if(!contents) {
+			throw new Error("save(contents) expects contents (was falsy)");
+		}
+
+		this.cache.set(type, contents, metadata);
+
+		// Dry-run handled downstream
+		this.cache.save();
+	}
+
+	getCachedContents() {
+		return this.cache.getContents();
+	}
+
+	getCachedValue() {
+		if(this.options.returnType === "response") {
+			return {
+				...this.cachedObject.metadata?.response,
+				body: this.getCachedContents(),
+				cache: "hit",
+			}
+		}
+
+		return this.getCachedContents();
+	}
+
+	getCachedTimestamp() {
+		return this.cachedObject?.cachedAt;
+	}
+
+	isCacheValid(duration = this.duration) {
+		if(!this.cachedObject || !this.cachedObject?.cachedAt) {
+			return false;
+		}
+
+		if(this.cachedObject?.type && DateCompare.isTimestampWithinDuration(this.cachedObject?.cachedAt, duration)) {
+			return this.cache.hasContents(this.cachedObject?.type); // check file system to make files haven’t been purged.
+		}
+
+		return false;
+	}
+
+	get cachedObject() {
+		return this.cache.get();
+	}
+
+	// Deprecated
+	needsToFetch(duration) {
+		return !this.isCacheValid(duration);
+	}
+
+	// This is only included for completenes—not on the docs.
+	async fetch(optionsOverride = {}) {
+		if (this.isCacheValid(optionsOverride.duration)) {
+			// promise
+			debug(`Using cached version of: ${this.uniqueKey}`);
+			return this.getCachedValue();
+		}
+
+		debug(`Saving ${this.uniqueKey} to ${this.cacheFilename}`);
+		await this.save(this.source, optionsOverride.type);
+
+		return this.source;
+	}
+
+	// for testing
+	hasAnyCacheFiles() {
+		for(let p of this.cache.getAllPossibleFilePaths()) {
+			if(fs.existsSync(p)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	// for testing
+	async destroy() {
+		await Promise.all(this.cache.getAllPossibleFilePaths().map(path => {
+			if (fs.existsSync(path)) {
+				return fs.unlinkSync(path);
+			}
+		}))
+	}
+}
+module.exports = AssetCache;

node_modules/@11ty/eleventy-fetch/src/DirectoryManager.js

@@ -0,0 +1,22 @@
+const fs = require("node:fs");
+const debugAssets = require("debug")("Eleventy:Assets");
+
+class DirectoryManager {
+  #dirs = new Set();
+
+  isCreated(dir) {
+    return this.#dirs.has(dir);
+  }
+
+  create(dir) {
+    if(this.isCreated(dir)) {
+      return;
+    }
+
+    this.#dirs.add(dir);
+    debugAssets("Creating directory %o", dir);
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+module.exports = DirectoryManager;

node_modules/@11ty/eleventy-fetch/src/ExistsCache.js

@@ -0,0 +1,24 @@
+const fs = require("node:fs");
+// const debug = require("debug")("Eleventy:Assets");
+
+class ExistsCache {
+	#checks = new Map();
+	#count = 0;
+
+	set(target, value) {
+		this.#checks.set(target, Boolean(value));
+	}
+
+	exists(target) {
+		if(this.#checks.has(target)) {
+			return this.#checks.get(target);
+		}
+
+		let exists = fs.existsSync(target);
+		this.#count++;
+		this.#checks.set(target, exists);
+		return exists;
+	}
+}
+
+module.exports = ExistsCache;

node_modules/@11ty/eleventy-fetch/src/FileCache.js

@@ -0,0 +1,237 @@
+const fs = require("node:fs");
+const path = require("node:path");
+const debugUtil = require("debug");
+const { parse } = require("flatted");
+
+const debug = debugUtil("Eleventy:Fetch");
+const debugAssets = debugUtil("Eleventy:Assets");
+
+const DirectoryManager = require("./DirectoryManager.js");
+const ExistsCache = require("./ExistsCache.js");
+
+let existsCache = new ExistsCache();
+
+class FileCache {
+	#source;
+	#directoryManager;
+	#metadata;
+	#defaultType;
+	#contents;
+	#dryRun = false;
+	#cacheDirectory = ".cache";
+	#savePending = false;
+	#counts = {
+		read: 0,
+		write: 0,
+	};
+
+	constructor(cacheFilename, options = {}) {
+		this.cacheFilename = cacheFilename;
+		if(options.dir) {
+			this.#cacheDirectory = options.dir;
+		}
+		if(options.source) {
+			this.#source = options.source;
+		}
+	}
+
+	setDefaultType(type) {
+		if(type) {
+			this.#defaultType = type;
+		}
+	}
+
+	setDryRun(val) {
+		this.#dryRun = Boolean(val);
+	}
+
+	setDirectoryManager(manager) {
+		this.#directoryManager = manager;
+	}
+
+	ensureDir() {
+		if (this.#dryRun || existsCache.exists(this.#cacheDirectory)) {
+			return;
+		}
+
+		if(!this.#directoryManager) {
+			// standalone fallback (for tests)
+			this.#directoryManager = new DirectoryManager();
+		}
+
+		this.#directoryManager.create(this.#cacheDirectory);
+	}
+
+	set(type, contents, extraMetadata = {}) {
+		this.#savePending = true;
+
+		this.#metadata = {
+			cachedAt: Date.now(),
+			type,
+			// source: this.#source,
+			metadata: extraMetadata,
+		};
+
+		this.#contents = contents;
+	}
+
+	get fsPath() {
+		return path.join(this.#cacheDirectory, this.cacheFilename);
+	}
+
+	getContentsPath(type) {
+		if(!type) {
+			throw new Error("Missing cache type for " + this.fsPath);
+		}
+
+		// normalize to storage type
+		if(type === "xml") {
+			type = "text";
+		} else if(type === "parsed-xml") {
+			type = "json";
+		}
+
+		return `${this.fsPath}.${type}`;
+	}
+
+	// only when side loaded (buffer content)
+	get contentsPath() {
+		return this.getContentsPath(this.#metadata?.type);
+	}
+
+	get() {
+		if(this.#metadata) {
+			return this.#metadata;
+		}
+
+		if(!existsCache.exists(this.fsPath)) {
+			return;
+		}
+
+		debug(`Fetching from cache ${this.fsPath}`);
+		if(this.#source) {
+			debugAssets("[11ty/eleventy-fetch] Reading via %o", this.#source);
+		} else {
+			debugAssets("[11ty/eleventy-fetch] Reading %o", this.fsPath);
+		}
+
+		this.#counts.read++;
+		let data = fs.readFileSync(this.fsPath, "utf8");
+
+		let json;
+		// Backwards compatibility with previous caches usingn flat-cache and `flatted`
+		if(data.startsWith(`[["1"],`)) {
+			let flattedParsed = parse(data);
+			if(flattedParsed?.[0]?.value) {
+				json = flattedParsed?.[0]?.value
+			}
+		} else {
+			json = JSON.parse(data);
+		}
+
+		this.#metadata = json;
+
+		return json;
+	}
+
+	_backwardsCompatGetContents(rawData, type) {
+		if (type === "json") {
+			return rawData.contents;
+		} else if (type === "text") {
+			return rawData.contents.toString();
+		}
+
+		// buffer
+		return Buffer.from(rawData.contents);
+	}
+
+	hasContents(type) {
+		if(this.#contents) {
+			return true;
+		}
+		if(this.get()?.contents) { // backwards compat with very old caches
+			return true;
+		}
+		return existsCache.exists(this.getContentsPath(type));
+	}
+
+	getType() {
+		return this.#metadata?.type || this.#defaultType;
+	}
+
+	getContents() {
+		if(this.#contents) {
+			return this.#contents;
+		}
+
+		let metadata = this.get();
+		// backwards compat with old caches
+		if(metadata?.contents) {
+			// already parsed, part of the top level file
+			let normalizedContent = this._backwardsCompatGetContents(this.get(), this.getType());
+			this.#contents = normalizedContent;
+			return normalizedContent;
+		}
+
+		if(!existsCache.exists(this.contentsPath)) {
+			return;
+		}
+
+		debug(`Fetching from cache ${this.contentsPath}`);
+		if(this.#source) {
+			debugAssets("[11ty/eleventy-fetch] Reading (side loaded) via %o", this.#source);
+		} else {
+			debugAssets("[11ty/eleventy-fetch] Reading (side loaded) %o", this.contentsPath);
+		}
+
+		// It is intentional to store contents in a separate file from the metadata: we don’t want to
+		// have to read the entire contents via JSON.parse (or otherwise) to check the cache validity.
+		this.#counts.read++;
+		let type = metadata?.type || this.getType();
+		let data = fs.readFileSync(this.contentsPath);
+		if (type === "json" || type === "parsed-xml") {
+			data = JSON.parse(data);
+		}
+		this.#contents = data;
+		return data;
+	}
+
+	save() {
+		if(this.#dryRun || !this.#savePending || this.#metadata && Object.keys(this.#metadata) === 0) {
+			return;
+		}
+
+		this.ensureDir(); // doesn’t add to counts (yet?)
+
+		// contents before metadata
+		debugAssets("[11ty/eleventy-fetch] Writing %o (side loaded) from %o", this.contentsPath, this.#source);
+
+		this.#counts.write++;
+		// the contents must exist before the cache metadata are saved below
+		let contents = this.#contents;
+		let type = this.getType();
+		if (type === "json" || type === "parsed-xml") {
+			contents = JSON.stringify(contents);
+		}
+		fs.writeFileSync(this.contentsPath, contents);
+		debug(`Writing ${this.contentsPath}`);
+
+		this.#counts.write++;
+		debugAssets("[11ty/eleventy-fetch] Writing %o from %o", this.fsPath, this.#source);
+		fs.writeFileSync(this.fsPath, JSON.stringify(this.#metadata), "utf8");
+		debug(`Writing ${this.fsPath}`);
+	}
+
+	// for testing
+	getAllPossibleFilePaths() {
+		let types = ["text", "buffer", "json"];
+		let paths = new Set();
+		paths.add(this.fsPath);
+		for(let type of types) {
+			paths.add(this.getContentsPath(type));
+		}
+		return Array.from(paths);
+	}
+}
+
+module.exports = FileCache;

node_modules/@11ty/eleventy-fetch/src/RemoteAssetCache.js

@@ -0,0 +1,235 @@
+const debugUtil = require("debug");
+const { parseXml } = require('@rgrove/parse-xml');
+
+const Sources = require("./Sources.js");
+const AssetCache = require("./AssetCache.js");
+
+const debug = debugUtil("Eleventy:Fetch");
+const debugAssets = debugUtil("Eleventy:Assets");
+
+class RemoteAssetCache extends AssetCache {
+	#queue;
+	#queuePromise;
+	#fetchPromise;
+	#lastFetchType;
+
+	constructor(source, cacheDirectory, options = {}) {
+		let requestId = RemoteAssetCache.getRequestId(source, options);
+		super(requestId, cacheDirectory, options);
+
+		this.source = source;
+		this.options = options;
+		this.displayUrl = RemoteAssetCache.convertUrlToString(source, options);
+		this.fetchCount = 0;
+	}
+
+	static getRequestId(source, options = {}) {
+		if (Sources.isValidComplexSource(source)) {
+			return this.getCacheKey(source, options);
+		}
+
+		if (options.removeUrlQueryParams) {
+			let cleaned = this.cleanUrl(source);
+			return this.getCacheKey(cleaned, options);
+		}
+
+		return this.getCacheKey(source, options);
+	}
+
+	static getCacheKey(source, options) {
+		let cacheKey = {
+			source: AssetCache.getCacheKey(source, options),
+		};
+
+		if(options.type === "xml" || options.type === "parsed-xml") {
+			cacheKey.type = options.type;
+		}
+
+		if (options.fetchOptions) {
+			if (options.fetchOptions.method && options.fetchOptions.method !== "GET") {
+				cacheKey.method = options.fetchOptions.method;
+			}
+			if (options.fetchOptions.body) {
+				cacheKey.body = options.fetchOptions.body;
+			}
+		}
+
+		if(Object.keys(cacheKey).length > 1) {
+			return JSON.stringify(cacheKey);
+		}
+
+		return cacheKey.source;
+	}
+
+	static cleanUrl(url) {
+		if(!Sources.isFullUrl(url)) {
+			return url;
+		}
+
+		let cleanUrl;
+		if(typeof url === "string" || typeof url.toString === "function") {
+			cleanUrl = new URL(url);
+		} else if(url instanceof URL) {
+			cleanUrl = url;
+		} else {
+			throw new Error("Invalid source for cleanUrl: " + url)
+		}
+
+		cleanUrl.search = new URLSearchParams([]);
+
+		return cleanUrl.toString();
+	}
+
+	static convertUrlToString(source, options = {}) {
+		// removes query params
+		source = RemoteAssetCache.cleanUrl(source);
+
+		let { formatUrlForDisplay } = options;
+		if (formatUrlForDisplay && typeof formatUrlForDisplay === "function") {
+			return "" + formatUrlForDisplay(source);
+		}
+
+		return "" + source;
+	}
+
+	async getResponseValue(response, type) {
+		if (type === "json") {
+			return response.json();
+		} else if (type === "text" || type === "xml") {
+			return response.text();
+		} else if(type === "parsed-xml") {
+			return parseXml(await response.text());
+		}
+		return Buffer.from(await response.arrayBuffer());
+	}
+
+	setQueue(queue) {
+		this.#queue = queue;
+	}
+
+	// Returns raw Promise
+	queue() {
+		if(!this.#queue) {
+			throw new Error("Missing `#queue` instance.");
+		}
+
+		if(!this.#queuePromise) {
+			// optionsOverride not supported on fetch here for re-use
+			this.#queuePromise = this.#queue.add(() => this.fetch()).catch((e) => {
+				this.#queuePromise = undefined;
+				throw e;
+			});
+		}
+
+		return this.#queuePromise;
+	}
+
+	isCacheValid(duration = undefined) {
+		// uses this.options.duration if not explicitly defined here
+		return super.isCacheValid(duration);
+	}
+
+	// if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur)
+	// used by Eleventy Image in disk cache checks.
+	wasLastFetchCacheHit() {
+		return this.#lastFetchType === "hit";
+	}
+
+	async #fetch(optionsOverride = {}) {
+		// Important: no disk writes when dryRun
+		// As of Fetch v4, reads are now allowed!
+		if (this.isCacheValid(optionsOverride.duration)) {
+			debug(`Cache hit for ${this.displayUrl}`);
+			this.#lastFetchType = "hit";
+			return super.getCachedValue();
+		}
+
+		this.#lastFetchType = "miss";
+
+		try {
+			let isDryRun = optionsOverride.dryRun || this.options.dryRun;
+			this.log(`Fetching ${this.displayUrl}`);
+
+			let body;
+			let metadata = {};
+			let type = optionsOverride.type || this.options.type;
+			if (typeof this.source === "object" && typeof this.source.then === "function") {
+				body = await this.source;
+			} else if (typeof this.source === "function") {
+				// sync or async function
+				body = await this.source();
+			} else {
+				let fetchOptions = optionsOverride.fetchOptions || this.options.fetchOptions || {};
+				if(!Sources.isFullUrl(this.source)) {
+					throw Sources.getInvalidSourceError(this.source);
+				}
+
+				this.fetchCount++;
+
+				debugAssets("[11ty/eleventy-fetch] Fetching %o", this.source);
+
+				// v5: now using global (Node-native or otherwise) fetch instead of node-fetch
+				let response;
+				let error;
+				try {
+					response = await fetch(this.source, fetchOptions);
+
+					if (response?.ok) {
+						metadata.response = {
+							url: response.url,
+							status: response.status,
+							headers:  Object.fromEntries(response.headers.entries()),
+						};
+
+						body = await this.getResponseValue(response, type);
+					}
+				} catch(e) {
+					error = e;
+				}
+
+				if(!response?.ok || error) {
+					let errorMessage = response?.status || response?.statusText ? ` (${response?.status}): ${response.statusText}` : `: ${error.message}`;
+					throw new Error(`Bad response for ${this.displayUrl}${errorMessage}`, {
+						cause: error || response
+					})
+				}
+			}
+
+			if (!isDryRun) {
+				await super.save(body, type, metadata);
+			}
+
+			if(this.options.returnType === "response") {
+				return {
+					...metadata.response,
+					body,
+					cache: "miss",
+				}
+			}
+
+			return body;
+		} catch (e) {
+			if (this.cachedObject && this.getDurationMs(this.duration) > 0) {
+				debug(`Error fetching ${this.displayUrl}. Message: ${e.message}`);
+				debug(`Failing gracefully with an expired cache entry.`);
+				return super.getCachedValue();
+			} else {
+				return Promise.reject(e);
+			}
+		}
+	}
+
+	// async but not explicitly declared for promise equality checks
+	// returns a Promise
+	async fetch(optionsOverride = {}) {
+		if(!this.#fetchPromise) {
+			// one at a time. clear when finished
+			this.#fetchPromise = this.#fetch(optionsOverride).finally(() => {
+				this.#fetchPromise = undefined;
+			});
+		}
+
+		return this.#fetchPromise;
+	}
+}
+module.exports = RemoteAssetCache;

node_modules/@11ty/eleventy-fetch/src/Sources.js

@@ -0,0 +1,50 @@
+class Sources {
+	static isFullUrl(url) {
+		try {
+			if(url instanceof URL) {
+				return true;
+			}
+
+			new URL(url);
+			return true;
+		} catch (e) {
+			// invalid url OR already a local path
+			return false;
+		}
+	}
+
+	static isValidSource(source) {
+		// String (url?)
+		if(typeof source === "string") {
+			return true;
+		}
+		if(this.isValidComplexSource(source)) {
+			return true;
+		}
+		return false;
+	}
+
+	static isValidComplexSource(source) {
+		// Async/sync Function
+		if(typeof source === "function") {
+			return true;
+		}
+		if(typeof source === "object") {
+			// Raw promise
+			if(typeof source.then === "function") {
+				return true;
+			}
+			// anything string-able
+			if(typeof source.toString === "function") {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	static getInvalidSourceError(source, errorCause) {
+		return new Error("Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: " + source, { cause: errorCause });
+	}
+}
+
+module.exports = Sources;