fix: Remove flat-cache dependency (#7636)

Author: Jason3S

packages/cspell/package.json

@@ -88,7 +88,6 @@
     "@cspell/cspell-types": "workspace:*",
     "@cspell/dynamic-import": "workspace:*",
     "@cspell/url": "workspace:*",
-    "@types/flat-cache": "^2.0.2",
     "chalk": "^5.4.1",
     "chalk-template": "^1.1.0",
     "commander": "^14.0.0",
@@ -99,7 +98,7 @@
     "cspell-io": "workspace:*",
     "cspell-lib": "workspace:*",
     "fast-json-stable-stringify": "^2.1.0",
-    "flat-cache": "^5.0.0",
+    "flatted": "^3.3.3",
     "semver": "^7.7.2",
     "tinyglobby": "^0.2.14"
   },

packages/cspell/src/util/cache/file-entry-cache/file-entry-cache.ts

@@ -0,0 +1,273 @@
+import crypto from 'node:crypto';
+import type { Stats } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { FlatCache } from './flatCache.js';
+import { loadCacheFile as loadFlatCache } from './flatCache.js';
+
+export async function createFromFile(
+    filePath: string,
+    useChecksum?: boolean,
+    currentWorkingDir?: string,
+): Promise<FileEntryCache> {
+    const cache = await loadFlatCache<Meta>(filePath);
+    const fec = new ImplFileEntryCache(cache, useChecksum ?? false, currentWorkingDir);
+    await fec.removeNotFoundFiles();
+    return fec;
+}
+
+class ImplFileEntryCache implements FileEntryCache {
+    readonly cache: FlatCache<Meta>;
+    readonly useChecksum: boolean;
+    readonly #normalizedEntries: Map<string, CacheEntry> = new Map();
+
+    /**
+     * To enable relative paths as the key with current working directory
+     */
+    readonly currentWorkingDir: string | undefined;
+
+    constructor(cache: FlatCache<Meta>, useChecksum?: boolean, currentWorkingDir?: string) {
+        this.cache = cache;
+        this.useChecksum = useChecksum || false;
+        this.currentWorkingDir = currentWorkingDir;
+    }
+
+    async removeNotFoundFiles() {
+        // Remove not found entries
+        for (const fPath of this.cache.keys()) {
+            try {
+                const filePath = this.resolveKeyToFile(fPath);
+                await fs.stat(filePath);
+            } catch (error) {
+                if (isNodeError(error) && error.code === 'ENOENT') {
+                    this.cache.removeKey(fPath);
+                }
+            }
+        }
+    }
+
+    /**
+     * Given a buffer, calculate md5 hash of its content.
+     * @param  buffer buffer to calculate hash on
+     * @return content hash digest
+     */
+    #getHash(buffer: Buffer | string): string {
+        return crypto.createHash('md5').update(buffer).digest('hex');
+    }
+
+    async getFileDescriptor(file: string): Promise<FileDescriptor> {
+        let fstat: Stats;
+
+        try {
+            fstat = await fs.stat(file);
+        } catch (error) {
+            this.#removeEntry(file);
+            return { key: file, notFound: true, err: toError(error) };
+        }
+
+        if (this.useChecksum) {
+            return this.#getFileDescriptorUsingChecksum(file);
+        }
+
+        return this.#getFileDescriptorUsingMtimeAndSize(file, fstat);
+    }
+
+    #getFileDescriptorUsingMtimeAndSize(file: string, fstat: Stats): FileDescriptor {
+        const key = this.#getFileKey(file);
+        let meta = this.cache.get(key);
+        const cacheExists = !!meta;
+
+        const cSize = fstat.size;
+        const cTime = fstat.mtime.getTime();
+
+        let isDifferentDate;
+        let isDifferentSize;
+
+        if (meta) {
+            isDifferentDate = cTime !== meta.mtime;
+            isDifferentSize = cSize !== meta.size;
+        } else {
+            meta = { size: cSize, mtime: cTime };
+        }
+
+        const nEntry = {
+            key,
+            changed: !cacheExists || isDifferentDate || isDifferentSize,
+            meta,
+        };
+
+        this.#normalizedEntries.set(key, nEntry);
+
+        return nEntry;
+    }
+
+    async #getFileDescriptorUsingChecksum(file: string): Promise<FileDescriptor> {
+        const key = this.#getFileKey(file);
+        let meta = this.cache.get(key);
+        const cacheExists = !!meta;
+
+        let contentBuffer;
+        try {
+            contentBuffer = await fs.readFile(file);
+        } catch {
+            contentBuffer = '';
+        }
+
+        let isDifferent = true;
+        const hash = this.#getHash(contentBuffer);
+
+        if (meta) {
+            isDifferent = hash !== meta.hash;
+        } else {
+            meta = { hash };
+        }
+
+        const nEntry = {
+            key,
+            changed: !cacheExists || isDifferent,
+            meta,
+        };
+
+        this.#normalizedEntries.set(key, nEntry);
+
+        return nEntry;
+    }
+
+    /**
+     * Remove an entry from the file-entry-cache. Useful to force the file to still be considered
+     * modified the next time the process is run
+     */
+    #removeEntry(file: string): void {
+        const key = this.#getFileKey(file);
+        this.#normalizedEntries.delete(key);
+        this.cache.removeKey(key);
+    }
+
+    /**
+     * Deletes the cache file from the disk and clears the memory cache
+     */
+    async destroy(): Promise<void> {
+        this.#normalizedEntries.clear();
+        await this.cache.destroy();
+    }
+
+    async #getMetaForFileUsingCheckSum(cacheEntry: CacheEntry): Promise<Meta> {
+        const filePath = this.resolveKeyToFile(cacheEntry.key);
+        const contentBuffer = await fs.readFile(filePath);
+        const hash = this.#getHash(contentBuffer);
+        const meta: Meta = { ...cacheEntry.meta, hash };
+        delete meta.size;
+        delete meta.mtime;
+        return meta;
+    }
+
+    async #getMetaForFileUsingMtimeAndSize(cacheEntry: CacheEntry): Promise<Meta> {
+        const filePath = this.resolveKeyToFile(cacheEntry.key);
+        const stat = await fs.stat(filePath);
+        const meta = { ...cacheEntry.meta, size: stat.size, mtime: stat.mtime.getTime() };
+        delete meta.hash;
+        return meta;
+    }
+
+    /**
+     * Sync the files and persist them to the cache
+     */
+    async reconcile(): Promise<void> {
+        await this.removeNotFoundFiles();
+
+        for (const [entryKey, cacheEntry] of this.#normalizedEntries.entries()) {
+            try {
+                const meta = this.useChecksum
+                    ? await this.#getMetaForFileUsingCheckSum(cacheEntry)
+                    : await this.#getMetaForFileUsingMtimeAndSize(cacheEntry);
+                this.cache.set(entryKey, meta);
+            } catch (error) {
+                // If the file does not exists we don't save it
+                // other errors are just thrown
+                if (!isNodeError(error) || error.code !== 'ENOENT') {
+                    throw error;
+                }
+            }
+        }
+
+        this.cache.save();
+    }
+
+    resolveKeyToFile(entryKey: string): string {
+        if (this.currentWorkingDir) {
+            return path.resolve(this.currentWorkingDir, entryKey);
+        }
+        return entryKey;
+    }
+
+    #getFileKey(file: string): string {
+        if (this.currentWorkingDir && path.isAbsolute(file)) {
+            return normalizePath(path.relative(this.currentWorkingDir, file));
+        }
+        return normalizePath(file);
+    }
+}
+
+interface NodeError extends Error {
+    code?: string;
+}
+
+function isNodeError(error: unknown): error is NodeError {
+    return typeof error === 'object' && error !== null && 'code' in error;
+}
+
+function toError(error: unknown): Error {
+    if (error instanceof Error) {
+        return error;
+    }
+    if (typeof error === 'string') {
+        return new Error(error);
+    }
+    return new Error('Unknown error', { cause: error });
+}
+
+export interface AnalyzedFilesInfo {
+    readonly changedFiles: string[];
+    readonly notFoundFiles: string[];
+    readonly notChangedFiles: string[];
+}
+
+interface UserData {
+    data?: unknown;
+}
+
+interface Meta extends UserData {
+    size?: number | undefined;
+    mtime?: number | undefined;
+    hash?: string | undefined;
+}
+
+interface CacheEntry {
+    key: string;
+    notFound?: boolean;
+    err?: Error | undefined;
+    changed?: boolean | undefined;
+    meta?: Meta | undefined;
+}
+
+export type FileDescriptor = Readonly<CacheEntry>;
+
+export interface FileEntryCache {
+    getFileDescriptor(file: string): Promise<FileDescriptor>;
+
+    /**
+     * Deletes the cache file from the disk and clears the memory cache
+     */
+    destroy(): Promise<void>;
+
+    /**
+     * Sync the files and persist them to the cache
+     */
+    reconcile(): Promise<void>;
+}
+
+export function normalizePath(filePath: string): string {
+    if (path.sep === '/') return filePath;
+    return filePath.split(path.sep).join('/');
+}

packages/cspell/src/util/cache/file-entry-cache/flatCache.ts

@@ -0,0 +1,67 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { parse, stringify } from 'flatted';
+
+export class FlatCache<T> {
+    #cache: Map<string, T>;
+
+    constructor(readonly cacheFilename: string) {
+        this.#cache = new Map<string, T>();
+    }
+
+    keys(): MapIterator<string> {
+        return this.#cache.keys();
+    }
+
+    set(key: string, value: T): this {
+        this.#cache.set(key, value);
+        return this;
+    }
+
+    removeKey(key: string): void {
+        this.#cache.delete(key);
+    }
+
+    get(key: string): T | undefined {
+        return this.#cache.get(key);
+    }
+
+    async load(ifFound: boolean = true): Promise<this> {
+        this.#cache.clear();
+        try {
+            const content = await fs.readFile(this.cacheFilename, 'utf8');
+            this.#cache = new Map<string, T>(Object.entries(parse(content)));
+        } catch (error) {
+            if (!ifFound) {
+                throw error;
+            }
+        }
+        return this;
+    }
+
+    async save(): Promise<void> {
+        const dir = path.dirname(this.cacheFilename);
+        await fs.mkdir(dir, { recursive: true });
+        const content = stringify(Object.fromEntries(this.#cache.entries()));
+        await fs.writeFile(this.cacheFilename, content, 'utf8');
+    }
+
+    /**
+     * Clear the cache and remove the cache file from disk.
+     */
+    async destroy(): Promise<void> {
+        this.#cache.clear();
+        try {
+            await fs.unlink(this.cacheFilename);
+        } catch {
+            // Ignore errors when deleting the cache file.
+            // It may not exist or may not be writable.
+        }
+    }
+}
+
+export function loadCacheFile<T>(file: string): Promise<FlatCache<T>> {
+    const cache = new FlatCache<T>(file);
+    return cache.load();
+}