feat: add gzip decompression for in-memory dataset loading

Mearman · Mearman · commit 44582d8c55a3 · 2026-01-17T23:26:10.000Z
add universal gzip decompression that works in both browser and node.js:

- browser: uses native decompressionstream api (chrome 80+, firefox 113+)
- node.js: uses built-in zlib module (all versions)

loadBenchmarkFromUrl now auto-detects .gz urls and decompresses in memory,
enabling direct loading from sources like snap without local file download.

new exports:
- decompressGzip: decompress uint8array to string
- fetchAndDecompressGzip: fetch and decompress in one step
- fetchWithAutoDecompress: auto-detect compression from url
- isGzipUrl: check if url ends with .gz/.gzip
diff --git a/src/experiments/evaluation/fixtures/benchmark-datasets.ts b/src/experiments/evaluation/fixtures/benchmark-datasets.ts
@@ -13,7 +13,7 @@ import { dirname,resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 
 import { Graph } from "../../../algorithms/graph/graph";
-import { type LoadedEdge,loadEdgeList, type LoadedNode } from "../loaders/index";
+import { fetchWithAutoDecompress, type LoadedEdge,loadEdgeList, type LoadedNode } from "../loaders/index";
 
 // ============================================================================
 // Types
@@ -296,30 +296,32 @@ export const loadAllBenchmarks = async (benchmarksRoot?: string): Promise<Map<st
 /**
  * Load a benchmark dataset from a URL.
  *
- * This function works in both browser and Node.js environments using fetch.
- * Use this when you have a direct URL to the edge list file.
+ * This function works in both browser and Node.js environments.
+ * Automatically handles gzip-compressed files (.gz extension).
  *
- * @param url - URL to the edge list file
+ * @param url - URL to the edge list file (can be .txt or .txt.gz)
  * @param meta - Dataset metadata (for parsing configuration)
  * @returns Loaded benchmark with graph and metadata
  * @throws Error if fetch fails or parsing fails
  *
  * @example
  * ```typescript
+ * // Plain text file
  * const benchmark = await loadBenchmarkFromUrl(
  *   'https://raw.githubusercontent.com/user/repo/main/data/karate.edges',
  *   KARATE
  * );
+ *
+ * // Gzip-compressed file (automatically decompressed)
+ * const compressed = await loadBenchmarkFromUrl(
+ *   'https://snap.stanford.edu/data/facebook_combined.txt.gz',
+ *   FACEBOOK
+ * );
  * ```
  */
 export const loadBenchmarkFromUrl = async (url: string, meta: BenchmarkDatasetMeta): Promise<LoadedBenchmark> => {
-	const response = await fetch(url);
-
-	if (!response.ok) {
-		throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-	}
-
-	const content = await response.text();
+	// Auto-detect and handle gzip compression
+	const content = await fetchWithAutoDecompress(url);
 
 	const result = loadEdgeList(content, {
 		directed: meta.directed,
diff --git a/src/experiments/evaluation/index.ts b/src/experiments/evaluation/index.ts
@@ -86,7 +86,12 @@ export {
 
 // Export graph loaders
 export {
+	// Decompression utilities
+	decompressGzip,
 	type EdgeListConfig,
+	fetchAndDecompressGzip,
+	fetchWithAutoDecompress,
+	isGzipUrl,
 	type LoadedEdge,
 	loadEdgeList,
 	type LoadedNode,
diff --git a/src/experiments/evaluation/loaders/decompress.ts b/src/experiments/evaluation/loaders/decompress.ts
@@ -0,0 +1,158 @@
+/**
+ * Universal decompression utilities for browser and Node.js environments.
+ *
+ * Supports gzip decompression using:
+ * - Node.js: Built-in `zlib` module (works in all Node versions)
+ * - Browser: Native `DecompressionStream` API (Chrome 80+, Firefox 113+, Safari 16.4+)
+ *
+ * No external dependencies required.
+ */
+
+/**
+ * Check if running in Node.js environment.
+ */
+const isNode = (): boolean =>
+	typeof process !== "undefined" &&
+	process.versions?.node != undefined;
+
+/**
+ * Check if DecompressionStream is available (modern browsers and Node 18+).
+ */
+const hasDecompressionStream = (): boolean =>
+	typeof DecompressionStream !== "undefined";
+
+/**
+ * Decompress gzip data using Node.js zlib module.
+ *
+ * @param data - Compressed data as Uint8Array
+ * @returns Decompressed string
+ */
+const decompressWithZlib = async (data: Uint8Array): Promise<string> => {
+	// Dynamic import to avoid bundling issues in browser builds
+	const { gunzip } = await import("node:zlib");
+	const { promisify } = await import("node:util");
+
+	const gunzipAsync = promisify(gunzip);
+	const decompressed = await gunzipAsync(Buffer.from(data));
+	return decompressed.toString("utf-8");
+};
+
+/**
+ * Decompress gzip data using browser DecompressionStream API.
+ *
+ * @param data - Compressed data as Uint8Array
+ * @returns Decompressed string
+ */
+const decompressWithStream = async (data: Uint8Array): Promise<string> => {
+	// Create a new ArrayBuffer copy to ensure compatibility
+	const buffer = new ArrayBuffer(data.byteLength);
+	new Uint8Array(buffer).set(data);
+	const blob = new Blob([buffer]);
+	const stream = blob.stream();
+	const decompressedStream = stream.pipeThrough(new DecompressionStream("gzip"));
+	const response = new Response(decompressedStream);
+	return response.text();
+};
+
+/**
+ * Decompress gzip-compressed data to a string.
+ *
+ * Automatically uses the best available method for the current environment:
+ * - Node.js: Uses built-in zlib module
+ * - Browser: Uses native DecompressionStream API
+ *
+ * @param data - Gzip-compressed data as Uint8Array
+ * @returns Promise resolving to decompressed UTF-8 string
+ * @throws Error if decompression fails or no decompression method is available
+ *
+ * @example
+ * ```typescript
+ * const response = await fetch('https://example.com/data.gz');
+ * const compressed = new Uint8Array(await response.arrayBuffer());
+ * const text = await decompressGzip(compressed);
+ * ```
+ */
+export const decompressGzip = async (data: Uint8Array): Promise<string> => {
+	if (isNode()) {
+		return decompressWithZlib(data);
+	}
+
+	if (hasDecompressionStream()) {
+		return decompressWithStream(data);
+	}
+
+	throw new Error(
+		"No decompression method available. " +
+			"Browser requires DecompressionStream API (Chrome 80+, Firefox 113+, Safari 16.4+). " +
+			"Consider using a polyfill like 'pako' for older browsers."
+	);
+};
+
+/**
+ * Fetch and decompress gzip content from a URL.
+ *
+ * @param url - URL to fetch gzip-compressed content from
+ * @returns Promise resolving to decompressed UTF-8 string
+ * @throws Error if fetch fails or decompression fails
+ *
+ * @example
+ * ```typescript
+ * const text = await fetchAndDecompressGzip('https://snap.stanford.edu/data/facebook_combined.txt.gz');
+ * ```
+ */
+export const fetchAndDecompressGzip = async (url: string): Promise<string> => {
+	const response = await fetch(url);
+
+	if (!response.ok) {
+		throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+	}
+
+	const arrayBuffer = await response.arrayBuffer();
+	const data = new Uint8Array(arrayBuffer);
+
+	return decompressGzip(data);
+};
+
+/**
+ * Detect if a URL points to a gzip-compressed file based on extension.
+ *
+ * @param url - URL to check
+ * @returns True if URL ends with .gz or .gzip
+ */
+export const isGzipUrl = (url: string): boolean => {
+	const lowered = url.toLowerCase();
+	return lowered.endsWith(".gz") || lowered.endsWith(".gzip");
+};
+
+/**
+ * Fetch content from URL, automatically decompressing if gzip.
+ *
+ * Detects compression based on:
+ * 1. URL extension (.gz, .gzip)
+ * 2. Content-Encoding header
+ *
+ * @param url - URL to fetch content from
+ * @returns Promise resolving to text content (decompressed if needed)
+ *
+ * @example
+ * ```typescript
+ * // Automatically handles both compressed and uncompressed URLs
+ * const text1 = await fetchWithAutoDecompress('https://example.com/data.txt');
+ * const text2 = await fetchWithAutoDecompress('https://example.com/data.txt.gz');
+ * ```
+ */
+export const fetchWithAutoDecompress = async (url: string): Promise<string> => {
+	// If URL indicates gzip, fetch as binary and decompress
+	if (isGzipUrl(url)) {
+		return fetchAndDecompressGzip(url);
+	}
+
+	// Otherwise, fetch as text (browser/fetch handles Content-Encoding automatically)
+	const response = await fetch(url);
+
+	if (!response.ok) {
+		throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+	}
+
+	return response.text();
+};
diff --git a/src/experiments/evaluation/loaders/decompress.unit.test.ts b/src/experiments/evaluation/loaders/decompress.unit.test.ts
@@ -0,0 +1,116 @@
+/**
+ * Unit tests for decompression utilities
+ */
+
+import { gzipSync } from "node:zlib";
+
+import { describe, expect, it } from "vitest";
+
+import { decompressGzip, fetchWithAutoDecompress, isGzipUrl } from "./decompress";
+
+describe("Decompression Utilities", () => {
+	describe("isGzipUrl", () => {
+		it("should detect .gz extension", () => {
+			expect(isGzipUrl("https://example.com/data.txt.gz")).toBe(true);
+			expect(isGzipUrl("https://example.com/data.gz")).toBe(true);
+			expect(isGzipUrl("/path/to/file.gz")).toBe(true);
+		});
+
+		it("should detect .gzip extension", () => {
+			expect(isGzipUrl("https://example.com/data.gzip")).toBe(true);
+			expect(isGzipUrl("/path/to/file.gzip")).toBe(true);
+		});
+
+		it("should be case insensitive", () => {
+			expect(isGzipUrl("https://example.com/data.GZ")).toBe(true);
+			expect(isGzipUrl("https://example.com/data.GZIP")).toBe(true);
+			expect(isGzipUrl("https://example.com/data.Gz")).toBe(true);
+		});
+
+		it("should return false for non-gzip URLs", () => {
+			expect(isGzipUrl("https://example.com/data.txt")).toBe(false);
+			expect(isGzipUrl("https://example.com/data.zip")).toBe(false);
+			expect(isGzipUrl("https://example.com/data.tar")).toBe(false);
+			expect(isGzipUrl("https://example.com/data")).toBe(false);
+		});
+	});
+
+	describe("decompressGzip", () => {
+		it("should decompress gzip data", async () => {
+			const originalText = "Hello, World! This is a test.";
+			const compressed = gzipSync(Buffer.from(originalText));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(originalText);
+		});
+
+		it("should handle multi-line content", async () => {
+			const originalText = "line1\nline2\nline3";
+			const compressed = gzipSync(Buffer.from(originalText));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(originalText);
+		});
+
+		it("should handle edge list format", async () => {
+			const edgeList = "1 2\n2 3\n3 4\n4 1";
+			const compressed = gzipSync(Buffer.from(edgeList));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(edgeList);
+			expect(result.split("\n")).toHaveLength(4);
+		});
+
+		it("should handle unicode content", async () => {
+			// Test with accented characters and CJK characters (no emoji per lint rules)
+			const unicodeText = "Hello World! \u65E5\u672C\u8A9E";
+			const compressed = gzipSync(Buffer.from(unicodeText));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(unicodeText);
+		});
+
+		it("should handle empty content", async () => {
+			const emptyText = "";
+			const compressed = gzipSync(Buffer.from(emptyText));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(emptyText);
+		});
+
+		it("should handle large content", async () => {
+			// Create a moderately large edge list (10k edges)
+			const lines: string[] = [];
+			for (let index = 0; index < 10_000; index++) {
+				lines.push(`${index} ${index + 1}`);
+			}
+			const largeText = lines.join("\n");
+			const compressed = gzipSync(Buffer.from(largeText));
+			const data = new Uint8Array(compressed);
+
+			const result = await decompressGzip(data);
+
+			expect(result).toBe(largeText);
+			expect(result.split("\n")).toHaveLength(10_000);
+		});
+	});
+
+	describe("fetchWithAutoDecompress", () => {
+		it("should export the function", () => {
+			expect(typeof fetchWithAutoDecompress).toBe("function");
+		});
+
+		// Network tests would require mocking fetch or using a test server
+		// These are integration tests that verify the function exists and has the right signature
+	});
+});
diff --git a/src/experiments/evaluation/loaders/index.ts b/src/experiments/evaluation/loaders/index.ts
@@ -4,6 +4,12 @@
  * Support for loading benchmark datasets in various formats.
  */
 
+export {
+	decompressGzip,
+	fetchAndDecompressGzip,
+	fetchWithAutoDecompress,
+	isGzipUrl,
+} from "./decompress";
 export {
 	type EdgeListConfig,
 	type LoadedEdge,
