Add streaming JSON parser architecture

.cursor/commands/improvement_pass.md

@@ -34,15 +34,15 @@ hyperfine -w 10 -r 20 \
   | tee "benchmarks/benchmark_pre_${TAG}.txt"
 
 # Optional: PRE flamegraph for this pass's main scenario
-sudo make flamegraph-prod \
+sudo TAG="$TAG" make flamegraph-prod \
   FLAME_FILE="samples/security_big_sample.evtx" \
   DURATION=30 \
   FORMAT=json \
   BIN="$PRE"
 
-mv profile/flamegraph.svg "profile/flamegraph_${TAG}_${TS}_pre.svg" || true
-cp profile/top_leaf.txt "profile/top_leaf_${TAG}_${TS}_pre.txt" || true
-cp profile/top_titles.txt "profile/top_titles_${TAG}_${TS}_pre.txt" || true
+mv "profile/flamegraph_${TAG}.svg" "profile/flamegraph_${TAG}_${TS}_pre.svg" || true
+cp "profile/top_leaf_${TAG}.txt" "profile/top_leaf_${TAG}_${TS}_pre.txt" || true
+cp "profile/top_titles_${TAG}.txt" "profile/top_titles_${TAG}_${TS}_pre.txt" || true
 ```
 
 - **Use the PRE benchmark + flamegraph** to:
@@ -104,12 +104,12 @@ hyperfine -w 10 -r 20 \
   | tee "/workspace/benchmarks/benchmark_pair_${TAG}_${TS}.txt"
 
 # POST flamegraph for the same scenario
-OUT_DIR=/workspace/profile_post FORMAT=jsonl DURATION=30 \
-  /workspace/scripts/flamegraph_prod.sh "$POST"
+OUT_DIR=/workspace/profile_post FORMAT=json DURATION=30 BIN="$POST" \
+  /workspace/scripts/flamegraph_prod.sh
 
-mv /workspace/profile/flamegraph.svg "/workspace/profile_post/flamegraph_${TAG}_${TS}_post.svg" || true
-cp /workspace/profile/top_leaf.txt "/workspace/profile_post/top_leaf_${TAG}_${TS}_post.txt" || true
-cp /workspace/profile/top_titles.txt "/workspace/profile_post/top_titles_${TAG}_${TS}_post.txt" || true
+mv "/workspace/profile_post/flamegraph_${TAG}.svg" "/workspace/profile_post/flamegraph_${TAG}_${TS}_post.svg" || true
+cp "/workspace/profile_post/top_leaf_${TAG}.txt" "/workspace/profile_post/top_leaf_${TAG}_${TS}_post.txt" || true
+cp "/workspace/profile_post/top_titles_${TAG}.txt" "/workspace/profile_post/top_titles_${TAG}_${TS}_post.txt" || true
 
 echo "PRE:  $PRE"
 echo "POST: $POST"

.github/workflows/deploy-pages.yml

@@ -27,11 +27,9 @@ jobs:
           bun-version: latest
 
       - name: Set up Rust toolchain (stable) with wasm target
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: stable
-          target: wasm32-unknown-unknown
-          override: true
+          targets: wasm32-unknown-unknown
 
       - name: Install & cache wasm-pack
         uses: jetli/wasm-pack-action@v0.4.0

.github/workflows/test.yml

@@ -10,13 +10,10 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+      - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
+      - name: Run tests
+        run: cargo test

.gitignore

@@ -13,3 +13,8 @@ repomix-output.txt
 evtx-wasm/evtx-viewer/public/pkg
 # Samples are being copied by build scripts before deploying
 **/public/samples/
+
+profile/*
+binaries/*
+benchmarks/*
+.PRE_PATH

Makefile

@@ -0,0 +1,22 @@
+FLAME_FILE ?= samples/security_big_sample.evtx
+FORMAT ?= json
+DURATION ?= 30
+BIN ?= ./target/release/evtx_dump
+
+.PHONY: flamegraph-prod
+flamegraph-prod:
+	@echo "Building release binary with fast allocator..."
+	cargo build --release --features fast-alloc
+	@echo "Cleaning up previous trace files..."
+	@rm -rf cargo-flamegraph.trace
+	BIN="$(BIN)" FLAME_FILE="$(FLAME_FILE)" FORMAT="$(FORMAT)" DURATION="$(DURATION)" \
+		bash scripts/flamegraph_prod.sh
+
+.PHONY: compare-streaming-legacy
+compare-streaming-legacy:
+	@echo "Building comparison tool with fast allocator..."
+	cargo build --release --features fast-alloc --bin compare_streaming_legacy
+	@echo "Running legacy vs streaming JSON comparison..."
+	./target/release/compare_streaming_legacy $(FILE)
+
+

scripts/flamegraph_prod.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Simple production-style flamegraph helper using perf + inferno (Linux)
+# or cargo-flamegraph (macOS).
+# Intended to be invoked via `make flamegraph-prod` with environment
+# overrides, e.g.:
+#   FLAME_FILE=samples/security_big_sample.evtx \
+#   FORMAT=json \
+#   DURATION=30 \
+#   BIN=./target/release/evtx_dump \
+#   make flamegraph-prod
+#
+OS="$(uname -s || echo unknown)"
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+# Optional label for this run (used in output filenames).
+: "${TAG:=default}"
+
+: "${BIN:=$ROOT_DIR/target/release/evtx_dump}"
+: "${FLAME_FILE:=$ROOT_DIR/samples/security_big_sample.evtx}"
+: "${FORMAT:=json}"
+: "${DURATION:=30}"
+# For JSON formats, choose parser implementation: streaming | legacy.
+: "${JSON_PARSER:=streaming}"
+: "${OUT_DIR:=$ROOT_DIR/profile}"
+
+mkdir -p "$OUT_DIR"
+
+echo "Profiling"
+echo "  FLAME_FILE=$FLAME_FILE"
+echo "  FORMAT=$FORMAT"
+echo "  DURATION=${DURATION}s"
+echo "  OUT_DIR=$OUT_DIR"
+echo "  TAG=$TAG"
+
+# Map FORMAT to evtx_dump arguments.
+case "$FORMAT" in
+  json|jsonl)
+    # Use streaming JSON path by default; caller can change via JSON_PARSER env.
+    FMT_ARGS=(-t 1 -o "$FORMAT" --json-parser "$JSON_PARSER")
+    ;;
+  xml)
+    FMT_ARGS=(-t 1 -o xml)
+    ;;
+  *)
+    echo "warning: unknown FORMAT='$FORMAT', defaulting to json" >&2
+    FMT_ARGS=(-t 1 -o json --json-parser streaming)
+    ;;
+esac
+
+if [[ "$OS" == "Darwin" ]]; then
+  # macOS path: use cargo-flamegraph (wraps dtrace + inferno).
+  if ! command -v cargo >/dev/null 2>&1; then
+    echo "error: cargo not found in PATH; required for cargo-flamegraph on macOS." >&2
+    exit 1
+  fi
+
+  echo "Detected macOS; using cargo flamegraph (you may be prompted for sudo)."
+
+  FOLDED_STACKS="$OUT_DIR/stacks_${TAG}.folded"
+
+  # Ask cargo-flamegraph to tee the folded stacks into our own file.
+  (cd "$ROOT_DIR" && \
+    cargo flamegraph \
+      --root \
+      --bin evtx_dump \
+      --output "$OUT_DIR/flamegraph_${TAG}.svg" \
+      --post-process "tee $FOLDED_STACKS" \
+      -- "${FMT_ARGS[@]}" "$FLAME_FILE")
+
+  if [[ -f "$FOLDED_STACKS" ]] && [[ -s "$FOLDED_STACKS" ]]; then
+    # Extract top leafs (leaf functions) from folded stacks
+    {
+      echo "Top leaf functions (by total samples):"
+      awk '{
+        n = split($1, stack, ";");
+        if (n > 0) {
+          leaf = stack[n];
+          count = $2 + 0;
+          leafs[leaf] += count;
+        }
+      }
+      END {
+        for (f in leafs) {
+          printf "%d %s\n", leafs[f], f;
+        }
+      }' "$FOLDED_STACKS" | sort -nr | head -20 | awk '{printf "  %s: %s\n", $2, $1}'
+    } > "$OUT_DIR/top_leaf_${TAG}.txt"
+
+    # Extract top titles (root functions) from folded stacks
+    {
+      echo "Top title functions (by total samples):"
+      awk '{
+        n = split($1, stack, ";");
+        if (n > 0) {
+          title = stack[1];
+          count = $2 + 0;
+          titles[title] += count;
+        }
+      }
+      END {
+        for (f in titles) {
+          printf "%d %s\n", titles[f], f;
+        }
+      }' "$FOLDED_STACKS" | sort -nr | head -20 | awk '{printf "  %s: %s\n", $2, $1}'
+    } > "$OUT_DIR/top_titles_${TAG}.txt"
+
+    echo "Top leafs written to $OUT_DIR/top_leaf_${TAG}.txt"
+    echo "Top titles written to $OUT_DIR/top_titles_${TAG}.txt"
+  else
+    echo "warning: folded stacks file is empty or missing, skipping text summaries" >&2
+  fi
+
+  echo "Flamegraph written to $OUT_DIR/flamegraph_${TAG}.svg"
+  exit 0
+fi
+
+# Linux / perf + inferno path.
+#
+# Requirements:
+#   - perf
+#   - inferno-collapse-perf
+#   - inferno-flamegraph
+
+if ! command -v perf >/dev/null 2>&1; then
+  echo "error: perf not found in PATH; flamegraph_prod.sh currently expects Linux + perf." >&2
+  exit 1
+fi
+
+if ! command -v inferno-collapse-perf >/dev/null 2>&1; then
+  echo "error: inferno-collapse-perf not found in PATH." >&2
+  exit 1
+fi
+
+if ! command -v inferno-flamegraph >/dev/null 2>&1; then
+  echo "error: inferno-flamegraph not found in PATH." >&2
+  exit 1
+fi
+
+perf record -F 999 -g --output "$OUT_DIR/perf.data" -- \
+  "$BIN" "${FMT_ARGS[@]}" "$FLAME_FILE" >/dev/null
+
+perf script -i "$OUT_DIR/perf.data" | inferno-collapse-perf > "$OUT_DIR/stacks.folded"
+cat "$OUT_DIR/stacks.folded" | inferno-flamegraph > "$OUT_DIR/flamegraph_${TAG}.svg"
+
+# Extract top leafs (functions at end of stack) and top titles (functions at start of stack)
+# Folded format: "func1;func2;func3 12345" where number is sample count
+{
+  echo "Top leaf functions (by total samples):"
+  awk '{
+    n = split($1, stack, ";");
+    if (n > 0) {
+      leaf = stack[n];
+      count = $2 + 0;
+      leafs[leaf] += count;
+    }
+  }
+  END {
+    for (f in leafs) {
+      printf "%d %s\n", leafs[f], f;
+    }
+  }' "$OUT_DIR/stacks.folded" | sort -nr | head -20 | awk '{printf "  %s: %s\n", $2, $1}'
+} > "$OUT_DIR/top_leaf_${TAG}.txt"
+
+{
+  echo "Top title functions (by total samples):"
+  awk '{
+    n = split($1, stack, ";");
+    if (n > 0) {
+      title = stack[1];
+      count = $2 + 0;
+      titles[title] += count;
+    }
+  }
+  END {
+    for (f in titles) {
+      printf "%d %s\n", titles[f], f;
+    }
+  }' "$OUT_DIR/stacks.folded" | sort -nr | head -20 | awk '{printf "  %s: %s\n", $2, $1}'
+} > "$OUT_DIR/top_titles_${TAG}.txt"
+
+echo "Flamegraph written to $OUT_DIR/flamegraph_${TAG}.svg"
+echo "Top leafs written to $OUT_DIR/top_leaf_${TAG}.txt"
+echo "Top titles written to $OUT_DIR/top_titles_${TAG}.txt"
+
+