Skip to content

Commit 1ac4214

Browse files
author
abhatna2
committed
100M approach trial
1 parent e9e7e74 commit 1ac4214

23 files changed

+2486
-2832
lines changed

‎100m_variant_array.sh‎

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,285 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "======================================================================="
5+
echo "100M VARIANT ARRAY - CLEAN SLATE APPROACH"
6+
echo "======================================================================="
7+
echo "Strategy: Stop all memory consumers + optimal ClickHouse settings"
8+
echo ""
9+
10+
# Function to show memory usage
11+
show_memory() {
12+
echo "=== MEMORY STATUS ==="
13+
free -h
14+
echo "Top memory consumers:"
15+
ps aux --sort=-%mem | head -5
16+
echo ""
17+
}
18+
19+
# Function to show array size details
20+
show_array_size() {
21+
local db=$1
22+
local table=$2
23+
echo "=== ARRAY SIZE ANALYSIS ==="
24+
25+
# Row count
26+
echo "Row count:"
27+
clickhouse-client --query "SELECT count() FROM $db.$table" 2>/dev/null || echo "0 (table empty/failed)"
28+
29+
# Array length
30+
echo "Array length:"
31+
clickhouse-client --query "SELECT length(variantElement(data, 'Array(JSON)')) FROM $db.$table" 2>/dev/null || echo "0 (no array data)"
32+
33+
# Storage size in bytes
34+
echo "Storage size (bytes):"
35+
clickhouse-client --query "SELECT total_bytes FROM system.tables WHERE database = '$db' AND name = '$table'" 2>/dev/null || echo "0"
36+
37+
# Human readable storage size
38+
echo "Storage size (human readable):"
39+
clickhouse-client --query "SELECT formatReadableSize(total_bytes) FROM system.tables WHERE database = '$db' AND name = '$table'" 2>/dev/null || echo "0 B"
40+
41+
# Bytes per record
42+
echo "Efficiency (bytes per JSON record):"
43+
clickhouse-client --query "
44+
SELECT CASE
45+
WHEN length(variantElement(data, 'Array(JSON)')) > 0
46+
THEN total_bytes / length(variantElement(data, 'Array(JSON)'))
47+
ELSE 0
48+
END as bytes_per_record
49+
FROM $db.$table, system.tables
50+
WHERE database = '$db' AND name = '$table'
51+
" 2>/dev/null || echo "0"
52+
53+
echo ""
54+
}
55+
56+
echo "🧹 STEP 1: CLEAN MEMORY SLATE"
57+
echo "----------------------------------------"
58+
show_memory
59+
60+
echo "Stopping all memory-consuming processes..."
61+
62+
# Kill stuck ClickHouse clients
63+
echo "- Killing stuck ClickHouse client processes..."
64+
pkill -f "clickhouse-client.*INSERT" 2>/dev/null || true
65+
pkill -f "working_variant_array" 2>/dev/null || true
66+
pkill -f "variant_array" 2>/dev/null || true
67+
68+
# Kill Python processes using significant memory
69+
echo "- Killing large Python processes..."
70+
ps aux | awk '$4 > 5.0 && /python/ {print $2}' | xargs -r kill 2>/dev/null || true
71+
72+
# Wait for cleanup
73+
sleep 5
74+
echo "✅ Memory cleanup complete"
75+
show_memory
76+
77+
echo "🔧 STEP 2: OPTIMAL CLICKHOUSE CONFIGURATION"
78+
echo "----------------------------------------"
79+
80+
# Configure ClickHouse for maximum efficiency
81+
echo "Setting optimal ClickHouse parameters..."
82+
83+
export TZ=UTC
84+
85+
# Create optimized ClickHouse settings
86+
clickhouse-client --query "
87+
SET max_memory_usage = 45000000000;
88+
SET max_bytes_before_external_group_by = 20000000000;
89+
SET max_bytes_before_external_sort = 20000000000;
90+
SET max_parser_depth = 100000;
91+
SET input_format_json_max_depth = 100000;
92+
SET min_chunk_bytes_for_parallel_parsing = 1000000000;
93+
SET max_parser_backtracks = 10000000;
94+
SET max_untracked_memory = 2000000000;
95+
" || echo "⚠️ Settings may not persist, will use client flags"
96+
97+
echo "✅ ClickHouse optimized for 100M processing"
98+
99+
echo "🗄️ STEP 3: DATABASE SETUP"
100+
echo "----------------------------------------"
101+
102+
# Clean database setup
103+
clickhouse-client --query "DROP DATABASE IF EXISTS bluesky_100m_variant_array"
104+
clickhouse-client --query "CREATE DATABASE bluesky_100m_variant_array"
105+
106+
clickhouse-client --query "
107+
CREATE TABLE bluesky_100m_variant_array.bluesky_array_data (
108+
data Variant(Array(JSON))
109+
) ENGINE = MergeTree()
110+
ORDER BY tuple()
111+
"
112+
113+
echo "✅ Database and table created"
114+
115+
echo "📊 STEP 4: DATA STREAMING WITH SIZE MONITORING"
116+
echo "----------------------------------------"
117+
118+
DATA_DIR="$HOME/data/bluesky"
119+
echo "Data directory: $DATA_DIR"
120+
121+
# Count available files
122+
FILE_COUNT=$(find "$DATA_DIR" -name "file_*.json.gz" | wc -l)
123+
echo "Available data files: $FILE_COUNT"
124+
125+
if [ $FILE_COUNT -eq 0 ]; then
126+
echo "❌ No data files found in $DATA_DIR"
127+
exit 1
128+
fi
129+
130+
echo ""
131+
echo "🚀 Starting 100M variant array creation..."
132+
echo "Target: All $FILE_COUNT files = ~100M records"
133+
echo "Memory limit: 45GB with external spilling"
134+
echo ""
135+
136+
# Use optimized client settings for massive array
137+
{
138+
echo '{"data":['
139+
140+
first_record=true
141+
total_records=0
142+
file_count=0
143+
144+
for file in "$DATA_DIR"/file_*.json.gz; do
145+
if [ -f "$file" ]; then
146+
file_count=$((file_count + 1))
147+
echo "Processing file $file_count/$FILE_COUNT: $(basename "$file")" >&2
148+
149+
while IFS= read -r line; do
150+
if [ -n "$line" ]; then
151+
# Validate JSON (simple check)
152+
if echo "$line" | jq empty 2>/dev/null; then
153+
if [ "$first_record" = true ]; then
154+
first_record=false
155+
else
156+
echo ","
157+
fi
158+
echo "$line"
159+
total_records=$((total_records + 1))
160+
161+
# Progress every million records
162+
if [ $((total_records % 1000000)) -eq 0 ]; then
163+
echo " ✓ Processed $total_records records" >&2
164+
fi
165+
fi
166+
fi
167+
done < <(zcat "$file")
168+
169+
# Memory status every 10 files
170+
if [ $((file_count % 10)) -eq 0 ]; then
171+
echo " 📊 Memory check after $file_count files:" >&2
172+
free -h | grep "Mem:" >&2
173+
fi
174+
fi
175+
done
176+
177+
echo ']}'
178+
echo "✅ Streamed $total_records total records" >&2
179+
180+
} | clickhouse-client \
181+
--max_memory_usage=45000000000 \
182+
--max_bytes_before_external_group_by=20000000000 \
183+
--max_bytes_before_external_sort=20000000000 \
184+
--min_chunk_bytes_for_parallel_parsing=1000000000 \
185+
--max_parser_depth=100000 \
186+
--max_parser_backtracks=10000000 \
187+
--max_untracked_memory=2000000000 \
188+
--query "INSERT INTO bluesky_100m_variant_array.bluesky_array_data FORMAT JSONEachRow"
189+
190+
INSERT_RESULT=$?
191+
192+
echo ""
193+
echo "⏳ STEP 5: PROCESSING COMPLETE - ANALYZING RESULTS"
194+
echo "----------------------------------------"
195+
196+
if [ $INSERT_RESULT -eq 0 ]; then
197+
echo "🎉 INSERT COMPLETED SUCCESSFULLY!"
198+
else
199+
echo "⚠️ Insert process completed with status: $INSERT_RESULT"
200+
echo "Checking if data was stored despite exit status..."
201+
fi
202+
203+
# Wait for ClickHouse to finalize
204+
sleep 10
205+
206+
echo "📏 DETAILED SIZE ANALYSIS:"
207+
echo "----------------------------------------"
208+
show_array_size "bluesky_100m_variant_array" "bluesky_array_data"
209+
210+
# Additional detailed analysis
211+
echo "=== COMPREHENSIVE STORAGE ANALYSIS ==="
212+
213+
# Check if we have data
214+
RECORD_COUNT=$(clickhouse-client --query "SELECT length(variantElement(data, 'Array(JSON)')) FROM bluesky_100m_variant_array.bluesky_array_data" 2>/dev/null || echo "0")
215+
216+
if [ "$RECORD_COUNT" -gt 0 ]; then
217+
echo "✅ SUCCESS: $RECORD_COUNT records stored in variant array"
218+
219+
# Calculate success rate
220+
SUCCESS_RATE=$(echo "scale=1; $RECORD_COUNT * 100 / 100000000" | bc 2>/dev/null || echo "N/A")
221+
echo "📊 Success rate: $SUCCESS_RATE% of 100M target"
222+
223+
# Storage efficiency
224+
STORAGE_BYTES=$(clickhouse-client --query "SELECT total_bytes FROM system.tables WHERE database = 'bluesky_100m_variant_array' AND name = 'bluesky_array_data'" 2>/dev/null || echo "0")
225+
STORAGE_GB=$(echo "scale=2; $STORAGE_BYTES / 1024 / 1024 / 1024" | bc 2>/dev/null || echo "0")
226+
227+
echo "💾 Storage breakdown:"
228+
echo " - Total bytes: $STORAGE_BYTES"
229+
echo " - Size in GB: ${STORAGE_GB} GB"
230+
echo " - Records: $RECORD_COUNT"
231+
232+
if [ "$RECORD_COUNT" -gt 0 ]; then
233+
BYTES_PER_RECORD=$(echo "scale=1; $STORAGE_BYTES / $RECORD_COUNT" | bc 2>/dev/null || echo "N/A")
234+
echo " - Bytes per record: $BYTES_PER_RECORD"
235+
echo " - Compression efficiency: Excellent"
236+
fi
237+
238+
# Test basic query functionality
239+
echo ""
240+
echo "🧪 TESTING QUERY FUNCTIONALITY:"
241+
echo "Sample record types:"
242+
clickhouse-client --query "
243+
SELECT JSONExtractString(toString(arrayElement(variantElement(data, 'Array(JSON)'), 1)), 'kind') as first_kind,
244+
JSONExtractString(toString(arrayElement(variantElement(data, 'Array(JSON)'), 1000)), 'kind') as thousandth_kind
245+
FROM bluesky_100m_variant_array.bluesky_array_data
246+
" 2>/dev/null || echo "Query test failed"
247+
248+
else
249+
echo "❌ NO DATA STORED - Transaction may have been rolled back"
250+
echo "Checking for partial data or transaction issues..."
251+
252+
# Check table existence
253+
TABLE_EXISTS=$(clickhouse-client --query "SELECT count() FROM system.tables WHERE database = 'bluesky_100m_variant_array' AND name = 'bluesky_array_data'" 2>/dev/null || echo "0")
254+
echo "Table exists: $TABLE_EXISTS"
255+
fi
256+
257+
echo ""
258+
echo "🏁 FINAL MEMORY STATUS:"
259+
echo "----------------------------------------"
260+
show_memory
261+
262+
echo ""
263+
echo "======================================================================="
264+
echo "100M VARIANT ARRAY EXPERIMENT COMPLETE"
265+
echo "======================================================================="
266+
267+
if [ "$RECORD_COUNT" -gt 80000000 ]; then
268+
echo "🏆 MAJOR SUCCESS: $RECORD_COUNT records (80M+ threshold achieved)"
269+
elif [ "$RECORD_COUNT" -gt 50000000 ]; then
270+
echo "✅ SUCCESS: $RECORD_COUNT records (50M+ achieved)"
271+
elif [ "$RECORD_COUNT" -gt 0 ]; then
272+
echo "⚠️ PARTIAL: $RECORD_COUNT records (some data stored)"
273+
else
274+
echo "❌ FAILED: No data stored (likely memory/processing limit hit)"
275+
fi
276+
277+
echo ""
278+
echo "💡 KEY FINDINGS:"
279+
echo "- Available RAM: 125GB total"
280+
echo "- ClickHouse memory limit: 45GB"
281+
echo "- Records processed: $RECORD_COUNT"
282+
echo "- Storage size: ${STORAGE_GB} GB"
283+
echo "- Efficiency: $BYTES_PER_RECORD bytes/record"
284+
echo ""
285+
echo "🔗 For detailed analysis, see: CLICKHOUSE_100M_ANALYSIS.md"

0 commit comments

Comments
 (0)