diff --git a/lib/common/huf.h b/lib/common/huf.h index 85518481ec6..595b2f6db5d 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -173,6 +173,11 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra +typedef enum { + HUF_depth_fast, /** Use heuristic to find the table depth**/ + HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/ + } HUF_depth_mode; /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -185,7 +190,10 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode); /* table is used as scratch space for building and testing tables, not a return value */ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); @@ -199,6 +207,7 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. @@ -209,7 +218,8 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, + unsigned suspectUncompressible, HUF_depth_mode depthMode); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. @@ -315,7 +325,8 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, + unsigned suspectUncompressible, HUF_depth_mode depthMode); size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ #ifndef HUF_FORCE_DECOMPRESS_X1 diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 5d90c162e73..e94b398da87 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -1222,15 +1222,6 @@ static size_t HUF_compressCTable_internal( return (size_t)(op-ostart); } - -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - unsigned tableLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); - assert(tableLog <= HUF_TABLELOG_MAX); - - return tableLog; -} - typedef struct { unsigned count[HUF_SYMBOLVALUE_MAX + 1]; HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; @@ -1244,6 +1235,61 @@ typedef struct { #define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode) +{ + unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + + if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/ + BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t optSize = ((size_t) ~0); + unsigned huffLog; + size_t maxBits, hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; + + for (huffLog = HUF_minTableLog(symbolCardinality); huffLog <= maxTableLog; huffLog++) { + maxBits = HUF_buildCTable_wksp(table, count, + maxSymbolValue, huffLog, + workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, + workSpace, wkspSize); + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize < optSize) { + optSize = newSize; + optLog = huffLog; + } + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; +} + /* HUF_compress_internal() : * `workSpace_align4` must be aligned on 4-bytes boundaries, * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ @@ -1254,7 +1300,7 @@ HUF_compress_internal (void* dst, size_t dstSize, HUF_nbStreams_e nbStreams, void* workSpace, size_t wkspSize, HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, - const int bmi2, unsigned suspectUncompressible) + const int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode) { HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); BYTE* const ostart = (BYTE*)dst; @@ -1318,7 +1364,7 @@ HUF_compress_internal (void* dst, size_t dstSize, } /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, depthMode); { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, maxSymbolValue, huffLog, &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); @@ -1367,7 +1413,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize, return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/, 0); + NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast); } size_t HUF_compress1X_repeat (void* dst, size_t dstSize, @@ -1375,13 +1421,13 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, - int bmi2, unsigned suspectUncompressible) + int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode) { DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, hufTable, - repeat, preferRepeat, bmi2, suspectUncompressible); + repeat, preferRepeat, bmi2, suspectUncompressible, depthMode); } /* HUF_compress4X_repeat(): @@ -1396,7 +1442,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/, 0); + NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast); } /* HUF_compress4X_repeat(): @@ -1407,13 +1453,14 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible) + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, + unsigned suspectUncompressible, HUF_depth_mode depthMode) { DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, - hufTable, repeat, preferRepeat, bmi2, suspectUncompressible); + hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode); } #ifndef ZSTD_NO_UNUSED_FUNCTIONS diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8a0c2f19165..adf1f6e7afc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2653,6 +2653,8 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, /* Base suspicion of uncompressibility on ratio of literals to sequences */ unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); size_t const litSize = (size_t)(seqStorePtr->lit - literals); + + HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast; size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, cctxParams->cParams.strategy, @@ -2660,7 +2662,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, op, dstCapacity, literals, litSize, entropyWorkspace, entropyWkspSize, - bmi2, suspectUncompressible); + bmi2, suspectUncompressible, depthMode); FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); assert(cSize <= dstCapacity); op += cSize; @@ -3107,7 +3109,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi ZSTD_hufCTables_t* nextHuf, ZSTD_hufCTablesMetadata_t* hufMetadata, const int literalsCompressionIsDisabled, - void* workspace, size_t wkspSize) + void* workspace, size_t wkspSize, HUF_depth_mode depthMode) { BYTE* const wkspStart = (BYTE*)workspace; BYTE* const wkspEnd = wkspStart + wkspSize; @@ -3164,7 +3166,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi /* Build Huffman Tree */ ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, depthMode); assert(huffLog <= LitHufLog); { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue, huffLog, @@ -3268,12 +3270,15 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, void* workspace, size_t wkspSize) { size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; + HUF_depth_mode depthMode = cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast; + entropyMetadata->hufMetadata.hufDesSize = ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, &prevEntropy->huf, &nextEntropy->huf, &entropyMetadata->hufMetadata, ZSTD_literalsCompressionIsDisabled(cctxParams), - workspace, wkspSize); + workspace, wkspSize, depthMode); + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); entropyMetadata->fseMetadata.fseTablesSize = ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index 15bde09e625..ea80a45c8fc 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -99,7 +99,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, const void* src, size_t srcSize, void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2, - unsigned suspectUncompressible) + unsigned suspectUncompressible, HUF_depth_mode depthMode) { size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); @@ -128,7 +128,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0; - typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned); + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode); huf_compress_f huf_compress; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; @@ -138,7 +138,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, entropyWorkspace, entropyWorkspaceSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, - bmi2, suspectUncompressible); + bmi2, suspectUncompressible, depthMode); if (repeat != HUF_repeat_none) { /* reused the existing table */ DEBUGLOG(5, "Reusing previous huffman table"); diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h index 9775fb97cb7..bb260db9b73 100644 --- a/lib/compress/zstd_compress_literals.h +++ b/lib/compress/zstd_compress_literals.h @@ -26,6 +26,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, const void* src, size_t srcSize, void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2, - unsigned suspectUncompressible); + unsigned suspectUncompressible, HUF_depth_mode depthMode); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/tests/fuzz/huf_round_trip.c b/tests/fuzz/huf_round_trip.c index bda9a9842ce..32b08d6d571 100644 --- a/tests/fuzz/huf_round_trip.c +++ b/tests/fuzz/huf_round_trip.c @@ -83,7 +83,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable)); dt[0] = tableLog * 0x01000001; - tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol); + HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal; + + tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode); FUZZ_ASSERT(tableLog <= 12); tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize); FUZZ_ZASSERT(tableLog); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 73069c5d6b2..dee7d572169 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -12,9 +12,9 @@ silesia.tar, level 7, compress silesia.tar, level 9, compress simple, 4552899 silesia.tar, level 13, compress simple, 4502956 silesia.tar, level 16, compress simple, 4360527 -silesia.tar, level 19, compress simple, 4267266 +silesia.tar, level 19, compress simple, 4266970 silesia.tar, uncompressed literals, compress simple, 4854086 -silesia.tar, uncompressed literals optimal, compress simple, 4267266 +silesia.tar, uncompressed literals optimal, compress simple, 4266970 silesia.tar, huffman literals, compress simple, 6179047 github.tar, level -5, compress simple, 52115 github.tar, level -3, compress simple, 45678 @@ -29,9 +29,9 @@ github.tar, level 7, compress github.tar, level 9, compress simple, 36760 github.tar, level 13, compress simple, 35501 github.tar, level 16, compress simple, 40471 -github.tar, level 19, compress simple, 32134 +github.tar, level 19, compress simple, 32149 github.tar, uncompressed literals, compress simple, 38831 -github.tar, uncompressed literals optimal, compress simple, 32134 +github.tar, uncompressed literals optimal, compress simple, 32149 github.tar, huffman literals, compress simple, 42560 silesia, level -5, compress cctx, 6857372 silesia, level -3, compress cctx, 6503412 @@ -46,7 +46,7 @@ silesia, level 7, compress silesia, level 9, compress cctx, 4543018 silesia, level 13, compress cctx, 4493990 silesia, level 16, compress cctx, 4359864 -silesia, level 19, compress cctx, 4296686 +silesia, level 19, compress cctx, 4296438 silesia, long distance mode, compress cctx, 4842075 silesia, multithreaded, compress cctx, 4842075 silesia, multithreaded long distance mode, compress cctx, 4842075 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912197 silesia, explicit params, compress cctx, 4794052 silesia, uncompressed literals, compress cctx, 4842075 -silesia, uncompressed literals optimal, compress cctx, 4296686 +silesia, uncompressed literals optimal, compress cctx, 4296438 silesia, huffman literals, compress cctx, 6172202 silesia, multithreaded with advanced params, compress cctx, 4842075 github, level -5, compress cctx, 204407 @@ -80,11 +80,11 @@ github, level 7, compress github, level 7 with dict, compress cctx, 38755 github, level 9, compress cctx, 135122 github, level 9 with dict, compress cctx, 39398 -github, level 13, compress cctx, 134064 +github, level 13, compress cctx, 132729 github, level 13 with dict, compress cctx, 39948 -github, level 16, compress cctx, 134064 +github, level 16, compress cctx, 132729 github, level 16 with dict, compress cctx, 37568 -github, level 19, compress cctx, 134064 +github, level 19, compress cctx, 132729 github, level 19 with dict, compress cctx, 37567 github, long distance mode, compress cctx, 141069 github, multithreaded, compress cctx, 141069 @@ -94,7 +94,7 @@ github, small hash log, compress github, small chain log, compress cctx, 139242 github, explicit params, compress cctx, 140932 github, uncompressed literals, compress cctx, 136332 -github, uncompressed literals optimal, compress cctx, 134064 +github, uncompressed literals optimal, compress cctx, 132729 github, huffman literals, compress cctx, 175468 github, multithreaded with advanced params, compress cctx, 141069 silesia, level -5, zstdcli, 6857420 @@ -110,7 +110,7 @@ silesia, level 7, zstdcli, silesia, level 9, zstdcli, 4543066 silesia, level 13, zstdcli, 4494038 silesia, level 16, zstdcli, 4359912 -silesia, level 19, zstdcli, 4296734 +silesia, level 19, zstdcli, 4296486 silesia, long distance mode, zstdcli, 4833785 silesia, multithreaded, zstdcli, 4842123 silesia, multithreaded long distance mode, zstdcli, 4833785 @@ -135,7 +135,7 @@ silesia.tar, level 7, zstdcli, silesia.tar, level 9, zstdcli, 4552903 silesia.tar, level 13, zstdcli, 4502960 silesia.tar, level 16, zstdcli, 4360531 -silesia.tar, level 19, zstdcli, 4267270 +silesia.tar, level 19, zstdcli, 4266974 silesia.tar, no source size, zstdcli, 4854160 silesia.tar, long distance mode, zstdcli, 4845745 silesia.tar, multithreaded, zstdcli, 4854164 @@ -170,11 +170,11 @@ github, level 7, zstdcli, github, level 7 with dict, zstdcli, 40745 github, level 9, zstdcli, 137122 github, level 9 with dict, zstdcli, 41393 -github, level 13, zstdcli, 136064 +github, level 13, zstdcli, 134729 github, level 13 with dict, zstdcli, 41900 -github, level 16, zstdcli, 136064 +github, level 16, zstdcli, 134729 github, level 16 with dict, zstdcli, 39577 -github, level 19, zstdcli, 136064 +github, level 19, zstdcli, 134729 github, level 19 with dict, zstdcli, 39576 github, long distance mode, zstdcli, 138332 github, multithreaded, zstdcli, 138332 @@ -212,9 +212,9 @@ github.tar, level 9 with dict, zstdcli, github.tar, level 13, zstdcli, 35505 github.tar, level 13 with dict, zstdcli, 37134 github.tar, level 16, zstdcli, 40475 -github.tar, level 16 with dict, zstdcli, 33382 -github.tar, level 19, zstdcli, 32138 -github.tar, level 19 with dict, zstdcli, 32713 +github.tar, level 16 with dict, zstdcli, 33378 +github.tar, level 19, zstdcli, 32153 +github.tar, level 19 with dict, zstdcli, 32716 github.tar, no source size, zstdcli, 38832 github.tar, no source size with dict, zstdcli, 38004 github.tar, long distance mode, zstdcli, 40236 @@ -249,7 +249,7 @@ silesia, level 12 row 1, advanced silesia, level 12 row 2, advanced one pass, 4503116 silesia, level 13, advanced one pass, 4493990 silesia, level 16, advanced one pass, 4359864 -silesia, level 19, advanced one pass, 4296686 +silesia, level 19, advanced one pass, 4296438 silesia, no source size, advanced one pass, 4842075 silesia, long distance mode, advanced one pass, 4833710 silesia, multithreaded, advanced one pass, 4842075 @@ -283,7 +283,7 @@ silesia.tar, level 12 row 1, advanced silesia.tar, level 12 row 2, advanced one pass, 4513797 silesia.tar, level 13, advanced one pass, 4502956 silesia.tar, level 16, advanced one pass, 4360527 -silesia.tar, level 19, advanced one pass, 4267266 +silesia.tar, level 19, advanced one pass, 4266970 silesia.tar, no source size, advanced one pass, 4854086 silesia.tar, long distance mode, advanced one pass, 4840452 silesia.tar, multithreaded, advanced one pass, 4854160 @@ -390,19 +390,19 @@ github, level 12 row 2 with dict dms, advanced github, level 12 row 2 with dict dds, advanced one pass, 39677 github, level 12 row 2 with dict copy, advanced one pass, 39677 github, level 12 row 2 with dict load, advanced one pass, 41166 -github, level 13, advanced one pass, 134064 +github, level 13, advanced one pass, 132729 github, level 13 with dict, advanced one pass, 39900 github, level 13 with dict dms, advanced one pass, 39900 github, level 13 with dict dds, advanced one pass, 39900 github, level 13 with dict copy, advanced one pass, 39948 -github, level 13 with dict load, advanced one pass, 42626 -github, level 16, advanced one pass, 134064 +github, level 13 with dict load, advanced one pass, 42624 +github, level 16, advanced one pass, 132729 github, level 16 with dict, advanced one pass, 37577 github, level 16 with dict dms, advanced one pass, 37577 github, level 16 with dict dds, advanced one pass, 37577 github, level 16 with dict copy, advanced one pass, 37568 -github, level 16 with dict load, advanced one pass, 42340 -github, level 19, advanced one pass, 134064 +github, level 16 with dict load, advanced one pass, 42338 +github, level 19, advanced one pass, 132729 github, level 19 with dict, advanced one pass, 37576 github, level 19 with dict dms, advanced one pass, 37576 github, level 19 with dict dds, advanced one pass, 37576 @@ -522,17 +522,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict load, advanced one pass, 36010 github.tar, level 16, advanced one pass, 40471 -github.tar, level 16 with dict, advanced one pass, 33378 -github.tar, level 16 with dict dms, advanced one pass, 33213 -github.tar, level 16 with dict dds, advanced one pass, 33213 -github.tar, level 16 with dict copy, advanced one pass, 33378 +github.tar, level 16 with dict, advanced one pass, 33374 +github.tar, level 16 with dict dms, advanced one pass, 33206 +github.tar, level 16 with dict dds, advanced one pass, 33206 +github.tar, level 16 with dict copy, advanced one pass, 33374 github.tar, level 16 with dict load, advanced one pass, 39081 -github.tar, level 19, advanced one pass, 32134 -github.tar, level 19 with dict, advanced one pass, 32709 -github.tar, level 19 with dict dms, advanced one pass, 32553 -github.tar, level 19 with dict dds, advanced one pass, 32553 -github.tar, level 19 with dict copy, advanced one pass, 32709 -github.tar, level 19 with dict load, advanced one pass, 32474 +github.tar, level 19, advanced one pass, 32149 +github.tar, level 19 with dict, advanced one pass, 32712 +github.tar, level 19 with dict dms, advanced one pass, 32555 +github.tar, level 19 with dict dds, advanced one pass, 32555 +github.tar, level 19 with dict copy, advanced one pass, 32712 +github.tar, level 19 with dict load, advanced one pass, 32479 github.tar, no source size, advanced one pass, 38831 github.tar, no source size with dict, advanced one pass, 37995 github.tar, long distance mode, advanced one pass, 40252 @@ -567,7 +567,7 @@ silesia, level 12 row 1, advanced silesia, level 12 row 2, advanced one pass small out, 4503116 silesia, level 13, advanced one pass small out, 4493990 silesia, level 16, advanced one pass small out, 4359864 -silesia, level 19, advanced one pass small out, 4296686 +silesia, level 19, advanced one pass small out, 4296438 silesia, no source size, advanced one pass small out, 4842075 silesia, long distance mode, advanced one pass small out, 4833710 silesia, multithreaded, advanced one pass small out, 4842075 @@ -601,7 +601,7 @@ silesia.tar, level 12 row 1, advanced silesia.tar, level 12 row 2, advanced one pass small out, 4513797 silesia.tar, level 13, advanced one pass small out, 4502956 silesia.tar, level 16, advanced one pass small out, 4360527 -silesia.tar, level 19, advanced one pass small out, 4267266 +silesia.tar, level 19, advanced one pass small out, 4266970 silesia.tar, no source size, advanced one pass small out, 4854086 silesia.tar, long distance mode, advanced one pass small out, 4840452 silesia.tar, multithreaded, advanced one pass small out, 4854160 @@ -708,19 +708,19 @@ github, level 12 row 2 with dict dms, advanced github, level 12 row 2 with dict dds, advanced one pass small out, 39677 github, level 12 row 2 with dict copy, advanced one pass small out, 39677 github, level 12 row 2 with dict load, advanced one pass small out, 41166 -github, level 13, advanced one pass small out, 134064 +github, level 13, advanced one pass small out, 132729 github, level 13 with dict, advanced one pass small out, 39900 github, level 13 with dict dms, advanced one pass small out, 39900 github, level 13 with dict dds, advanced one pass small out, 39900 github, level 13 with dict copy, advanced one pass small out, 39948 -github, level 13 with dict load, advanced one pass small out, 42626 -github, level 16, advanced one pass small out, 134064 +github, level 13 with dict load, advanced one pass small out, 42624 +github, level 16, advanced one pass small out, 132729 github, level 16 with dict, advanced one pass small out, 37577 github, level 16 with dict dms, advanced one pass small out, 37577 github, level 16 with dict dds, advanced one pass small out, 37577 github, level 16 with dict copy, advanced one pass small out, 37568 -github, level 16 with dict load, advanced one pass small out, 42340 -github, level 19, advanced one pass small out, 134064 +github, level 16 with dict load, advanced one pass small out, 42338 +github, level 19, advanced one pass small out, 132729 github, level 19 with dict, advanced one pass small out, 37576 github, level 19 with dict dms, advanced one pass small out, 37576 github, level 19 with dict dds, advanced one pass small out, 37576 @@ -840,17 +840,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict load, advanced one pass small out, 36010 github.tar, level 16, advanced one pass small out, 40471 -github.tar, level 16 with dict, advanced one pass small out, 33378 -github.tar, level 16 with dict dms, advanced one pass small out, 33213 -github.tar, level 16 with dict dds, advanced one pass small out, 33213 -github.tar, level 16 with dict copy, advanced one pass small out, 33378 +github.tar, level 16 with dict, advanced one pass small out, 33374 +github.tar, level 16 with dict dms, advanced one pass small out, 33206 +github.tar, level 16 with dict dds, advanced one pass small out, 33206 +github.tar, level 16 with dict copy, advanced one pass small out, 33374 github.tar, level 16 with dict load, advanced one pass small out, 39081 -github.tar, level 19, advanced one pass small out, 32134 -github.tar, level 19 with dict, advanced one pass small out, 32709 -github.tar, level 19 with dict dms, advanced one pass small out, 32553 -github.tar, level 19 with dict dds, advanced one pass small out, 32553 -github.tar, level 19 with dict copy, advanced one pass small out, 32709 -github.tar, level 19 with dict load, advanced one pass small out, 32474 +github.tar, level 19, advanced one pass small out, 32149 +github.tar, level 19 with dict, advanced one pass small out, 32712 +github.tar, level 19 with dict dms, advanced one pass small out, 32555 +github.tar, level 19 with dict dds, advanced one pass small out, 32555 +github.tar, level 19 with dict copy, advanced one pass small out, 32712 +github.tar, level 19 with dict load, advanced one pass small out, 32479 github.tar, no source size, advanced one pass small out, 38831 github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, long distance mode, advanced one pass small out, 40252 @@ -885,7 +885,7 @@ silesia, level 12 row 1, advanced silesia, level 12 row 2, advanced streaming, 4503116 silesia, level 13, advanced streaming, 4493990 silesia, level 16, advanced streaming, 4359864 -silesia, level 19, advanced streaming, 4296686 +silesia, level 19, advanced streaming, 4296438 silesia, no source size, advanced streaming, 4842039 silesia, long distance mode, advanced streaming, 4833710 silesia, multithreaded, advanced streaming, 4842075 @@ -919,7 +919,7 @@ silesia.tar, level 12 row 1, advanced silesia.tar, level 12 row 2, advanced streaming, 4513797 silesia.tar, level 13, advanced streaming, 4502956 silesia.tar, level 16, advanced streaming, 4360527 -silesia.tar, level 19, advanced streaming, 4267266 +silesia.tar, level 19, advanced streaming, 4266970 silesia.tar, no source size, advanced streaming, 4859267 silesia.tar, long distance mode, advanced streaming, 4840452 silesia.tar, multithreaded, advanced streaming, 4854160 @@ -1026,19 +1026,19 @@ github, level 12 row 2 with dict dms, advanced github, level 12 row 2 with dict dds, advanced streaming, 39677 github, level 12 row 2 with dict copy, advanced streaming, 39677 github, level 12 row 2 with dict load, advanced streaming, 41166 -github, level 13, advanced streaming, 134064 +github, level 13, advanced streaming, 132729 github, level 13 with dict, advanced streaming, 39900 github, level 13 with dict dms, advanced streaming, 39900 github, level 13 with dict dds, advanced streaming, 39900 github, level 13 with dict copy, advanced streaming, 39948 -github, level 13 with dict load, advanced streaming, 42626 -github, level 16, advanced streaming, 134064 +github, level 13 with dict load, advanced streaming, 42624 +github, level 16, advanced streaming, 132729 github, level 16 with dict, advanced streaming, 37577 github, level 16 with dict dms, advanced streaming, 37577 github, level 16 with dict dds, advanced streaming, 37577 github, level 16 with dict copy, advanced streaming, 37568 -github, level 16 with dict load, advanced streaming, 42340 -github, level 19, advanced streaming, 134064 +github, level 16 with dict load, advanced streaming, 42338 +github, level 19, advanced streaming, 132729 github, level 19 with dict, advanced streaming, 37576 github, level 19 with dict dms, advanced streaming, 37576 github, level 19 with dict dds, advanced streaming, 37576 @@ -1158,17 +1158,17 @@ github.tar, level 13 with dict dds, advanced github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict load, advanced streaming, 36010 github.tar, level 16, advanced streaming, 40471 -github.tar, level 16 with dict, advanced streaming, 33378 -github.tar, level 16 with dict dms, advanced streaming, 33213 -github.tar, level 16 with dict dds, advanced streaming, 33213 -github.tar, level 16 with dict copy, advanced streaming, 33378 +github.tar, level 16 with dict, advanced streaming, 33374 +github.tar, level 16 with dict dms, advanced streaming, 33206 +github.tar, level 16 with dict dds, advanced streaming, 33206 +github.tar, level 16 with dict copy, advanced streaming, 33374 github.tar, level 16 with dict load, advanced streaming, 39081 -github.tar, level 19, advanced streaming, 32134 -github.tar, level 19 with dict, advanced streaming, 32709 -github.tar, level 19 with dict dms, advanced streaming, 32553 -github.tar, level 19 with dict dds, advanced streaming, 32553 -github.tar, level 19 with dict copy, advanced streaming, 32709 -github.tar, level 19 with dict load, advanced streaming, 32474 +github.tar, level 19, advanced streaming, 32149 +github.tar, level 19 with dict, advanced streaming, 32712 +github.tar, level 19 with dict dms, advanced streaming, 32555 +github.tar, level 19 with dict dds, advanced streaming, 32555 +github.tar, level 19 with dict copy, advanced streaming, 32712 +github.tar, level 19 with dict load, advanced streaming, 32479 github.tar, no source size, advanced streaming, 38828 github.tar, no source size with dict, advanced streaming, 38000 github.tar, long distance mode, advanced streaming, 40252 @@ -1195,10 +1195,10 @@ silesia, level 7, old stre silesia, level 9, old streaming, 4543018 silesia, level 13, old streaming, 4493990 silesia, level 16, old streaming, 4359864 -silesia, level 19, old streaming, 4296686 +silesia, level 19, old streaming, 4296438 silesia, no source size, old streaming, 4842039 silesia, uncompressed literals, old streaming, 4842075 -silesia, uncompressed literals optimal, old streaming, 4296686 +silesia, uncompressed literals optimal, old streaming, 4296438 silesia, huffman literals, old streaming, 6172207 silesia.tar, level -5, old streaming, 6856523 silesia.tar, level -3, old streaming, 6505954 @@ -1213,10 +1213,10 @@ silesia.tar, level 7, old stre silesia.tar, level 9, old streaming, 4552900 silesia.tar, level 13, old streaming, 4502956 silesia.tar, level 16, old streaming, 4360527 -silesia.tar, level 19, old streaming, 4267266 +silesia.tar, level 19, old streaming, 4266970 silesia.tar, no source size, old streaming, 4859267 silesia.tar, uncompressed literals, old streaming, 4859271 -silesia.tar, uncompressed literals optimal, old streaming, 4267266 +silesia.tar, uncompressed literals optimal, old streaming, 4266970 silesia.tar, huffman literals, old streaming, 6179056 github, level -5, old streaming, 204407 github, level -5 with dict, old streaming, 46718 @@ -1240,16 +1240,16 @@ github, level 7, old stre github, level 7 with dict, old streaming, 38758 github, level 9, old streaming, 135122 github, level 9 with dict, old streaming, 39437 -github, level 13, old streaming, 134064 +github, level 13, old streaming, 132729 github, level 13 with dict, old streaming, 39900 -github, level 16, old streaming, 134064 +github, level 16, old streaming, 132729 github, level 16 with dict, old streaming, 37577 -github, level 19, old streaming, 134064 +github, level 19, old streaming, 132729 github, level 19 with dict, old streaming, 37576 github, no source size, old streaming, 140599 github, no source size with dict, old streaming, 40654 github, uncompressed literals, old streaming, 136332 -github, uncompressed literals optimal, old streaming, 134064 +github, uncompressed literals optimal, old streaming, 132729 github, huffman literals, old streaming, 175468 github.tar, level -5, old streaming, 52152 github.tar, level -5 with dict, old streaming, 51045 @@ -1276,13 +1276,13 @@ github.tar, level 9 with dict, old stre github.tar, level 13, old streaming, 35501 github.tar, level 13 with dict, old streaming, 37130 github.tar, level 16, old streaming, 40471 -github.tar, level 16 with dict, old streaming, 33378 -github.tar, level 19, old streaming, 32134 -github.tar, level 19 with dict, old streaming, 32709 +github.tar, level 16 with dict, old streaming, 33374 +github.tar, level 19, old streaming, 32149 +github.tar, level 19 with dict, old streaming, 32712 github.tar, no source size, old streaming, 38828 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38831 -github.tar, uncompressed literals optimal, old streaming, 32134 +github.tar, uncompressed literals optimal, old streaming, 32149 github.tar, huffman literals, old streaming, 42560 silesia, level -5, old streaming advanced, 6854744 silesia, level -3, old streaming advanced, 6503319 @@ -1297,7 +1297,7 @@ silesia, level 7, old stre silesia, level 9, old streaming advanced, 4543018 silesia, level 13, old streaming advanced, 4493990 silesia, level 16, old streaming advanced, 4359864 -silesia, level 19, old streaming advanced, 4296686 +silesia, level 19, old streaming advanced, 4296438 silesia, no source size, old streaming advanced, 4842039 silesia, long distance mode, old streaming advanced, 4842075 silesia, multithreaded, old streaming advanced, 4842075 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912197 silesia, explicit params, old streaming advanced, 4795452 silesia, uncompressed literals, old streaming advanced, 4842075 -silesia, uncompressed literals optimal, old streaming advanced, 4296686 +silesia, uncompressed literals optimal, old streaming advanced, 4296438 silesia, huffman literals, old streaming advanced, 6172207 silesia, multithreaded with advanced params, old streaming advanced, 4842075 silesia.tar, level -5, old streaming advanced, 6856523 @@ -1323,7 +1323,7 @@ silesia.tar, level 7, old stre silesia.tar, level 9, old streaming advanced, 4552900 silesia.tar, level 13, old streaming advanced, 4502956 silesia.tar, level 16, old streaming advanced, 4360527 -silesia.tar, level 19, old streaming advanced, 4267266 +silesia.tar, level 19, old streaming advanced, 4266970 silesia.tar, no source size, old streaming advanced, 4859267 silesia.tar, long distance mode, old streaming advanced, 4859271 silesia.tar, multithreaded, old streaming advanced, 4859271 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, explicit params, old streaming advanced, 4806873 silesia.tar, uncompressed literals, old streaming advanced, 4859271 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4267266 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4266970 silesia.tar, huffman literals, old streaming advanced, 6179056 silesia.tar, multithreaded with advanced params, old streaming advanced, 4859271 github, level -5, old streaming advanced, 213265 @@ -1362,7 +1362,7 @@ github, level 13, old stre github, level 13 with dict, old streaming advanced, 39725 github, level 16, old streaming advanced, 138676 github, level 16 with dict, old streaming advanced, 40789 -github, level 19, old streaming advanced, 134064 +github, level 19, old streaming advanced, 132729 github, level 19 with dict, old streaming advanced, 37576 github, no source size, old streaming advanced, 140599 github, no source size with dict, old streaming advanced, 40608 @@ -1374,7 +1374,7 @@ github, small hash log, old stre github, small chain log, old streaming advanced, 139275 github, explicit params, old streaming advanced, 140937 github, uncompressed literals, old streaming advanced, 141104 -github, uncompressed literals optimal, old streaming advanced, 134064 +github, uncompressed literals optimal, old streaming advanced, 132729 github, huffman literals, old streaming advanced, 181107 github, multithreaded with advanced params, old streaming advanced, 141104 github.tar, level -5, old streaming advanced, 52152 @@ -1403,8 +1403,8 @@ github.tar, level 13, old stre github.tar, level 13 with dict, old streaming advanced, 35807 github.tar, level 16, old streaming advanced, 40471 github.tar, level 16 with dict, old streaming advanced, 38578 -github.tar, level 19, old streaming advanced, 32134 -github.tar, level 19 with dict, old streaming advanced, 32702 +github.tar, level 19, old streaming advanced, 32149 +github.tar, level 19 with dict, old streaming advanced, 32704 github.tar, no source size, old streaming advanced, 38828 github.tar, no source size with dict, old streaming advanced, 38015 github.tar, long distance mode, old streaming advanced, 38831 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41385 github.tar, uncompressed literals, old streaming advanced, 38831 -github.tar, uncompressed literals optimal, old streaming advanced, 32134 +github.tar, uncompressed literals optimal, old streaming advanced, 32149 github.tar, huffman literals, old streaming advanced, 42560 github.tar, multithreaded with advanced params, old streaming advanced, 38831 github, level -5 with dict, old streaming cdict, 46718 @@ -1446,7 +1446,7 @@ github.tar, level 7 with dict, old stre github.tar, level 9 with dict, old streaming cdict, 36401 github.tar, level 13 with dict, old streaming cdict, 36010 github.tar, level 16 with dict, old streaming cdict, 39081 -github.tar, level 19 with dict, old streaming cdict, 32474 +github.tar, level 19 with dict, old streaming cdict, 32479 github.tar, no source size with dict, old streaming cdict, 38000 github, level -5 with dict, old streaming advanced cdict, 49562 github, level -3 with dict, old streaming advanced cdict, 44956 @@ -1476,5 +1476,5 @@ github.tar, level 7 with dict, old stre github.tar, level 9 with dict, old streaming advanced cdict, 36312 github.tar, level 13 with dict, old streaming advanced cdict, 35807 github.tar, level 16 with dict, old streaming advanced cdict, 38578 -github.tar, level 19 with dict, old streaming advanced cdict, 32702 +github.tar, level 19 with dict, old streaming advanced cdict, 32704 github.tar, no source size with dict, old streaming advanced cdict, 38015