From 77ee44c7b6fc722e8f208599a775162c8308abc3 Mon Sep 17 00:00:00 2001 From: xaphier Date: Sat, 10 Oct 2015 12:14:51 +0200 Subject: [PATCH 1/4] Remove one malloc/free from compression By making the buffer part of the zstd context structure, one malloc/free can be removed from the compression. --- lib/zstd.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index a574efc9ecc..a150e6ea6a6 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -314,6 +314,7 @@ typedef struct ZSTD_Cctx_s #else U32 hashTable[HASH_TABLESIZE]; #endif + BYTE buffer[WORKPLACESIZE]; } cctxi_t; @@ -321,12 +322,7 @@ ZSTD_Cctx* ZSTD_createCCtx(void) { ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) ); if (ctx==NULL) return NULL; - ctx->seqStore.buffer = malloc(WORKPLACESIZE); - if (ctx->seqStore.buffer==NULL) - { - free(ctx); - return NULL; - } + ctx->seqStore.buffer = ctx->buffer; ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer); ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2)); ctx->seqStore.litStart = ctx->seqStore.offCodeStart + (BLOCKSIZE>>2); @@ -344,7 +340,6 @@ void ZSTD_resetCCtx(ZSTD_Cctx* ctx) size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx) { - free(ctx->seqStore.buffer); free(ctx); return 0; } From 59aac5f467b2d231567970e14b2b4613b40abdbb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Oct 2015 16:28:19 +0100 Subject: [PATCH 2/4] Clarified comments --- lib/zstd_static.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 55a41012f55..560720a5acd 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -32,6 +32,11 @@ */ #pragma once +/* The objects defined into this file should be considered experimental. + * They are not labelled stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risk of future changes. + */ + #if defined (__cplusplus) extern "C" { #endif @@ -63,10 +68,9 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx); size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /* Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue(). - This value is expected to be provided, precisely, as 'srcSize'. - Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() ) - ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'. + ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() will use previous data blocks to improve compresson if they are located prior to current block. + Result is the number of bytes regenerated within 'dst'. It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. */ From 8f86c700cdb9190901613124100c9be4c6e69827 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Oct 2015 17:53:44 +0100 Subject: [PATCH 3/4] Fix uninitialized warnings reported by @nemequ --- lib/zstd.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index a150e6ea6a6..5531aac1a8f 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1234,7 +1234,7 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx, } -size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, +size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize) { @@ -1267,6 +1267,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, } *dumpsPtr = ip; ip += dumpsLength; + *dumpsLengthPtr = dumpsLength; /* check */ if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ @@ -1351,6 +1352,7 @@ typedef struct { FSE_DState_t stateML; size_t prevOffset; const BYTE* dumps; + const BYTE* dumpsEnd; } seqState_t; @@ -1361,6 +1363,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) size_t offset; size_t matchLength; const BYTE* dumps = seqState->dumps; + const BYTE* const de = seqState->dumpsEnd; /* Literal length */ litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream)); @@ -1368,12 +1371,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) seqState->prevOffset = seq->offset; if (litLength == MaxLL) { - U32 add = *dumps++; + U32 add = dumps 1 byte */ + dumps += 3; + } } } @@ -1393,12 +1399,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream)); if (matchLength == MaxML) { - U32 add = *dumps++; + U32 add = dumps 1 byte */ - dumps += 3; + if (dumps<=(de-3)) + { + matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */ + dumps += 3; + } } } matchLength += MINMATCH; @@ -1515,7 +1524,7 @@ static size_t ZSTD_decompressSequences( BYTE* const ostart = (BYTE* const)dst; BYTE* op = ostart; BYTE* const oend = ostart + maxDstSize; - size_t errorCode; + size_t errorCode, dumpsLength; const BYTE* litPtr = litStart; const BYTE* const litEnd = litStart + litSize; int nbSeq; @@ -1526,7 +1535,7 @@ static size_t ZSTD_decompressSequences( BYTE* const base = (BYTE*) (dctx->base); /* Build Decoding Tables */ - errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, + errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, DTableLL, DTableML, DTableOffb, ip, iend-ip); if (ZSTD_isError(errorCode)) return errorCode; @@ -1539,6 +1548,7 @@ static size_t ZSTD_decompressSequences( memset(&sequence, 0, sizeof(sequence)); seqState.dumps = dumps; + seqState.dumpsEnd = dumps + dumpsLength; seqState.prevOffset = 1; errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip); if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_corruption; @@ -1546,7 +1556,7 @@ static size_t ZSTD_decompressSequences( FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); - for ( ; (FSE_reloadDStream(&(seqState.DStream)) < FSE_DStream_completed) || (nbSeq>0) ; ) + for ( ; (FSE_reloadDStream(&(seqState.DStream)) <= FSE_DStream_completed) && (nbSeq>0) ; ) { size_t oneSeqSize; nbSeq--; @@ -1557,7 +1567,7 @@ static size_t ZSTD_decompressSequences( } /* check if reached exact end */ - if (FSE_reloadDStream(&(seqState.DStream)) > FSE_DStream_completed) return (size_t)-ZSTD_ERROR_corruption; /* requested too much : data is corrupted */ + if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (size_t)-ZSTD_ERROR_corruption; /* requested too much : data is corrupted */ if (nbSeq<0) return (size_t)-ZSTD_ERROR_corruption; /* requested too many sequences : data is corrupted */ /* last literal segment */ From 0fef5be1cc0d00a293e336e43565eff753a10c62 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Oct 2015 18:07:24 +0100 Subject: [PATCH 4/4] Fix fullbench --- programs/fullbench.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fullbench.c b/programs/fullbench.c index 3ca2199793a..d4515f39507 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -229,7 +229,7 @@ typedef struct static size_t g_cSize = 0; extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); -extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize); +extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize); size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { @@ -258,9 +258,10 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const { U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9]; const BYTE* dumps; + size_t length; int nbSeq; (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, DTableLL, DTableML, DTableOffb, buff2, g_cSize); + return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize); } size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)