Skip to content

Commit adcffd0

Browse files
Cyan4973Cyan4973
committed
Merge pull request #1 from Cyan4973/dev
Dev
2 parents 04d8e15 + 759433d commit adcffd0

File tree

4 files changed

+98
-39
lines changed

4 files changed

+98
-39
lines changed

‎lib/zstd.c‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
10041004
//U32 limit = 4 * BLOCKSIZE;
10051005
//const U32 updateRate = 2 * BLOCKSIZE;
10061006

1007-
// Init
1007+
/* Init */
10081008
if (ctx->base==NULL) ctx->base = src, ctx->current=0;
1009-
if (src != ctx->base + ctx->current) // not contiguous
1009+
if (src != ctx->base + ctx->current) /* not contiguous */
10101010
{
10111011
ZSTD_resetCCtx(ctx);
10121012
ctx->base = src;
@@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
10351035
}
10361036
*/
10371037

1038-
// compress
1038+
/* compress */
10391039
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
10401040
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
10411041
if (cSize == 0)
10421042
{
1043-
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
1043+
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
10441044
if (ZSTD_isError(cSize)) return cSize;
10451045
}
10461046
else
@@ -1084,23 +1084,23 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
10841084
BYTE* const ostart = dst;
10851085
BYTE* op = ostart;
10861086

1087-
// Header
1087+
/* Header */
10881088
{
10891089
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
10901090
if(ZSTD_isError(headerSize)) return headerSize;
10911091
op += headerSize;
10921092
maxDstSize -= headerSize;
10931093
}
10941094

1095-
// Compression
1095+
/* Compression */
10961096
{
10971097
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
10981098
if (ZSTD_isError(cSize)) return cSize;
10991099
op += cSize;
11001100
maxDstSize -= cSize;
11011101
}
11021102

1103-
// Close frame
1103+
/* Close frame */
11041104
{
11051105
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
11061106
if(ZSTD_isError(endSize)) return endSize;

‎programs/Makefile‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
127127
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
128128
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
129129

130-
test-fullbench: fullbench
130+
test-fullbench: fullbench datagen
131131
./fullbench -i1
132+
./datagen -P0 -g516K > tmp
133+
./fullbench -i1 tmp
134+
@rm tmp
132135

133-
test-fullbench32: fullbench32
136+
test-fullbench32: fullbench32 datagen
134137
./fullbench32 -i1
138+
./datagen -P0 -g516K > tmp
139+
./fullbench32 -i1 tmp
140+
@rm tmp
135141

136142
test-fuzzer: fuzzer
137143
./fuzzer

‎programs/datagen.c‎

Lines changed: 62 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,24 @@
5656
#endif
5757

5858

59+
/**************************************
60+
* OS-specific Includes
61+
**************************************/
62+
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
63+
# include <fcntl.h> /* _O_BINARY */
64+
# include <io.h> /* _setmode, _isatty */
65+
# ifdef __MINGW32__
66+
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
67+
# endif
68+
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
69+
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
70+
#else
71+
# include <unistd.h> /* isatty */
72+
# define SET_BINARY_MODE(file)
73+
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
74+
#endif
75+
76+
5977
/**************************************
6078
* Constants
6179
**************************************/
@@ -85,7 +103,6 @@
85103
* Local Parameters
86104
**************************************/
87105
static unsigned no_prompt = 0;
88-
static char* programName;
89106
static unsigned displayLevel = 2;
90107

91108

@@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
97114
{
98115
U32 rand32 = *src;
99116
rand32 *= PRIME1;
100-
rand32 += PRIME2;
117+
rand32 ^= PRIME2;
101118
rand32 = CDG_rotl32(rand32, 13);
102119
*src = rand32;
103120
return rand32;
@@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
106123

107124
#define LTSIZE 8192
108125
#define LTMASK (LTSIZE-1)
109-
static const char firstChar = '(';
110-
static const char lastChar = '}';
111126
static void* CDG_createLiteralDistrib(double ld)
112127
{
113-
char* lt = malloc(LTSIZE);
128+
BYTE* lt = malloc(LTSIZE);
114129
U32 i = 0;
115-
char character = '0';
130+
BYTE character = '0';
131+
BYTE firstChar = '(';
132+
BYTE lastChar = '}';
116133

134+
if (ld==0.0)
135+
{
136+
character = 0;
137+
firstChar = 0;
138+
lastChar =255;
139+
}
117140
while (i<LTSIZE)
118141
{
119142
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@@ -137,18 +160,19 @@ static char CDG_genChar(U32* seed, const void* ltctx)
137160
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
138161
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15)
139162
#define CDG_DICTSIZE (32 KB)
140-
static void CDG_generate(U64 size, U32* seed, double matchProba)
163+
static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba)
141164
{
142165
BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1];
143166
BYTE* buff = fullbuff + CDG_DICTSIZE;
144167
U64 total=0;
145168
U32 P32 = (U32)(32768 * matchProba);
146169
U32 pos=1;
147170
U32 genBlockSize = 128 KB;
148-
double literalDistrib = 0.13;
149-
void* ldctx = CDG_createLiteralDistrib(literalDistrib);
171+
void* ldctx = CDG_createLiteralDistrib(litProba);
172+
FILE* fout = stdout;
150173

151-
/* Build initial prefix */
174+
/* init */
175+
SET_BINARY_MODE(stdout);
152176
fullbuff[0] = CDG_genChar(seed, ldctx);
153177
while (pos<32 KB)
154178
{
@@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
207231
}
208232
}
209233

210-
/* output datagen */
211-
pos=0;
212-
for (;pos+512<=genBlockSize;pos+=512)
213-
printf("%512.512s", buff+pos);
214-
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
234+
/* output generated data */
235+
fwrite(buff, 1, genBlockSize, fout);
215236
/* Regenerate prefix */
216237
memcpy(fullbuff, buff + 96 KB, 32 KB);
217238
}
@@ -221,7 +242,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
221242
/*********************************************************
222243
* Command line
223244
*********************************************************/
224-
static int CDG_usage(void)
245+
static int CDG_usage(char* programName)
225246
{
226247
DISPLAY( "Compressible data generator\n");
227248
DISPLAY( "Usage :\n");
@@ -239,9 +260,11 @@ static int CDG_usage(void)
239260
int main(int argc, char** argv)
240261
{
241262
int argNb;
242-
int proba = CDG_COMPRESSIBILITY_DEFAULT;
263+
double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
264+
double litProba = proba / 3.6;
243265
U64 size = CDG_SIZE_DEFAULT;
244266
U32 seed = CDG_SEED_DEFAULT;
267+
char* programName;
245268

246269
/* Check command line */
247270
programName = argv[0];
@@ -262,7 +285,7 @@ int main(int argc, char** argv)
262285
switch(*argument)
263286
{
264287
case 'h':
265-
return CDG_usage();
288+
return CDG_usage(programName);
266289
case 'g':
267290
argument++;
268291
size=0;
@@ -287,23 +310,37 @@ int main(int argc, char** argv)
287310
argument++;
288311
}
289312
break;
290-
case 'p':
313+
case 'P':
291314
argument++;
292-
proba=0;
315+
proba=0.0;
293316
while ((*argument>='0') && (*argument<='9'))
294317
{
295318
proba *= 10;
296319
proba += *argument - '0';
297320
argument++;
298321
}
299-
if (proba<0) proba=0;
300-
if (proba>100) proba=100;
322+
if (proba>100.) proba=100.;
323+
proba /= 100.;
324+
litProba = proba / 4.;
325+
break;
326+
case 'L':
327+
argument++;
328+
litProba=0.;
329+
while ((*argument>='0') && (*argument<='9'))
330+
{
331+
litProba *= 10;
332+
litProba += *argument - '0';
333+
argument++;
334+
}
335+
if (litProba>100.) litProba=100.;
336+
litProba /= 100.;
301337
break;
302338
case 'v':
303339
displayLevel = 4;
304340
argument++;
305341
break;
306-
default: ;
342+
default:
343+
return CDG_usage(programName);
307344
}
308345
}
309346

@@ -312,9 +349,9 @@ int main(int argc, char** argv)
312349

313350
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
314351
DISPLAYLEVEL(3, "Seed = %u \n", seed);
315-
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
352+
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
316353

317-
CDG_generate(size, &seed, ((double)proba) / 100);
354+
CDG_generate(size, &seed, proba, litProba);
318355

319356
return 0;
320357
}

‎programs/fullbench.c‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
383383
}
384384

385385
/* Allocation */
386-
dstBuffSize = srcSize + 512;
386+
dstBuffSize = ZSTD_compressBound(srcSize);
387387
dstBuff = malloc(dstBuffSize);
388388
buff2 = malloc(dstBuffSize);
389389
if ((!dstBuff) || (!buff2))
@@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
403403
{
404404
blockProperties_t bp;
405405
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
406+
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
407+
if (bp.blockType != bt_compressed)
408+
{
409+
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
410+
free(dstBuff);
411+
free(buff2);
412+
return 0;
413+
}
406414
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
407415
memcpy(buff2, dstBuff+7, g_cSize);
408416
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
@@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
418426
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
419427
ip += 4; // Jump magic Number
420428
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
429+
if (bp.blockType != bt_compressed)
430+
{
431+
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
432+
free(dstBuff);
433+
free(buff2);
434+
return 0;
435+
}
421436
iend = ip + 3 + blockSize; // Get end of first block
422437
ip += 3; // jump first block header
423438
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
@@ -450,14 +465,15 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
450465
default : ;
451466
}
452467

468+
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
469+
453470
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
454471
{
455472
double averageTime;
456473
int milliTime;
457474
U32 nbRounds=0;
458475

459476
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
460-
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
461477

462478
milliTime = BMK_GetMilliStart();
463479
while(BMK_GetMilliStart() == milliTime);
@@ -545,7 +561,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
545561
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
546562
}
547563

548-
// Alloc
564+
/* Alloc */
549565
origBuff = (char*) malloc((size_t)benchedSize);
550566
if(!origBuff)
551567
{
@@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
554570
return 12;
555571
}
556572

557-
// Fill input buffer
573+
/* Fill input buffer */
558574
DISPLAY("Loading %s... \r", inFileName);
559575
readSize = fread(origBuff, 1, benchedSize, inFile);
560576
fclose(inFile);
@@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
566582
return 13;
567583
}
568584

569-
// bench
585+
/* bench */
570586
DISPLAY("\r%79s\r", "");
571587
DISPLAY(" %s : \n", inFileName);
572588
if (benchNb)

0 commit comments

Comments
 (0)