Skip to content

Commit e42fcb6

Browse files
add manual flag to mmap dictionary
1 parent 8a189b1 commit e42fcb6

File tree

4 files changed

+20
-5
lines changed

4 files changed

+20
-5
lines changed

programs/fileio.c

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,11 @@ void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
485485
prefs->passThrough = (value != 0);
486486
}
487487

488+
void FIO_setMMapDict(FIO_prefs_t* const prefs, int value)
489+
{
490+
prefs->mmapDict = value;
491+
}
492+
488493
/* FIO_ctx_t functions */
489494

490495
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@@ -1028,7 +1033,7 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
10281033
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
10291034
const char* dictFileName, unsigned long long const maxSrcFileSize,
10301035
int cLevel, ZSTD_compressionParameters comprParams) {
1031-
int mmapDict = 0;
1036+
int mmapDict = prefs->mmapDict;
10321037
cRess_t ress;
10331038
memset(&ress, 0, sizeof(ress));
10341039

@@ -1045,10 +1050,12 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
10451050
if (prefs->patchFromMode) {
10461051
U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
10471052
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
1048-
mmapDict = dictSize > prefs->memLimit;
1053+
mmapDict |= dictSize > prefs->memLimit;
10491054
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
10501055
}
10511056

1057+
DISPLAYLEVEL(1, "mmapCDict: %i", mmapDict);
1058+
10521059
ress.mmapDict = mmapDict;
10531060

10541061
if (!ress.mmapDict) {
@@ -2136,7 +2143,7 @@ typedef struct {
21362143

21372144
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
21382145
{
2139-
int mmapDict = 0;
2146+
int mmapDict = prefs->mmapDict;
21402147
stat_t statbuf;
21412148
dRess_t ress;
21422149
memset(&ress, 0, sizeof(ress));
@@ -2145,10 +2152,12 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
21452152

21462153
if (prefs->patchFromMode){
21472154
U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
2148-
mmapDict = dictSize > prefs->memLimit;
2155+
mmapDict |= dictSize > prefs->memLimit;
21492156
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
21502157
}
21512158

2159+
DISPLAYLEVEL(1, "mmapDDict: %i", mmapDict);
2160+
21522161
/* Allocation */
21532162
ress.mmapDict = mmapDict;
21542163
ress.dctx = ZSTD_createDStream();

programs/fileio.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
106106
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
107107
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
108108
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
109+
void FIO_setMMapDict(FIO_prefs_t* const prefs, int value);
109110

110111
/* FIO_ctx_t functions */
111112
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

programs/fileio_types.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ typedef struct FIO_prefs_s {
6969
int contentSize;
7070
int allowBlockDevices;
7171
int passThrough;
72+
int mmapDict;
7273
} FIO_prefs_t;
7374

7475
#endif /* FILEIO_TYPES_HEADER */

programs/zstdcli.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ static void usage_advanced(const char* programName)
254254

255255
DISPLAYOUT("\n");
256256
DISPLAYOUT(" --format=zstd Compress files to the `.zst` format. [Default]\n");
257+
DISPLAYOUT(" --mmap-dict Memory-map dictionary file rather than mallocing and loading all at once");
257258
#ifdef ZSTD_GZCOMPRESS
258259
DISPLAYOUT(" --format=gzip Compress files to the `.gz` format.\n");
259260
#endif
@@ -850,7 +851,8 @@ int main(int argCount, const char* argv[])
850851
showDefaultCParams = 0,
851852
ultra=0,
852853
contentSize=1,
853-
removeSrcFile=0;
854+
removeSrcFile=0,
855+
mmapDict=0;
854856
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
855857
FIO_compressionType_t cType = FIO_zstdCompression;
856858
unsigned nbWorkers = 0;
@@ -984,6 +986,7 @@ int main(int argCount, const char* argv[])
984986
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
985987
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
986988
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
989+
if (!strcmp(argument, "--mmap-dict")) { mmapDict = 1; continue; }
987990
#ifdef ZSTD_GZCOMPRESS
988991
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
989992
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
@@ -1526,6 +1529,7 @@ int main(int argCount, const char* argv[])
15261529
FIO_setNotificationLevel(g_displayLevel);
15271530
FIO_setAllowBlockDevices(prefs, allowBlockDevices);
15281531
FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
1532+
FIO_setMMapDict(prefs, mmapDict);
15291533
if (memLimit == 0) {
15301534
if (compressionParams.windowLog == 0) {
15311535
memLimit = (U32)1 << g_defaultMaxWindowLog;

0 commit comments

Comments
 (0)