Merge pull request #531 from facebook/dev

Zstandard v1.1.3

Author: Cyan4973

.buckconfig

@@ -0,0 +1,9 @@
+[cxx]
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
+  cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
+  cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations
+  gtest_dep = //contrib/pzstd:gtest
+
+[httpserver]
+  port = 0

.buckversion

@@ -0,0 +1 @@
+c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b

.gitignore

@@ -15,6 +15,7 @@
 
 # Executables
 zstd
+zstdmt
 *.exe
 *.out
 *.app
@@ -36,3 +37,5 @@ googletest/
 
 # Directories
 bin/
+.buckd/
+buck-out/

.travis.yml

@@ -8,11 +8,7 @@ matrix:
 
 
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
-    - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install"
-      os: linux
-      sudo: false
-
-    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
+    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
       os: linux
       sudo: false
       language: cpp
@@ -37,7 +33,7 @@ matrix:
 
 
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
-    - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C tests test-gzstd && make -C programs clean && make -C tests versionsTest"
+    - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest"
       os: linux
       sudo: required
 
@@ -69,7 +65,7 @@ matrix:
 
 
     # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
+    - env: Ubu=14.04 Cmd="make armtest"
       dist: trusty
       sudo: required
       addons:
@@ -78,11 +74,30 @@ matrix:
             - qemu-system-arm
             - qemu-user-static
             - gcc-arm-linux-gnueabi
-            - libc6-dev-armel-cross 
+            - libc6-dev-armel-cross
+
+    - env: Ubu=14.04 Cmd="make aarch64test"
+      dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-arm
+            - qemu-user-static
             - gcc-aarch64-linux-gnu
             - libc6-dev-arm64-cross
 
-    - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test'
+    - env: Ubu=14.04 Cmd='make ppctest'
+      dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-ppc
+            - qemu-user-static
+            - gcc-powerpc-linux-gnu
+
+    - env: Ubu=14.04 Cmd='make ppc64test'
       dist: trusty
       sudo: required
       addons:
@@ -101,33 +116,43 @@ matrix:
           packages:
             - valgrind
 
-    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
+
+
+    # other feature branches => short tests
+    - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install"
+      os: linux
+      sudo: false
+
+    - env: Ubu=14.04 Cmd="make -C tests test32"
       os: linux
       dist: trusty
       sudo: required
-      install:
-        - export CXX="g++-4.8" CC="gcc-4.8"
       addons:
         apt:
           packages:
             - libc6-dev-i386
-            - g++-multilib
-            - gcc-4.8
-            - gcc-4.8-multilib
-            - g++-4.8
-            - g++-4.8-multilib
+            - gcc-multilib
 
-    - env: Ubu=14.04 Cmd="make -C tests test32"
+    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
+                       && make c99test && make clean && make gnu99test && make clean
+                       && make clangtest && make clean && make -C contrib/pzstd googletest32
+                       && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
       os: linux
       dist: trusty
       sudo: required
+      install:
+        - export CXX="g++-4.8" CC="gcc-4.8"
       addons:
         apt:
           packages:
             - libc6-dev-i386
-            - gcc-multilib
+            - g++-multilib
+            - gcc-4.8
+            - gcc-4.8-multilib
+            - g++-4.8
+            - g++-4.8-multilib
 
-    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
+    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
       os: linux
       dist: trusty
       sudo: required
@@ -144,5 +169,7 @@ matrix:
 
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  # - if [ $JOB_NUMBER -eq 9 ] || [ $JOB_NUMBER -eq 10 ]; then sh -c "$Cmd"; fi
-  - sh -c "$Cmd"
+  #  dev => normal tests;  other feature branches => short tests (number > 11)
+  - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi
+  #  master => long tests, as this is the final step towards a Release
+  - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi

Makefile

@@ -23,13 +23,23 @@ EXT =
 endif
 
 .PHONY: default
-default: lib zstd
+default: lib zstd-release
 
 .PHONY: all
-all:
-	$(MAKE) -C $(ZSTDDIR) $@
-	$(MAKE) -C $(PRGDIR) $@ zstd32
-	$(MAKE) -C $(TESTDIR) $@ all32
+all: allmost
+	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
+
+.PHONY: allmost
+allmost:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+	$(MAKE) -C $(ZWRAPDIR) all
+
+.PHONY: all32
+all32:
+	$(MAKE) -C $(PRGDIR) zstd32
+	$(MAKE) -C $(TESTDIR) all32
 
 .PHONY: lib
 lib:
@@ -40,6 +50,16 @@ zstd:
 	@$(MAKE) -C $(PRGDIR) $@
 	cp $(PRGDIR)/zstd$(EXT) .
 
+.PHONY: zstd-release
+zstd-release:
+	@$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd$(EXT) .
+
+.PHONY: zstdmt
+zstdmt:
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+
 .PHONY: zlibwrapper
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test
@@ -54,7 +74,8 @@ clean:
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	@$(RM) zstd$(EXT) tmp*
+	@$(MAKE) -C examples/ $@ > $(VOID)
+	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@echo Cleaning completed
 
 
@@ -77,7 +98,7 @@ travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
 gpptest: clean
-	$(MAKE) -C programs all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
 gcc5test: clean
 	gcc-5 -v
@@ -148,13 +169,13 @@ gnu90test: clean
 	CFLAGS="-std=gnu90" $(MAKE) all
 
 c99test: clean
-	CFLAGS="-std=c99" $(MAKE) all
+	CFLAGS="-std=c99" $(MAKE) allmost
 
 gnu99test: clean
 	CFLAGS="-std=gnu99" $(MAKE) all
 
 c11test: clean
-	CFLAGS="-std=c11" $(MAKE) all
+	CFLAGS="-std=c11" $(MAKE) allmost
 
 bmix64test: clean
 	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test

NEWS

@@ -1,3 +1,19 @@
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Andrew Purcell
+
 v1.1.2
 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
 API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()

README.md

@@ -2,7 +2,7 @@
  targeting real-time compression scenarios at zlib-level and better compression ratios.
 
 It is provided as an open-source BSD-licensed **C** library,
-and a command line utility producing and decoding `.zst` compressed files.
+and a command line utility producing and decoding `.zst` and `.gz` files.
 For other programming languages,
 you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
@@ -47,18 +47,27 @@ For a larger picture including very slow modes, [click on this link](doc/images/
 
 ### The case for Small Data compression
 
-Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. The smaller the amount of data to compress, the more difficult it is to achieve any significant compression.
+Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
 
-This problem is common to many compression algorithms. The reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new file, there is no "past" to build upon.
+The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon.
 
-To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored in a file called "dictionary", which can be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically:
+To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data.
+Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
+Using this dictionary, the compression ratio achievable on small data improves dramatically.
 
-![Compressing Small Data](doc/images/smallData.png "Compressing Small Data")
+The following example uses the `github-users` [sample set](https://www.dropbox.com/s/mnktkomhkjbf1i2/github_users.tar.zst?dl=0), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
+It consists of roughly 10K records weighting about 1KB each.
 
-These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+Compression Ratio | Compression Speed | Decompression Speed
+------------------|-------------------|--------------------
+![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed")
 
-Dictionary works if there is some correlation in a family of small data (there is no _universal dictionary_).
-Hence, deploying one dictionary per type of data will provide the greatest benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will rely more and more on previously decoded content to compress the rest of the file.
+
+These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds.
+
+Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_).
+Hence, deploying one dictionary per type of data will provide the greatest benefits.
+Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
 
 #### Dictionary compression How To :
 
@@ -68,11 +77,12 @@ Hence, deploying one dictionary per type of data will provide the greatest benef
 
 2) Compress with dictionary
 
-`zstd FILE -D dictionaryName`
+`zstd -D dictionaryName FILE`
 
 3) Decompress with dictionary
 
-`zstd --decompress FILE.zst -D dictionaryName`
+`zstd -D dictionaryName --decompress FILE.zst`
+
 
 ### Build
 

appveyor.yml

@@ -2,10 +2,10 @@ version: 1.0.{build}
 environment:
   matrix:
   - COMPILER: "gcc"
-    MAKE_PARAMS: "test"
+    MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
     PLATFORM: "mingw64"
   - COMPILER: "gcc"
-    MAKE_PARAMS: "test"
+    MAKE_PARAMS: "make test"
     PLATFORM: "mingw32"
   - COMPILER: "visual"
     CONFIGURATION: "Debug"
@@ -46,6 +46,13 @@ build_script:
       ECHO make %CLANG_PARAMS% &&
       make %CLANG_PARAMS% &&
       COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
+      ECHO *** &&
+      ECHO *** Building cmake for %PLATFORM% &&
+      ECHO *** &&
+      mkdir build\cmake\build &&
+      cd build\cmake\build &&
+      cmake -G "Visual Studio 14 2015 Win64" .. &&
+      cd ..\..\.. &&
       make clean &&
       ECHO *** &&
       ECHO *** Building pzstd for %PLATFORM% &&
@@ -62,8 +69,8 @@ build_script:
       ECHO *** &&
       make -v &&
       cc -v &&
-      ECHO make %MAKE_PARAMS% &&
-      make %MAKE_PARAMS%
+      ECHO %MAKE_PARAMS% &&
+      sh -c %MAKE_PARAMS%
     )
   - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
       COPY programs\zstd.exe bin\zstd.exe &&