Merge pull request #4171 from facebook/lvl3_ratio+

Improve compression ratio of levels 3 & 4

Author: Cyan4973

.github/workflows/commit.yml

@@ -3,7 +3,12 @@ on:
   push:
     branches:
     - dev
+  pull_request:
+    branches:
+    - dev
+
 permissions: read-all
+
 jobs:
   short-tests-0:
     runs-on: ubuntu-latest
@@ -27,6 +32,7 @@ jobs:
         make -j regressiontest; make clean
         make shortest; make clean
         make cxxtest; make clean
+
   short-tests-1:
     runs-on: ubuntu-latest
     services:
@@ -49,6 +55,7 @@ jobs:
         make aarch64build V=1; make clean
         make -C tests test-legacy test-longmatch; make clean
         make -C lib libzstd-nomt; make clean
+
   regression-test:
     runs-on: ubuntu-latest
     services:

lib/compress/zstd_double_fast.c

@@ -252,21 +252,23 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
 
 _search_next_long:
 
-        /* check prefix long +1 match */
-        if (idxl1 > prefixLowestIndex) {
-            if (MEM_read64(matchl1) == MEM_read64(ip1)) {
+        /* short match found: let's check for a longer one */
+        mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
+        offset = (U32)(ip - matchs0);
+
+        /* check long match at +1 position */
+        if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
+            size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
+            if (l1len > mLength) {
+                /* use the long match instead */
                 ip = ip1;
-                mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
+                mLength = l1len;
                 offset = (U32)(ip-matchl1);
-                while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
-                goto _match_found;
+                matchs0 = matchl1;
             }
         }
 
-        /* if no long +1 match, explore the short match we found */
-        mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
-        offset = (U32)(ip - matchs0);
-        while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
+        while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
 
         /* fall-through */
 

tests/fuzz/.gitignore

@@ -23,6 +23,8 @@ fuzz-*.log
 rt_lib_*
 d_lib_*
 crash-*
+decompress_cross_format
+generate_sequences
 
 # misc
 trace