Skip to content

Commit 781b247

Browse files
klauspostklauspost
authored
deflate: Improve level 7-9 (#739)
* deflate: Improve level 7-9 * Adjust tests * Add disabled next check * Tweak fuzz tests.
1 parent 60e376b commit 781b247

File tree

3 files changed

+141
-48
lines changed

3 files changed

+141
-48
lines changed

‎flate/deflate.go‎

Lines changed: 101 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,6 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
294294
}
295295
offset = 0
296296

297-
cGain := 0
298297
if d.chain < 100 {
299298
for i := prevHead; tries > 0; tries-- {
300299
if wEnd == win[i+length] {
@@ -322,18 +321,22 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
322321
return
323322
}
324323

324+
// Minimum gain to accept a match.
325+
cGain := 4
326+
325327
// Some like it higher (CSV), some like it lower (JSON)
326-
const baseCost = 6
328+
const baseCost = 3
327329
// Base is 4 bytes at with an additional cost.
328330
// Matches must be better than this.
331+
329332
for i := prevHead; tries > 0; tries-- {
330333
if wEnd == win[i+length] {
331334
n := matchLen(win[i:i+minMatchLook], wPos)
332335
if n > length {
333336
// Calculate gain. Estimate
334337
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
335338

336-
//fmt.Println(n, "gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]))
339+
//fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length)
337340
if newGain > cGain {
338341
length = n
339342
offset = pos - i
@@ -490,27 +493,103 @@ func (d *compressor) deflateLazy() {
490493
}
491494

492495
if prevLength >= minMatchLength && s.length <= prevLength {
493-
// Check for better match at end...
496+
// No better match, but check for better match at end...
494497
//
495-
// checkOff must be >=2 since we otherwise risk checking s.index
496-
// Offset of 2 seems to yield best results.
498+
// Skip forward a number of bytes.
499+
// Offset of 2 seems to yield best results. 3 is sometimes better.
497500
const checkOff = 2
498-
prevIndex := s.index - 1
499-
if prevIndex+prevLength+checkOff < s.maxInsertIndex {
500-
end := lookahead
501-
if lookahead > maxMatchLength {
502-
end = maxMatchLength
503-
}
504-
end += prevIndex
505-
idx := prevIndex + prevLength - (4 - checkOff)
506-
h := hash4(d.window[idx:])
507-
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + (4 - checkOff)
508-
if ch2 > minIndex {
509-
length := matchLen(d.window[prevIndex:end], d.window[ch2:])
510-
// It seems like a pure length metric is best.
511-
if length > prevLength {
512-
prevLength = length
513-
prevOffset = prevIndex - ch2
501+
502+
// Check all, except full length
503+
if prevLength < maxMatchLength-checkOff {
504+
prevIndex := s.index - 1
505+
if prevIndex+prevLength < s.maxInsertIndex {
506+
end := lookahead
507+
if lookahead > maxMatchLength+checkOff {
508+
end = maxMatchLength + checkOff
509+
}
510+
end += prevIndex
511+
512+
// Hash at match end.
513+
h := hash4(d.window[prevIndex+prevLength:])
514+
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
515+
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
516+
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
517+
// It seems like a pure length metric is best.
518+
if length > prevLength {
519+
prevLength = length
520+
prevOffset = prevIndex - ch2
521+
522+
// Extend back...
523+
for i := checkOff - 1; i >= 0; i-- {
524+
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] {
525+
// Emit tokens we "owe"
526+
for j := 0; j <= i; j++ {
527+
d.tokens.AddLiteral(d.window[prevIndex+j])
528+
if d.tokens.n == maxFlateBlockTokens {
529+
// The block includes the current character
530+
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
531+
return
532+
}
533+
d.tokens.Reset()
534+
}
535+
s.index++
536+
if s.index < s.maxInsertIndex {
537+
h := hash4(d.window[s.index:])
538+
ch := s.hashHead[h]
539+
s.chainHead = int(ch)
540+
s.hashPrev[s.index&windowMask] = ch
541+
s.hashHead[h] = uint32(s.index + s.hashOffset)
542+
}
543+
}
544+
break
545+
} else {
546+
prevLength++
547+
}
548+
}
549+
} else if false {
550+
// Check one further ahead.
551+
// Only rarely better, disabled for now.
552+
prevIndex++
553+
h := hash4(d.window[prevIndex+prevLength:])
554+
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
555+
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
556+
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
557+
// It seems like a pure length metric is best.
558+
if length > prevLength+checkOff {
559+
prevLength = length
560+
prevOffset = prevIndex - ch2
561+
prevIndex--
562+
563+
// Extend back...
564+
for i := checkOff; i >= 0; i-- {
565+
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] {
566+
// Emit tokens we "owe"
567+
for j := 0; j <= i; j++ {
568+
d.tokens.AddLiteral(d.window[prevIndex+j])
569+
if d.tokens.n == maxFlateBlockTokens {
570+
// The block includes the current character
571+
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
572+
return
573+
}
574+
d.tokens.Reset()
575+
}
576+
s.index++
577+
if s.index < s.maxInsertIndex {
578+
h := hash4(d.window[s.index:])
579+
ch := s.hashHead[h]
580+
s.chainHead = int(ch)
581+
s.hashPrev[s.index&windowMask] = ch
582+
s.hashHead[h] = uint32(s.index + s.hashOffset)
583+
}
584+
}
585+
break
586+
} else {
587+
prevLength++
588+
}
589+
}
590+
}
591+
}
592+
}
514593
}
515594
}
516595
}

‎flate/deflate_test.go‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -33,24 +33,24 @@ type reverseBitsTest struct {
3333
}
3434

3535
var deflateTests = []*deflateTest{
36-
{[]byte{}, 0, []byte{0x3, 0x0}},
37-
{[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
38-
{[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
39-
{[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
40-
41-
{[]byte{0x11}, 0, []byte{0x0, 0x1, 0x0, 0xfe, 0xff, 0x11, 0x3, 0x0}},
42-
{[]byte{0x11, 0x12}, 0, []byte{0x0, 0x2, 0x0, 0xfd, 0xff, 0x11, 0x12, 0x3, 0x0}},
43-
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0,
36+
0: {[]byte{}, 0, []byte{0x3, 0x0}},
37+
1: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
38+
2: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
39+
3: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
40+
41+
4: {[]byte{0x11}, 0, []byte{0x0, 0x1, 0x0, 0xfe, 0xff, 0x11, 0x3, 0x0}},
42+
5: {[]byte{0x11, 0x12}, 0, []byte{0x0, 0x2, 0x0, 0xfd, 0xff, 0x11, 0x12, 0x3, 0x0}},
43+
6: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0,
4444
[]byte{0x0, 0x8, 0x0, 0xf7, 0xff, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x3, 0x0},
4545
},
46-
{[]byte{}, 1, []byte{0x3, 0x0}},
47-
{[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
48-
{[]byte{0x11, 0x12}, BestCompression, []byte{0x12, 0x14, 0x2, 0xc, 0x0}},
49-
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, BestCompression, []byte{0x12, 0x84, 0x2, 0xc0, 0x0}},
50-
{[]byte{}, 9, []byte{0x3, 0x0}},
51-
{[]byte{0x11}, 9, []byte{0x12, 0x4, 0xc, 0x0}},
52-
{[]byte{0x11, 0x12}, 9, []byte{0x12, 0x14, 0x2, 0xc, 0x0}},
53-
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{0x12, 0x84, 0x2, 0xc0, 0x0}},
46+
7: {[]byte{}, 1, []byte{0x3, 0x0}},
47+
8: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}},
48+
9: {[]byte{0x11, 0x12}, BestCompression, []byte{0x12, 0x14, 0x2, 0xc, 0x0}},
49+
10: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, BestCompression, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}},
50+
11: {[]byte{}, 9, []byte{0x3, 0x0}},
51+
12: {[]byte{0x11}, 9, []byte{0x12, 0x4, 0xc, 0x0}},
52+
13: {[]byte{0x11, 0x12}, 9, []byte{0x12, 0x14, 0x2, 0xc, 0x0}},
53+
14: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}},
5454
}
5555

5656
var deflateInflateTests = []*deflateInflateTest{
@@ -120,7 +120,7 @@ func TestDeflate(t *testing.T) {
120120
w.Write(h.in)
121121
w.Close()
122122
if !bytes.Equal(buf.Bytes(), h.out) {
123-
t.Errorf("%d: Deflate(%d, %x) = \n%#v, want \n%#v", i, h.level, h.in, buf.Bytes(), h.out)
123+
t.Errorf("%d: Deflate(%d, %x) got \n%#v, want \n%#v", i, h.level, h.in, buf.Bytes(), h.out)
124124
}
125125
}
126126
}
@@ -331,6 +331,8 @@ func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name str
331331
}
332332
r.Close()
333333
if !bytes.Equal(input, out) {
334+
os.WriteFile("testdata/fails/"+t.Name()+".got", out, os.ModePerm)
335+
os.WriteFile("testdata/fails/"+t.Name()+".want", input, os.ModePerm)
334336
t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name)
335337
return
336338
}

‎flate/fuzz_test.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,36 @@ package flate
55

66
import (
77
"bytes"
8+
"flag"
89
"io"
10+
"os"
911
"strconv"
1012
"testing"
1113

1214
"github.com/klauspost/compress/internal/fuzz"
1315
)
1416

17+
// Fuzzing tweaks:
18+
var fuzzStartF = flag.Int("start", HuffmanOnly, "Start fuzzing at this level")
19+
var fuzzEndF = flag.Int("end", BestCompression, "End fuzzing at this level (inclusive)")
20+
var fuzzMaxF = flag.Int("max", 1<<20, "Maximum input size")
21+
var fuzzSLF = flag.Bool("sl", true, "Include stateless encodes")
22+
23+
func TestMain(m *testing.M) {
24+
flag.Parse()
25+
os.Exit(m.Run())
26+
}
27+
1528
func FuzzEncoding(f *testing.F) {
1629
fuzz.AddFromZip(f, "testdata/regression.zip", true, false)
1730
fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", true, testing.Short())
1831
fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", false, testing.Short())
19-
// Fuzzing tweaks:
20-
const (
21-
// Test a subset of encoders.
22-
startFuzz = HuffmanOnly
23-
endFuzz = BestCompression
2432

25-
// Max input size:
26-
maxSize = 1 << 20
27-
)
33+
startFuzz := *fuzzStartF
34+
endFuzz := *fuzzEndF
35+
maxSize := *fuzzMaxF
36+
stateless := *fuzzSLF
37+
2838
decoder := NewReader(nil)
2939
buf := new(bytes.Buffer)
3040
encs := make([]*Writer, endFuzz-startFuzz+1)
@@ -88,7 +98,9 @@ func FuzzEncoding(f *testing.F) {
8898
t.Fatal(msg + "not equal")
8999
}
90100
}
91-
101+
if !stateless {
102+
return
103+
}
92104
// Split into two and use history...
93105
buf.Reset()
94106
err := StatelessDeflate(buf, data[:len(data)/2], false, nil)

0 commit comments

Comments
 (0)