Merge pull request #3551 from embg/seq_prod_fuzz

Provide an interface for fuzzing sequence producer plugins

Author: embg

tests/fuzz/Makefile

@@ -34,13 +34,13 @@ ZSTDDIR = ../../lib
 PRGDIR = ../../programs
 CONTRIBDIR = ../../contrib
 
-# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
-MATCHFINDER_DIR = $(CONTRIBDIR)/externalSequenceProducer
-MATCHFINDER_SRC = $(MATCHFINDER_DIR)/sequence_producer.c
+DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer
+DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c
+THIRD_PARTY_SEQ_PROD_OBJ ?=
 
 FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 	-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(MATCHFINDER_DIR) \
+	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \
 	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -75,7 +75,7 @@ FUZZ_SRC       := \
 	$(ZSTDCOMP_SRC) \
 	$(ZSTDDICT_SRC) \
 	$(ZSTDLEGACY_SRC) \
-	$(MATCHFINDER_SRC)
+	$(DEFAULT_SEQ_PROD_SRC)
 FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
 
 FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
@@ -84,21 +84,23 @@ FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
 FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
 FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
 FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
-FUZZ_D_OBJ7 := $(subst $(MATCHFINDER_DIR)/,d_matchfinder_,$(FUZZ_D_OBJ6))
+FUZZ_D_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,d_default_seq_prod_,$(FUZZ_D_OBJ6))
 FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
 FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
-FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ9:.S=.o)
+FUZZ_D_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_D_OBJ9)
+FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ10:.S=.o)
 
 FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
 FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
 FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2))
 FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
 FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
 FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
-FUZZ_RT_OBJ7 := $(subst $(MATCHFINDER_DIR)/,rt_matchfinder_,$(FUZZ_RT_OBJ6))
+FUZZ_RT_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,rt_default_seq_prod_,$(FUZZ_RT_OBJ6))
 FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
 FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
-FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ9:.S=.o)
+FUZZ_RT_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_RT_OBJ9)
+FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ10:.S=.o)
 
 .PHONY: default all clean cleanall
 
@@ -151,7 +153,7 @@ rt_prg_%.o: $(PRGDIR)/%.c
 rt_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
-rt_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+rt_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
 d_lib_common_%.o: $(ZSTDDIR)/common/%.c
@@ -178,7 +180,7 @@ d_prg_%.o: $(PRGDIR)/%.c
 d_fuzz_%.o: %.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
-d_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
+d_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
 simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o

tests/fuzz/README.md

@@ -61,7 +61,7 @@ Alternatively, you can fuzz all targets in parallel, using one core per target:
 ```
 python3 ./fuzz.py list | xargs -P$(python3 ./fuzz.py list | wc -l) -I__ sh -c "python3 ./fuzz.py libfuzzer __ 2>&1 | tee __.log"
 ```
-Either way, to double-check that no crashes were found, run `ls corpora/*crash`. 
+Either way, to double-check that no crashes were found, run `ls corpora/*crash`.
 If any crashes were found, you can use the hashes to reproduce them.
 
 ## LibFuzzer
@@ -113,3 +113,7 @@ CC=clang CXX=clang++ ./fuzz.py build all --enable-asan --enable-ubsan
 CC=clang CXX=clang++ ./fuzz.py build all --enable-msan
 ./fuzz.py regression all
 ```
+
+## Fuzzing a custom sequence producer plugin
+Sequence producer plugin authors can use the zstd fuzzers to stress-test their code.
+See the documentation in `fuzz_third_party_seq_prod.h` for details.

tests/fuzz/block_round_trip.c

@@ -23,6 +23,7 @@
 #include "zstd.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
@@ -54,6 +55,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -95,5 +98,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/decompress_dstSize_tooSmall.c

@@ -22,12 +22,15 @@
 #include "zstd_errors.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -66,5 +69,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/dictionary_decompress.c

@@ -20,11 +20,14 @@
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 static ZSTD_DCtx *dctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -69,5 +72,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/dictionary_loader.c

@@ -20,6 +20,7 @@
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 /**
  * Compresses the data and returns the compressed size or an error.
@@ -35,7 +36,7 @@ static size_t compress(void* compressed, size_t compressedCapacity,
     if (refPrefix)
         FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced(
             cctx, dict, dictSize, dictContentType));
-    else 
+    else
         FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
             cctx, dict, dictSize, dictLoadMethod, dictContentType));
     size_t const compressedSize = ZSTD_compress2(
@@ -67,6 +68,7 @@ static size_t decompress(void* result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
     int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0;
     ZSTD_dictLoadMethod_e const dlm =
@@ -99,5 +101,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     free(cBuf);
     free(rBuf);
     FUZZ_dataProducer_free(producer);
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/dictionary_round_trip.c

@@ -21,6 +21,7 @@
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
@@ -108,6 +109,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -147,5 +150,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/dictionary_stream_round_trip.c

@@ -22,6 +22,7 @@
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
 
 ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
@@ -147,6 +148,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    FUZZ_SEQ_PROD_SETUP();
     size_t neededBufSize;
 
     /* Give a random portion of src data to the producer, to use for
@@ -202,5 +204,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;
 #endif
+    FUZZ_SEQ_PROD_TEARDOWN();
     return 0;
 }

tests/fuzz/fuzz.h

@@ -31,6 +31,11 @@
  *        This is the canonical flag to enable deterministic builds for fuzzing.
  *        Changes to zstd for fuzzing are gated behind this define.
  *        It is recommended to define this when building zstd for fuzzing.
+ * @param FUZZ_THIRD_PARTY_SEQ_PROD
+ *        This flag allows sequence producer plugin authors to replace the built-in
+ *        default sequence producer with their own code. If you are not a plugin
+ *        author, you should not define this flag. See the docs at
+ *        fuzz_third_party_seq_prod.h for more information.
  */
 
 #ifndef FUZZ_H

tests/fuzz/fuzz.py

@@ -78,6 +78,7 @@ def __init__(self, input_type, frame_type=FrameType.ZSTD):
 CXXFLAGS = os.environ.get('CXXFLAGS', CFLAGS)
 LDFLAGS = os.environ.get('LDFLAGS', '')
 MFLAGS = os.environ.get('MFLAGS', '-j')
+THIRD_PARTY_SEQ_PROD_OBJ = os.environ.get('THIRD_PARTY_SEQ_PROD_OBJ', '')
 
 # Fuzzing environment variables
 LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a')
@@ -319,6 +320,12 @@ def build_parser(args):
         dest='stateful_fuzzing',
         action='store_true',
         help='Reuse contexts between runs (makes reproduction impossible)')
+    parser.add_argument(
+        '--custom-seq-prod',
+        dest='third_party_seq_prod_obj',
+        type=str,
+        default=THIRD_PARTY_SEQ_PROD_OBJ,
+        help='Path to an object file with symbols for fuzzing your sequence producer plugin.')
     parser.add_argument(
         '--cc',
         dest='cc',
@@ -450,6 +457,10 @@ def build(args):
     if args.stateful_fuzzing:
         cppflags += ['-DSTATEFUL_FUZZING']
 
+    if args.third_party_seq_prod_obj:
+        cppflags += ['-DFUZZ_THIRD_PARTY_SEQ_PROD']
+        mflags += ['THIRD_PARTY_SEQ_PROD_OBJ={}'.format(args.third_party_seq_prod_obj)]
+
     if args.fuzzing_mode:
         cppflags += ['-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION']