deflate_compress: optimize gen_codewords()

This can be slightly faster.

.cirrus.yml

@@ -0,0 +1,8 @@
+task:
+  freebsd_instance:
+    matrix:
+      - image_family: freebsd-12-3
+      - image_family: freebsd-13-0
+  install_script: pkg install -y gmake
+  script:
+    - gmake check

.github/workflows/android.yml

@@ -0,0 +1,104 @@
+name: cross compile with android-ndk
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  cross-compile-for-android:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11]
+        target-abi: [armeabi-v7a, arm64-v8a, x86, x86_64]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - run:  brew install tree file
+
+      - run:  tree --version
+      - run:  file --version
+      - run:  make --version
+
+      - uses: actions/checkout@v2
+
+      - name: build for ${{ matrix.target-abi }}
+        run: |
+          COLOR_GREEN='\033[0;32m'        # Green
+          COLOR_PURPLE='\033[0;35m'       # Purple
+          COLOR_OFF='\033[0m'             # Reset
+
+          run() {
+            printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF $COLOR_GREEN$*$COLOR_OFF"
+            eval "$*"
+          }
+
+          BUILD_MACHINE_OS_TYPE=$(uname | tr A-Z a-z)
+          BUILD_MACHINE_OS_ARCH=$(uname -m)
+
+          export ANDROID_NDK_HOME="$ANDROID_NDK_LATEST_HOME"
+
+          env | sed -n '/^ANDROID_/p'
+
+          ANDROID_NDK_VERS=$(grep "Pkg.Revision" "$ANDROID_NDK_HOME/source.properties" | cut -d " " -f3)
+          ANDROID_NDK_BASE=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$BUILD_MACHINE_OS_TYPE-$BUILD_MACHINE_OS_ARCH
+          ANDROID_NDK_BIND=$ANDROID_NDK_BASE/bin
+          SYSROOT=$ANDROID_NDK_BASE/sysroot
+
+          printf '%s = %s\n' ANDROID_NDK_HOME "$ANDROID_NDK_HOME"
+          printf '%s = %s\n' ANDROID_NDK_BIND "$ANDROID_NDK_BIND"
+          printf '%s = %s\n' ANDROID_NDK_VERS "$ANDROID_NDK_VERS"
+          printf '%s\n'
+
+          export CPPFLAGS="--sysroot $SYSROOT"
+          export   CFLAGS="--sysroot $SYSROOT -fpic"
+          export  LDFLAGS="--sysroot $SYSROOT"
+
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) CC=armv7a-linux-androideabi21-clang ;;
+            arm64-v8a)   CC=aarch64-linux-android21-clang    ;;
+            x86)         CC=i686-linux-android21-clang       ;;
+            x86_64)      CC=x86_64-linux-android21-clang     ;;
+          esac
+
+          AR=llvm-ar
+
+          export PATH="$ANDROID_NDK_BIND:$PATH"
+
+          printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF ${COLOR_GREEN}show PATH$COLOR_OFF"
+          printf '%s\n' "$PATH" | tr ':' '\n'
+          printf '%s\n'
+
+          run make clean install PREFIX=$PWD/install.d CC="$CC" CPPFLAGS="'$CPPFLAGS'" CFLAGS="'$CFLAGS'" AR="$AR" V=1
+
+      - run:  tree install.d
+
+      - run:  |
+          file install.d/lib/libdeflate.so
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, ARM aarch64,' ;;
+            x86)         file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, Intel 80386,' ;;
+            x86_64)      file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, x86-64,'      ;;
+          esac
+
+      - run:  |
+          file install.d/bin/libdeflate-gzip
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
+            x86)         file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
+            x86_64)      file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
+          esac
+
+      - run:  |
+          file install.d/bin/libdeflate-gunzip
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
+            x86)         file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
+            x86_64)      file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
+          esac

.github/workflows/ci.yml

@@ -0,0 +1,118 @@
+name: CI
+on: [pull_request]
+env:
+  CFLAGS: -Werror
+
+jobs:
+  x86_64-build-and-test:
+    name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, ubuntu-18.04]
+        compiler: [gcc, clang]
+    runs-on: ${{ matrix.os }}
+    env:
+      CC: ${{ matrix.compiler }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang llvm libz-dev valgrind
+    - run: scripts/run_tests.sh
+
+  other-arch-build-and-test:
+    name: Build and test (${{ matrix.arch }}, Debian Buster, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        arch: [armv6, armv7, aarch64, s390x, ppc64le]
+        compiler: [gcc, clang]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: uraimo/run-on-arch-action@v2.0.5
+        with:
+          arch: ${{ matrix.arch }}
+          distro: buster
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update
+            apt-get install -y build-essential clang llvm libz-dev valgrind
+          run: |
+            # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64
+            # here.  (It's probably something related to the QEMU user-mode
+            # emulation that run-on-arch-action uses.)
+            export SKIP_VALGRIND=1
+            export SKIP_ASAN=1
+
+            case ${{ matrix.arch }} in
+            s390x)
+              # On s390x, in freestanding builds the shared library links to an
+              # external symbol __clzdi2, even when -static-libgcc is used.
+              export SKIP_FREESTANDING=1
+              ;;
+            aarch64)
+              # "ldd: exited with unknown exit code (139)"
+              if [ ${{ matrix.compiler }} = clang ]; then
+                export SKIP_SHARED_LIB=1
+              fi
+              ;;
+            esac
+
+            export CC=${{ matrix.compiler }}
+            scripts/run_tests.sh
+
+  macos-build-and-test:
+    name: Build and test (macOS)
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: make all check
+
+  windows-build-and-test:
+    name: Build and test (Windows)
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - shell: bash
+      run: |
+        PATH="C:\\msys64\\mingw64\\bin:C:\\msys64\\usr\\bin:$PATH" \
+          make CC=gcc all check
+
+  run-clang-static-analyzer:
+    name: Run clang static analyzer
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang-tools
+    - name: Run clang static analyzer
+      run: make scan-build
+
+  run-shellcheck:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y shellcheck
+    - name: Run shellcheck
+      run: make shellcheck
+
+  cross-compile-for-windows:
+    name: Cross compile for Windows
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev
+    - name: 32-bit build
+      run: make CC=i686-w64-mingw32-gcc all test_programs
+    - name: 64-bit build
+      run: make CC=x86_64-w64-mingw32-gcc all test_programs

.gitignore

@@ -1,19 +1,21 @@
 *.a
+*.def
 *.dll
+*.dllobj
+*.dylib
 *.exe
 *.exp
 *.lib
 *.o
 *.obj
 *.so
-/.lib-cflags
-/.prog-cflags
+*.so.*
+/.build-config
 /programs/config.h
 /benchmark
 /checksum
 /gzip
 /gunzip
-/run_tests.log
-/test_checksums
+/test_*
 tags
 cscope*

Makefile

@@ -3,6 +3,12 @@
 #
 # Define V=1 to enable "verbose" mode, showing all executed commands.
 #
+# Define USE_SHARED_LIB to link the binaries to the shared library version of
+# libdeflate rather than to the static library version.
+#
+# Define DISABLE_SHARED to disable building and installing of shared library.
+# Defining this variable undefines USE_SHARED_LIB.
+#
 # Define DECOMPRESSION_ONLY to omit all compression code, building a
 # decompression-only library.  If doing this, you must also build a specific
 # library target such as 'libdeflate.a', as the programs will no longer compile.
@@ -11,51 +17,136 @@
 #
 # Define DISABLE_ZLIB to disable support for the zlib wrapper format.
 #
+# Define PREFIX to override the installation prefix, like './configure --prefix'
+# in autotools-based projects (default: /usr/local)
+#
+# Define BINDIR to override where to install binaries, like './configure
+# --bindir' in autotools-based projects (default: PREFIX/bin)
+#
+# Define INCDIR to override where to install headers, like './configure
+# --includedir' in autotools-based projects (default: PREFIX/include)
+#
+# Define LIBDIR to override where to install libraries, like './configure
+# --libdir' in autotools-based projects (default: PREFIX/lib)
+#
+# Define DESTDIR to override the installation destination directory
+# (default: empty string)
+#
+# Define FREESTANDING to build a freestanding library, i.e. a library that
+# doesn't link to any libc functions like malloc(), free(), and memcpy().
+# All users will need to call libdeflate_set_memory_allocator().
+#
+# You can also specify a custom CC, CFLAGS, CPPFLAGS, and/or LDFLAGS.
+#
 ##############################################################################
 
-#### Common compiler flags.
-#### Flags given here are not intended to be overridden, but you can add more
-#### by defining CFLAGS in the environment or on the 'make' command line.
+VERSION = $(shell sed -n 's/\#define LIBDEFLATE_VERSION_STRING.*"\(.*\)"/\1/p' \
+	    libdeflate.h)
+
+#### Common compiler flags.  You can add additional flags by defining CFLAGS
+#### in the environment or on the 'make' command line.
+####
+#### The default optimization flags can be overridden, e.g. via CFLAGS="-O3" or
+#### CFLAGS="-O0 -fno-omit-frame-pointer".  But this usually isn't recommended;
+#### you're unlikely to get significantly better performance even with -O3.
 
 cc-option = $(shell if $(CC) $(1) -c -x c /dev/null -o /dev/null \
 	      1>&2 2>/dev/null; then echo $(1); fi)
 
 override CFLAGS :=							\
-	$(CFLAGS) -O2 -fomit-frame-pointer -std=c99 -I. -Icommon	\
-	-Wall -Wundef							\
+	-O2 -fomit-frame-pointer -std=c99 -I. -Wall -Wundef		\
 	$(call cc-option,-Wpedantic)					\
 	$(call cc-option,-Wdeclaration-after-statement)			\
 	$(call cc-option,-Wmissing-prototypes)				\
 	$(call cc-option,-Wstrict-prototypes)				\
-	$(call cc-option,-Wvla)
+	$(call cc-option,-Wvla)						\
+	$(call cc-option,-Wimplicit-fallthrough)			\
+	$(CFLAGS)
+
+FREESTANDING :=
+ifdef FREESTANDING
+override CPPFLAGS += -DFREESTANDING
+LIB_CFLAGS += -ffreestanding -nostdlib
+endif
+
+# Don't use this option except for testing; it isn't a stable interface.
+TEST_SUPPORT__DO_NOT_USE :=
+ifdef TEST_SUPPORT__DO_NOT_USE
+override CPPFLAGS += -DTEST_SUPPORT__DO_NOT_USE
+endif
 
 ##############################################################################
 
-STATIC_LIB_SUFFIX := .a
-SHARED_LIB_SUFFIX := .so
-SHARED_LIB_CFLAGS := -fPIC
-PROG_SUFFIX       :=
-PROG_CFLAGS       :=
-HARD_LINKS        := 1
+PREFIX ?= /usr/local
+BINDIR ?= $(PREFIX)/bin
+INCDIR ?= $(PREFIX)/include
+LIBDIR ?= $(PREFIX)/lib
 
-# Compiling for Windows with MinGW?
-ifneq ($(findstring -mingw,$(shell $(CC) -dumpmachine 2>/dev/null)),)
-    STATIC_LIB_SUFFIX := .lib
-    SHARED_LIB_SUFFIX := .dll
-    SHARED_LIB_CFLAGS :=
-    PROG_SUFFIX       := .exe
-    PROG_CFLAGS       := -static -municode
-    HARD_LINKS        :=
-    override CFLAGS   := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
+SOVERSION          := 0
+
+STATIC_LIB_SUFFIX  := .a
+PROG_SUFFIX        :=
+PROG_CFLAGS        :=
+HARD_LINKS         := 1
+
+TARGET_MACHINE     := $(shell $(CC) -dumpmachine 2>/dev/null)
+
+# Compiling for Windows with MinGW or LLVM?
+ifneq ($(findstring -mingw,$(TARGET_MACHINE))$(findstring -windows-gnu,$(TARGET_MACHINE)),)
+    STATIC_LIB_SUFFIX  := static.lib
+    SHARED_LIB         := libdeflate.dll
+    SHARED_LIB_SYMLINK :=
+    SHARED_LIB_CFLAGS  :=
+    SHARED_LIB_LDFLAGS := -Wl,--out-implib,libdeflate.lib \
+                          -Wl,--output-def,libdeflate.def
+    # Only if not LLVM
+    ifeq ($(findstring -windows-gnu,$(TARGET_MACHINE)),)
+        SHARED_LIB_LDFLAGS += -Wl,--add-stdcall-alias
+    endif
+
+    PROG_SUFFIX        := .exe
+    PROG_CFLAGS        := -static -municode
+    HARD_LINKS         :=
+    override CFLAGS    := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
 
     # If AR was not already overridden, then derive it from $(CC).
-    # Note that CC may take different forms, e.g. "cc", "gcc",
-    # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1".
+    # Note that CC may take different forms, e.g. "cc", "gcc", "clang",
+    # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1", or
+    # "x86_64-w64-mingw32-clang".
     # On Windows it may also have a .exe extension.
     ifeq ($(AR),ar)
         AR := $(shell echo $(CC) | \
-                sed -E 's/g?cc(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\3/')
+                sed -E 's/(g?cc|clang)(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\4/')
     endif
+
+# Compiling for macOS?
+else ifneq ($(findstring -darwin,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.$(SOVERSION).dylib
+   SHARED_LIB_SYMLINK := libdeflate.dylib
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -install_name $(LIBDIR)/$(SHARED_LIB)
+
+# Compiling for Android?
+else ifneq ($(findstring -android,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.so
+   SHARED_LIB_SYMLINK :=
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
+
+# Compiling for Haiku?
+else ifneq ($(findstring -haiku,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.so.$(SOVERSION)
+   SHARED_LIB_SYMLINK := libdeflate.so
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
+   HARD_LINKS         :=
+
+# Compiling for Linux, FreeBSD, etc.
+else
+   SHARED_LIB         := libdeflate.so.$(SOVERSION)
+   SHARED_LIB_SYMLINK := libdeflate.so
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
 endif
 
 ##############################################################################
@@ -75,19 +166,39 @@ endif
 
 ##############################################################################
 
-COMMON_HEADERS := $(wildcard common/*.h)
+# Rebuild if a user-specified setting that affects the build changed.
+.build-config: FORCE
+	@flags=$$(							\
+		echo 'USE_SHARED_LIB=$(USE_SHARED_LIB)';		\
+		echo 'DISABLE_SHARED=$(DISABLE_SHARED)';		\
+		echo 'DECOMPRESSION_ONLY=$(DECOMPRESSION_ONLY)';	\
+		echo 'DISABLE_GZIP=$(DISABLE_GZIP)';			\
+		echo 'DISABLE_ZLIB=$(DISABLE_ZLIB)';			\
+		echo 'FREESTANDING=$(FREESTANDING)';			\
+		echo 'CC=$(CC)';					\
+		echo 'CFLAGS=$(CFLAGS)';				\
+		echo 'CPPFLAGS=$(CPPFLAGS)';				\
+		echo 'LDFLAGS=$(LDFLAGS)';				\
+	);								\
+	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then		\
+		[ -e $@ ] && echo "Rebuilding due to new settings";	\
+		echo "$$flags" > $@;					\
+	fi
+
+##############################################################################
+
+COMMON_HEADERS := $(wildcard common/*.h) libdeflate.h
 DEFAULT_TARGETS :=
 
 #### Library
 
 STATIC_LIB := libdeflate$(STATIC_LIB_SUFFIX)
-SHARED_LIB := libdeflate$(SHARED_LIB_SUFFIX)
 
 LIB_CFLAGS += $(CFLAGS) -fvisibility=hidden -D_ANSI_SOURCE
 
 LIB_HEADERS := $(wildcard lib/*.h) $(wildcard lib/*/*.h)
 
-LIB_SRC := lib/aligned_malloc.c lib/deflate_decompress.c \
+LIB_SRC := lib/deflate_decompress.c lib/utils.c \
 	   $(wildcard lib/*/cpu_features.c)
 
 DECOMPRESSION_ONLY :=
@@ -115,12 +226,13 @@ STATIC_LIB_OBJ := $(LIB_SRC:.c=.o)
 SHARED_LIB_OBJ := $(LIB_SRC:.c=.shlib.o)
 
 # Compile static library object files
-$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) $<
+$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) $<
 
 # Compile shared library object files
-$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) $(SHARED_LIB_CFLAGS) -DLIBDEFLATE_DLL $<
+$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_CFLAGS) -DLIBDEFLATE_DLL $<
 
 # Create static library
 $(STATIC_LIB):$(STATIC_LIB_OBJ)
@@ -130,17 +242,21 @@ DEFAULT_TARGETS += $(STATIC_LIB)
 
 # Create shared library
 $(SHARED_LIB):$(SHARED_LIB_OBJ)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) -shared $+
+	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_LDFLAGS) -shared $+
 
+ifdef DISABLE_SHARED
+undefine SHARED_LIB_SYMLINK
+else
 DEFAULT_TARGETS += $(SHARED_LIB)
+endif
 
-# Rebuild if CC or LIB_CFLAGS changed
-.lib-cflags: FORCE
-	@flags='$(CC):$(LIB_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding library due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
+ifdef SHARED_LIB_SYMLINK
+# Create the symlink libdeflate.so => libdeflate.so.$SOVERSION
+$(SHARED_LIB_SYMLINK):$(SHARED_LIB)
+	$(QUIET_LN) ln -sf $+ $@
+DEFAULT_TARGETS += $(SHARED_LIB_SYMLINK)
+endif
 
 ##############################################################################
 
@@ -151,38 +267,54 @@ PROG_CFLAGS += $(CFLAGS)		 \
 	       -D_FILE_OFFSET_BITS=64	 \
 	       -DHAVE_CONFIG_H
 
-PROG_COMMON_HEADERS := programs/prog_util.h programs/config.h
-PROG_COMMON_SRC     := programs/prog_util.c programs/tgetopt.c
-NONTEST_PROGRAM_SRC := programs/gzip.c
-TEST_PROGRAM_SRC    := programs/benchmark.c programs/test_checksums.c \
-			programs/checksum.c
+ALL_PROG_COMMON_HEADERS := programs/config.h \
+			   programs/prog_util.h \
+			   programs/test_util.h
+PROG_COMMON_SRC      := programs/prog_util.c \
+			programs/tgetopt.c
+NONTEST_PROG_SRC     := programs/gzip.c
+TEST_PROG_COMMON_SRC := programs/test_util.c
+TEST_PROG_SRC        := programs/benchmark.c \
+			programs/checksum.c \
+			$(filter-out $(TEST_PROG_COMMON_SRC),$(wildcard programs/test_*.c))
 
-NONTEST_PROGRAMS := $(NONTEST_PROGRAM_SRC:programs/%.c=%$(PROG_SUFFIX))
+NONTEST_PROGRAMS := $(NONTEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
 DEFAULT_TARGETS  += $(NONTEST_PROGRAMS)
-TEST_PROGRAMS    := $(TEST_PROGRAM_SRC:programs/%.c=%$(PROG_SUFFIX))
+TEST_PROGRAMS    := $(TEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
 
-PROG_COMMON_OBJ     := $(PROG_COMMON_SRC:%.c=%.o)
-NONTEST_PROGRAM_OBJ := $(NONTEST_PROGRAM_SRC:%.c=%.o)
-TEST_PROGRAM_OBJ    := $(TEST_PROGRAM_SRC:%.c=%.o)
-PROG_OBJ := $(PROG_COMMON_OBJ) $(NONTEST_PROGRAM_OBJ) $(TEST_PROGRAM_OBJ)
+PROG_COMMON_OBJ      := $(PROG_COMMON_SRC:%.c=%.o)
+NONTEST_PROG_OBJ     := $(NONTEST_PROG_SRC:%.c=%.o)
+TEST_PROG_COMMON_OBJ := $(TEST_PROG_COMMON_SRC:%.c=%.o)
+TEST_PROG_OBJ        := $(TEST_PROG_SRC:%.c=%.o)
+
+ALL_PROG_OBJ	     := $(PROG_COMMON_OBJ) $(NONTEST_PROG_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(TEST_PROG_OBJ)
 
 # Generate autodetected configuration header
-programs/config.h:programs/detect.sh .prog-cflags
+programs/config.h:scripts/detect.sh .build-config
 	$(QUIET_GEN) CC="$(CC)" CFLAGS="$(PROG_CFLAGS)" $< > $@
 
 # Compile program object files
-$(PROG_OBJ): %.o: %.c $(PROG_COMMON_HEADERS) $(COMMON_HEADERS) .prog-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(PROG_CFLAGS) $<
+$(ALL_PROG_OBJ): %.o: %.c $(ALL_PROG_COMMON_HEADERS) $(COMMON_HEADERS) \
+			.build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(PROG_CFLAGS) $<
 
 # Link the programs.
 #
 # Note: the test programs are not compiled by default.  One reason is that the
 # test programs must be linked with zlib for doing comparisons.
 
-$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
+ifdef USE_SHARED_LIB
+LIB := $(SHARED_LIB)
+else
+LIB := $(STATIC_LIB)
+endif
+
+$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(LIB)
 	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+
 
-$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
+$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(LIB)
 	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
 
 ifdef HARD_LINKS
@@ -197,20 +329,73 @@ endif
 
 DEFAULT_TARGETS += gunzip$(PROG_SUFFIX)
 
-# Rebuild if CC or PROG_CFLAGS changed
-.prog-cflags: FORCE
-	@flags='$(CC):$(PROG_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding programs due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
-
 ##############################################################################
 
 all:$(DEFAULT_TARGETS)
 
+# We don't generate the pkg-config file until install time, as needed
+# definitions might not exist until then.
+PKGCONFBASE := libdeflate.pc.in
+
+# Install the files.  Note: not all versions of the 'install' program have the
+# '-D' and '-t' options, so don't use them; use portable commands only.
+install:all $(PKGCONFBASE)
+	install -d $(DESTDIR)$(LIBDIR)/pkgconfig $(DESTDIR)$(INCDIR)	\
+		$(DESTDIR)$(BINDIR)
+	install -m644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)
+	if [ -z "$(DISABLE_SHARED)" ]; then				\
+		install -m755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR);	\
+	fi
+	sed -e "s|@PREFIX@|$(PREFIX)|"					\
+	    -e "s|@LIBDIR@|$(LIBDIR)|"					\
+	    -e "s|@INCDIR@|$(INCDIR)|"					\
+	    -e "s|@VERSION@|$(VERSION)|"				\
+	    $(PKGCONFBASE) > $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
+	chmod 644 $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
+	install -m644 libdeflate.h $(DESTDIR)$(INCDIR)
+	install -m755 gzip$(PROG_SUFFIX) \
+		$(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)
+	if [ -n "$(HARD_LINKS)" ]; then					\
+		ln -f $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)	\
+		      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
+	else								\
+		ln -sf libdeflate-gzip$(PROG_SUFFIX)			\
+		       $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
+	fi
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		ln -sf $(SHARED_LIB)					\
+		       $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
+uninstall:
+	rm -f $(DESTDIR)$(LIBDIR)/$(STATIC_LIB)				\
+	      $(DESTDIR)$(LIBDIR)/$(SHARED_LIB)				\
+	      $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc		\
+	      $(DESTDIR)$(INCDIR)/libdeflate.h				\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)		\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX)
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
 test_programs:$(TEST_PROGRAMS)
 
+# A minimal 'make check' target.  This only runs some quick tests;
+# use scripts/run_tests.sh if you want to run the full tests.
+check:test_programs
+	LD_LIBRARY_PATH=. ./benchmark$(PROG_SUFFIX) < ./benchmark$(PROG_SUFFIX)
+	for prog in test_*; do		\
+		LD_LIBRARY_PATH=. ./$$prog || exit 1;	\
+	done
+
+# Run the clang static analyzer.
+scan-build:
+	scan-build --status-bugs make all test_programs
+
+# Run shellcheck on all shell scripts.
+shellcheck:
+	shellcheck scripts/*.sh
+
 help:
 	@echo "Available targets:"
 	@echo "------------------"
@@ -219,20 +404,21 @@ help:
 	done
 
 clean:
-	rm -f *.a *.dll *.exe *.exp *.so \
+	rm -f *.a *.dll *.exe *.exp *.dylib *.so \
 		lib/*.o lib/*/*.o \
 		lib/*.obj lib/*/*.obj \
 		lib/*.dllobj lib/*/*.dllobj \
 		programs/*.o programs/*.obj \
 		$(DEFAULT_TARGETS) $(TEST_PROGRAMS) programs/config.h \
-		libdeflate.lib libdeflatestatic.lib \
-		.lib-cflags .prog-cflags
+		libdeflate.lib libdeflate.def libdeflatestatic.lib \
+		.build-config
 
 realclean: clean
-	rm -f tags cscope* run_tests.log
+	rm -f tags cscope*
 
 FORCE:
 
-.PHONY: all test_programs help clean realclean
+.PHONY: all install uninstall test_programs check scan-build shellcheck help \
+	clean realclean
 
 .DEFAULT_GOAL = all

Makefile.msc

@@ -10,7 +10,7 @@
 CC = cl
 LD = link
 AR = lib
-CFLAGS = /MD /O2 -I. -Icommon
+CFLAGS = /MD /O2 -I.
 LDFLAGS =
 
 STATIC_LIB = libdeflatestatic.lib
@@ -18,13 +18,13 @@ SHARED_LIB = libdeflate.dll
 IMPORT_LIB = libdeflate.lib
 
 STATIC_LIB_OBJ =			\
-	  lib/aligned_malloc.obj	\
 	  lib/adler32.obj		\
 	  lib/crc32.obj			\
 	  lib/deflate_compress.obj	\
 	  lib/deflate_decompress.obj	\
 	  lib/gzip_compress.obj		\
 	  lib/gzip_decompress.obj	\
+	  lib/utils.obj			\
 	  lib/x86/cpu_features.obj	\
 	  lib/zlib_compress.obj		\
 	  lib/zlib_decompress.obj
@@ -61,5 +61,5 @@ gunzip.exe:gzip.exe
 
 clean:
 	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
-		lib\*.obj lib\*\*.obj lib\*.dllobj lib\*\*.dllobj \
+		lib\*.obj lib\x86\*.obj lib\*.dllobj lib\x86\*.dllobj \
 		programs\*.obj 2>nul

NEWS

@@ -1,84 +0,0 @@
-Version 1.0:
-	Added support for multi-member gzip files.
-
-	Moved architecture-specific code into subdirectories.  If you
-	aren't using the provided Makefile to build libdeflate, you now
-	need to compile lib/*.c and lib/*/*.c instead of just lib/*.c.
-
-	Added an ARM PMULL implementation of CRC-32, which speeds up
-	gzip compression and decompression on 32-bit and 64-bit ARM
-	processors that have the Cryptography Extensions.
-
-	Improved detection of CPU features, resulting in accelerated
-	functions being used in more cases.  This includes:
-
-	   - Detect CPU features on 32-bit x86, not just 64-bit as was
-	     done previously.
-
-	   - Detect CPU features on ARM, both 32 and 64-bit.
-	     (Limited to Linux only currently.)
-
-Version 0.8:
-	Build fixes for certain platforms and compilers.
-
-	libdeflate now produces the same output on all CPU architectures.
-
-	Improved documentation for building libdeflate on Windows.
-
-Version 0.7:
-	Fixed a very rare bug that caused data to be compressed incorrectly.
-	The bug affected compression levels 7 and below since libdeflate v0.2.
-	Although there have been no user reports of the bug, and I believe it
-	would have been highly unlikely to encounter on realistic data, it could
-	occur on data specially crafted to reproduce it.
-
-	Fixed a compilation error when building with clang 3.7.
-
-Version 0.6:
-	Various improvements to the gzip program's behavior.
-
-	Faster CRC-32 on AVX-capable processors.
-
-	Other minor changes.
-
-Version 0.5:
-	The CRC-32 checksum algorithm has been optimized with carryless
-	multiplication instructions for x86_64 (PCLMUL).  This speeds up gzip
-	compression and decompression.
-
-	Build fixes for certain platforms and compilers.
-
-	Added more test programs and scripts.
-
-	libdeflate is now entirely MIT-licensed.
-
-Version 0.4:
-	The Adler-32 checksum algorithm has been optimized with vector
-	instructions for x86_64 (SSE2 and AVX2) and ARM (NEON).  This speeds up
-	zlib compression and decompression.
-
-	To avoid naming collisions, functions and definitions in libdeflate's
-	API have been renamed to be prefixed with "libdeflate_" or
-	"LIBDEFLATE_".  Programs using the old API will need to be updated.
-
-	Various bug fixes and other improvements.
-
-Version 0.3:
-	Some bug fixes and other minor changes.
-
-Version 0.2:
-	Implemented a new block splitting algorithm which typically improves the
-	compression ratio slightly at all compression levels.
-
-	The compressor now outputs each block using the cheapest type (dynamic
-	Huffman, static Huffman, or uncompressed).
-
-	The gzip program has received an overhaul and now behaves more like the
-	standard version.
-
-	Build system updates, including: some build options were changed and
-	some build options were removed, and the default 'make' target now
-	includes the gzip program as well as the library.
-
-Version 0.1:
-	Initial official release.

NEWS.md

@@ -0,0 +1,279 @@
+# libdeflate release notes
+
+## Version 1.10
+
+* Added an additional check to the decompressor to make it quickly detect
+  certain bad inputs and not try to generate an unbounded amount of output.
+
+  Note: this was only a problem when decompressing with an unknown output size,
+  which isn't the recommended use case of libdeflate.  However,
+  `libdeflate-gunzip` has to do this, and it would run out of memory as it would
+  keep trying to allocate a larger output buffer.
+
+* Fixed a build error on Solaris.
+
+* Cleaned up a few things in the compression code.
+
+## Version 1.9
+
+* Made many improvements to the compression algorithms, and rebalanced the
+  compression levels:
+
+  * Heuristics were implemented which significantly improve the compression
+    ratio on data where short matches aren't useful, such as DNA sequencing
+    data.  This applies to all compression levels, but primarily to levels 1-9.
+
+  * Level 1 was made much faster, though it often compresses slightly worse than
+    before (but still better than zlib).
+
+  * Levels 8-9 were also made faster, though they often compress slightly worse
+    than before (but still better than zlib).  On some data, levels 8-9 are much
+    faster and compress much better than before; this change addressed an issue
+    where levels 8-9 did poorly on certain files.  The algorithm used by levels
+    8-9 is now more similar to that of levels 6-7 than to that of levels 10-12.
+
+  * Levels 2-3, 7, and 10-12 were strengthened slightly.
+
+  * Levels 4-6 were also strengthened slightly, but some of this improvement was
+    traded off to speed them up slightly as well.
+
+  * Levels 1-9 had their per-compressor memory usage greatly reduced.
+
+  As always, compression ratios will vary depending on the input data, and
+  compression speeds will vary depending on the input data and target platform.
+
+* `make install` will now install a pkg-config file for libdeflate.
+
+* The Makefile now supports the `DISABLE_SHARED` parameter to disable building
+  the shared library.
+
+* Improved the Android build support in the Makefile.
+
+## Version 1.8
+
+* Added `-t` (test) option to `libdeflate-gunzip`.
+
+* Unaligned access optimizations are now enabled on WebAssembly builds.
+
+* Fixed a build error when building with the Intel C Compiler (ICC).
+
+* Fixed a build error when building with uClibc.
+
+* libdeflate's CI system has switched from Travis CI to GitHub Actions.
+
+* Made some improvements to test scripts.
+
+## Version 1.7
+
+* Added support for compression level 0, "no compression".
+
+* Added an ARM CRC32 instruction accelerated implementation of CRC32.
+
+* Added support for linking the programs to the shared library version of
+  libdeflate rather than to the static library version.
+
+* Made the compression level affect the minimum input size at which compression
+  is attempted.
+
+* Fixed undefined behavior in x86 Adler32 implementation.  (No miscompilations
+  were observed in practice.)
+
+* Fixed undefined behavior in x86 CPU feature code.  (No miscompilations were
+  observed in practice.)
+
+* Fixed installing shared lib symlink on macOS.
+
+* Documented third-party bindings.
+
+* Made a lot of improvements to the testing scripts and the CI configuration
+  file.
+
+* Lots of other small improvements and cleanups.
+
+## Version 1.6
+
+* Prevented gcc 10 from miscompiling libdeflate (workaround for
+  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+
+* Removed workaround for gcc 5 and earlier producing slow code on ARM32.  If
+  this affects you, please upgrade your compiler.
+
+* New API function: `libdeflate_zlib_decompress_ex()`.  It provides the actual
+  size of the stream that was decompressed, like the gzip and DEFLATE
+  equivalents.
+
+* `libdeflate_zlib_decompress()` now accepts trailing bytes after the end of the
+  stream, like the gzip and DEFLATE equivalents.
+
+* Added support for custom memory allocators.  (New API function:
+  `libdeflate_set_memory_allocator()`)
+
+* Added support for building the library in freestanding mode.
+
+* Building libdeflate no longer requires `CPPFLAGS=-Icommon`.
+
+## Version 1.5
+
+* Fixed up stdcall support on 32-bit Windows: the functions are now exported
+  using both suffixed and non-suffixed names, and fixed `libdeflate.h` to be
+  MSVC-compatible again.
+
+## Version 1.4
+
+* The 32-bit Windows build of libdeflate now uses the "stdcall" calling
+  convention instead of "cdecl".  If you're calling `libdeflate.dll` directly
+  from C or C++, you'll need to recompile your code.  If you're calling it from
+  another language, or calling it indirectly using `LoadLibrary()`, you'll need
+  to update your code to use the stdcall calling convention.
+
+* The Makefile now supports building libdeflate as a shared
+  library (`.dylib`) on macOS.
+
+* Fixed a bug where support for certain optimizations and optional features
+  (file access hints and more precise timestamps) was incorrectly omitted when
+  libdeflate was compiled with `-Werror`.
+
+* Added `make check` target to the Makefile.
+
+* Added CI configuration files.
+
+## Version 1.3
+
+* `make install` now supports customizing the directories into which binaries,
+  headers, and libraries are installed.
+
+* `make install` now installs into `/usr/local` by default.  To change it, use
+  e.g. `make install PREFIX=/usr`.
+
+* `make install` now works on more platforms.
+
+* The Makefile now supports overriding the optimization flags.
+
+* The compression functions now correctly handle an output data buffer >= 4 GiB
+  in size, and `gzip` and `gunzip` now correctly handle multi-gigabyte files (if
+  enough memory is available).
+
+## Version 1.2
+
+* Slight improvements to decompression speed.
+
+* Added an AVX-512BW implementation of Adler-32.
+
+* The Makefile now supports a user-specified installation `PREFIX`.
+
+* Fixed build error with some Visual Studio versions.
+
+## Version 1.1
+
+* Fixed crash in CRC-32 code when the prebuilt libdeflate for 32-bit Windows was
+  called by a program built with Visual Studio.
+
+* Improved the worst-case decompression speed of malicious data.
+
+* Fixed build error when compiling for an ARM processor without hardware
+  floating point support.
+
+* Improved performance on the PowerPC64 architecture.
+
+* Added soname to `libdeflate.so`, to make packaging easier.
+
+* Added `make install` target to the Makefile.
+
+* The Makefile now supports user-specified `CPPFLAGS`.
+
+* The Windows binary releases now include the import library for
+  `libdeflate.dll`.  `libdeflate.lib` is now the import library, and
+  `libdeflatestatic.lib` is the static library.
+
+## Version 1.0
+
+* Added support for multi-member gzip files.
+
+* Moved architecture-specific code into subdirectories.  If you aren't using the
+  provided Makefile to build libdeflate, you now need to compile `lib/*.c` and
+  `lib/*/*.c` instead of just `lib/*.c`.
+
+* Added an ARM PMULL implementation of CRC-32, which speeds up gzip compression
+  and decompression on 32-bit and 64-bit ARM processors that have the
+  Cryptography Extensions.
+
+* Improved detection of CPU features, resulting in accelerated functions being
+  used in more cases.  This includes:
+
+  * Detect CPU features on 32-bit x86, not just 64-bit as was done previously.
+
+  * Detect CPU features on ARM, both 32 and 64-bit.  (Limited to Linux only
+    currently.)
+
+## Version 0.8
+
+* Build fixes for certain platforms and compilers.
+
+* libdeflate now produces the same output on all CPU architectures.
+
+* Improved documentation for building libdeflate on Windows.
+
+## Version 0.7
+
+* Fixed a very rare bug that caused data to be compressed incorrectly.  The bug
+  affected compression levels 7 and below since libdeflate v0.2.  Although there
+  have been no user reports of the bug, and I believe it would have been highly
+  unlikely to encounter on realistic data, it could occur on data specially
+  crafted to reproduce it.
+
+* Fixed a compilation error when building with clang 3.7.
+
+## Version 0.6
+
+* Various improvements to the gzip program's behavior.
+
+* Faster CRC-32 on AVX-capable processors.
+
+* Other minor changes.
+
+## Version 0.5
+
+* The CRC-32 checksum algorithm has been optimized with carryless multiplication
+  instructions for `x86_64` (PCLMUL).  This speeds up gzip compression and
+  decompression.
+
+* Build fixes for certain platforms and compilers.
+
+* Added more test programs and scripts.
+
+* libdeflate is now entirely MIT-licensed.
+
+## Version 0.4
+
+* The Adler-32 checksum algorithm has been optimized with vector instructions
+  for `x86_64` (SSE2 and AVX2) and ARM (NEON).  This speeds up zlib compression
+  and decompression.
+
+* To avoid naming collisions, functions and definitions in libdeflate's API have
+  been renamed to be prefixed with `libdeflate_` or `LIBDEFLATE_`.  Programs
+  using the old API will need to be updated.
+
+* Various bug fixes and other improvements.
+
+## Version 0.3
+
+* Some bug fixes and other minor changes.
+
+## Version 0.2
+
+* Implemented a new block splitting algorithm which typically improves the
+  compression ratio slightly at all compression levels.
+
+* The compressor now outputs each block using the cheapest type (dynamic
+  Huffman, static Huffman, or uncompressed).
+
+* The gzip program has received an overhaul and now behaves more like the
+  standard version.
+
+* Build system updates, including: some build options were changed and some
+  build options were removed, and the default 'make' target now includes the
+  gzip program as well as the library.
+
+## Version 0.1
+
+* Initial official release.

README.md

@@ -22,18 +22,48 @@ use this library are also provided:
   yet support very large files
 * benchmark, a program for benchmarking in-memory compression and decompression
 
+For the release notes, see the [NEWS file](NEWS.md).
+
+## Table of Contents
+
+- [Building](#building)
+  - [For UNIX](#for-unix)
+  - [For macOS](#for-macos)
+  - [For Windows](#for-windows)
+    - [Using Cygwin](#using-cygwin)
+    - [Using MSYS2](#using-msys2)
+- [API](#api)
+- [Bindings for other programming languages](#bindings-for-other-programming-languages)
+- [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip)
+- [Compression levels](#compression-levels)
+- [Motivation](#motivation)
+- [License](#license)
+
+
 # Building
 
 ## For UNIX
 
-Just run `make`.  You need GNU Make and either GCC or Clang.  GCC is recommended
-because it builds slightly faster binaries.  There is no `make install` yet;
-just copy the file(s) to where you want.
+Just run `make`, then (if desired) `make install`.  You need GNU Make and either
+GCC or Clang.  GCC is recommended because it builds slightly faster binaries.
 
-By default, all targets are built, including the library and programs, with the
-exception of the `benchmark` program.  `make help` shows the available targets.
-There are also several options which can be set on the `make` command line.  See
-the Makefile for details.
+By default, the following targets are built: the static library `libdeflate.a`,
+the shared library `libdeflate.so`, the `gzip` program, and the `gunzip` program
+(which is actually just a hard link to `gzip`).  Benchmarking and test programs
+such as `benchmark` are not built by default.  You can run `make help` to
+display the available build targets.
+
+There are also many options which can be set on the `make` command line, e.g. to
+omit library features or to customize the directories into which `make install`
+installs files.  See the Makefile for details.
+
+## For macOS
+
+Prebuilt macOS binaries can be installed with [Homebrew](https://brew.sh):
+
+    brew install libdeflate
+
+But if you need to build the binaries yourself, see the section for UNIX above.
 
 ## For Windows
 
@@ -152,6 +182,30 @@ This enables you to allocate an output buffer of the correct size without
 guessing.  However, libdeflate's decompression routines do optionally provide
 the actual number of output bytes in case you need it.
 
+Windows developers: note that the calling convention of libdeflate.dll is
+"stdcall" -- the same as the Win32 API.  If you call into libdeflate.dll using a
+non-C/C++ language, or dynamically using LoadLibrary(), make sure to use the
+stdcall convention.  Using the wrong convention may crash your application.
+(Note: older versions of libdeflate used the "cdecl" convention instead.)
+
+# Bindings for other programming languages
+
+The libdeflate project itself only provides a C library.  If you need to use
+libdeflate from a programming language other than C or C++, consider using the
+following bindings:
+
+* C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET)
+* Go: [go-libdeflate](https://github.com/4kills/go-libdeflate)
+* Java: [libdeflate-java](https://github.com/astei/libdeflate-java)
+* Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl)
+* Python: [deflate](https://github.com/dcwatson/deflate)
+* Ruby: [libdeflate-ruby](https://github.com/kaorimatz/libdeflate-ruby)
+* Rust: [libdeflater](https://github.com/adamkewley/libdeflater)
+
+Note: these are third-party projects which haven't necessarily been vetted by
+the authors of libdeflate.  Please direct all questions, bugs, and improvements
+for these bindings to their authors.
+
 # DEFLATE vs. zlib vs. gzip
 
 The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib
@@ -224,21 +278,8 @@ processors.  libdeflate addresses this by providing an optimized DEFLATE
 implementation which can be used for benchmarking purposes.  And, of course,
 real applications can use it as well.
 
-That being said, I have also started [a separate
-project](https://github.com/ebiggers/xpack) for an experimental, more modern
-compression format.
-
 # License
 
 libdeflate is [MIT-licensed](COPYING).
 
-Additional notes (informational only):
-
-- I am not aware of any patents covering libdeflate.
-
-- Old versions of libdeflate were public domain; I only started copyrighting
-  changes in newer versions.  Portions of the source code that have not been
-  changed since being released in a public domain version can theoretically
-  still be used as public domain if you want to.  But for practical purposes, it
-  probably would be easier to just take the MIT license option, which is nearly
-  the same anyway.
+I am not aware of any patents or patent applications relevant to libdeflate.

common/common_defs.h

@@ -47,9 +47,7 @@
 #endif
 
 /* Fixed-width integer types */
-#ifndef PRIu32
-#  include <inttypes.h>
-#endif
+#include <stdint.h>
 typedef uint8_t u8;
 typedef uint16_t u16;
 typedef uint32_t u32;
@@ -128,6 +126,9 @@ typedef size_t machine_word_t;
 #ifndef COMPILER_SUPPORTS_AVX_TARGET
 #  define COMPILER_SUPPORTS_AVX_TARGET 0
 #endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET 0
+#endif
 
 /*
  * Which targets are supported with the 'target' function attribute and have
@@ -142,12 +143,18 @@ typedef size_t machine_word_t;
 #ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS
 #  define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0
 #endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0
+#endif
 #ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS
 #  define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0
 #endif
 #ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS
 #  define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0
 #endif
+#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0
+#endif
 
 /* _aligned_attribute(n) - declare that the annotated variable, or variables of
  * the annotated type, are to be aligned on n-byte boundaries */
@@ -240,39 +247,6 @@ static forceinline u64 bswap64(u64 n)
 #  define UNALIGNED_ACCESS_IS_FAST 0
 #endif
 
-/*
- * DEFINE_UNALIGNED_TYPE(type) - a macro that, given an integer type 'type',
- * defines load_type_unaligned(addr) and store_type_unaligned(v, addr) functions
- * which load and store variables of type 'type' from/to unaligned memory
- * addresses.  If not defined, a fallback is used.
- */
-#ifndef DEFINE_UNALIGNED_TYPE
-
-/*
- * Although memcpy() may seem inefficient, it *usually* gets optimized
- * appropriately by modern compilers.  It's portable and may be the best we can
- * do for a fallback...
- */
-#include <string.h>
-
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	type v;							\
-	memcpy(&v, p, sizeof(v));				\
-	return v;						\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	memcpy(p, &v, sizeof(v));				\
-}
-
-#endif /* !DEFINE_UNALIGNED_TYPE */
-
 /* ========================================================================== */
 /*                             Bit scan functions                             */
 /* ========================================================================== */

common/compiler_gcc.h

@@ -71,7 +71,10 @@
 	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di))
 
 #    define COMPILER_SUPPORTS_AVX2_TARGET	\
-	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pmaddwd256))
+	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256))
+
+#    define COMPILER_SUPPORTS_AVX512BW_TARGET	\
+	(GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512))
 
 	/*
 	 * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics
@@ -85,27 +88,98 @@
 		COMPILER_SUPPORTS_PCLMUL_TARGET
 #      define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS	\
 		COMPILER_SUPPORTS_AVX2_TARGET
+#      define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_AVX512BW_TARGET
 #    endif
+
 #  elif defined(__arm__) || defined(__aarch64__)
-	/*
-	 * Prior to gcc 6.1 (r230411 for arm, r226563 for aarch64), NEON
-	 * and crypto intrinsics not available in the main target could not be
-	 * used in 'target' attribute functions.
-	 *
-	 * clang as of 5.0.1 still doesn't allow it.  But, it does seem to allow
-	 * the pmull intrinsics if only __ARM_NEON is enabled.
-	 */
-#    define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS	GCC_PREREQ(6, 1)
-#    ifdef __ARM_NEON
-#      define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS	\
-		(GCC_PREREQ(6, 1) || __has_builtin(__builtin_neon_vmull_p64))
-#    else
-#      define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS	\
-		(GCC_PREREQ(6, 1))
+
+    /*
+     * Determine whether NEON and crypto intrinsics are supported.
+     *
+     * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither
+     * was available unless enabled in the main target.
+     *
+     * But even after that, to include <arm_neon.h> (which contains both the
+     * basic NEON intrinsics and the crypto intrinsics) the main target still
+     * needs to have:
+     *   - gcc: hardware floating point support
+     *   - clang: NEON support (but not necessarily crypto support)
+     */
+#    if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \
+        (defined(__clang__) && defined(__ARM_NEON))
+#      define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1
+       /*
+        * The crypto intrinsics are broken on arm32 with clang, even when using
+        * -mfpu=crypto-neon-fp-armv8, because clang's <arm_neon.h> puts them
+        * behind __aarch64__.  Undefine __ARM_FEATURE_CRYPTO in that case...
+        */
+#      if defined(__clang__) && defined(__arm__)
+#        undef __ARM_FEATURE_CRYPTO
+#      elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__)
+#        define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1
+#      endif
 #    endif
-#  endif
+
+     /*
+      * Determine whether ARM CRC32 intrinsics are supported.
+      *
+      * This support has been affected by several gcc bugs, which we must avoid
+      * by only allowing gcc versions that have the corresponding fixes.  First,
+      * gcc commit 943766d37ae4 ("[arm] Fix use of CRC32 intrinsics with Armv8-a
+      * and hard-float"), i.e. gcc 8.4+, 9.3+, 10.1+, or 11+, is needed.
+      * Second, gcc commit c1cdabe3aab8 ("arm: reorder assembler architecture
+      * directives [PR101723]"), i.e. gcc 9.5+, 10.4+, 11.3+, or 12+, is needed
+      * when binutils is 2.34 or later, due to
+      * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104439.  We use the second
+      * set of prerequisites, as they are stricter and we have no way to detect
+      * the binutils version in C source without requiring a configure script.
+      *
+      * Yet another gcc bug makes arm_acle.h sometimes not define the crc
+      * functions even when the corresponding builtins are available.  However,
+      * we work around this later when including arm_acle.h.
+      *
+      * Things are a bit easier with clang -- we can just check whether the
+      * crc builtins are available.  However, clang's arm_acle.h is broken in
+      * the same way as gcc's, which we work around later in the same way.
+      */
+#    if GCC_PREREQ(11, 3) || \
+        (GCC_PREREQ(10, 4) && !GCC_PREREQ(11, 0)) || \
+        (GCC_PREREQ(9, 5) && !GCC_PREREQ(10, 0)) || \
+        (defined(__clang__) && __has_builtin(__builtin_arm_crc32b))
+#      define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1
+#    endif
+
+#  endif /* __arm__ || __aarch64__ */
+
 #endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */
 
+/*
+ * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed
+ * integers (e.g. __v4si), not vectors of unsigned integers (e.g.  __v4su).  But
+ * we need the unsigned ones in order to avoid signed integer overflow, which is
+ * undefined behavior.  Add the missing definitions for the unsigned ones if
+ * needed.
+ */
+#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \
+    (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) || \
+    defined(__INTEL_COMPILER)
+typedef unsigned long long  __v2du __attribute__((__vector_size__(16)));
+typedef unsigned int        __v4su __attribute__((__vector_size__(16)));
+typedef unsigned short      __v8hu __attribute__((__vector_size__(16)));
+typedef unsigned char      __v16qu __attribute__((__vector_size__(16)));
+typedef unsigned long long  __v4du __attribute__((__vector_size__(32)));
+typedef unsigned int        __v8su __attribute__((__vector_size__(32)));
+typedef unsigned short     __v16hu __attribute__((__vector_size__(32)));
+typedef unsigned char      __v32qu __attribute__((__vector_size__(32)));
+#endif
+
+#ifdef __INTEL_COMPILER
+typedef int   __v16si __attribute__((__vector_size__(64)));
+typedef short __v32hi __attribute__((__vector_size__(64)));
+typedef char  __v64qi __attribute__((__vector_size__(64)));
+#endif
+
 /* Newer gcc supports __BYTE_ORDER__.  Older gcc doesn't. */
 #ifdef __BYTE_ORDER__
 #  define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
@@ -123,30 +197,39 @@
 #  define bswap64	__builtin_bswap64
 #endif
 
-#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED)
+#if defined(__x86_64__) || defined(__i386__) || \
+    defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
+    /*
+     * For all compilation purposes, WebAssembly behaves like any other CPU
+     * instruction set. Even though WebAssembly engine might be running on top
+     * of different actual CPU architectures, the WebAssembly spec itself
+     * permits unaligned access and it will be fast on most of those platforms,
+     * and simulated at the engine level on others, so it's worth treating it
+     * as a CPU architecture with fast unaligned access.
+    */ defined(__wasm__)
 #  define UNALIGNED_ACCESS_IS_FAST 1
 #endif
 
-/* With gcc, we can access unaligned memory through 'packed' structures. */
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-struct type##unaligned {					\
-	type v;							\
-} __attribute__((packed));					\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	return ((const struct type##unaligned *)p)->v;		\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	((struct type##unaligned *)p)->v = v;			\
-}
-
 #define bsr32(n)	(31 - __builtin_clz(n))
 #define bsr64(n)	(63 - __builtin_clzll(n))
 #define bsf32(n)	__builtin_ctz(n)
 #define bsf64(n)	__builtin_ctzll(n)
+
+#if defined(__arm__) && \
+	(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
+static forceinline unsigned int
+rbit32(unsigned int v)
+{
+	__asm__("rbit %0, %1\n" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#elif defined(__aarch64__)
+static forceinline unsigned int
+rbit32(unsigned int v)
+{
+	__asm__("rbit %w0, %w1\n" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#endif /* __aarch64__ */

common/compiler_msc.h

@@ -2,6 +2,9 @@
  * compiler_msc.h - definitions for the Microsoft C Compiler
  */
 
+#include <stdint.h>
+#include <stdlib.h> /* for _byteswap_*() */
+
 #define LIBEXPORT	__declspec(dllexport)
 
 /*
@@ -22,24 +25,6 @@ typedef long long ssize_t;
 typedef int ssize_t;
 #endif
 
-/*
- * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
- * inttypes.h.  Work around this by defining the PRI* macros ourselves.
- */
-#include <stdint.h>
-#define PRIu8  "hhu"
-#define PRIu16 "hu"
-#define PRIu32 "u"
-#define PRIu64 "llu"
-#define PRIi8  "hhi"
-#define PRIi16 "hi"
-#define PRIi32 "i"
-#define PRIi64 "lli"
-#define PRIx8  "hhx"
-#define PRIx16 "hx"
-#define PRIx32 "x"
-#define PRIx64 "llx"
-
 /* Assume a little endian architecture with fast unaligned access */
 #define CPU_IS_LITTLE_ENDIAN()		1
 #define UNALIGNED_ACCESS_IS_FAST	1

lib/adler32.c

@@ -121,7 +121,7 @@ static u32 dispatch(u32 adler, const u8 *buffer, size_t size)
 #  define adler32_impl DEFAULT_IMPL /* only one implementation, use it */
 #endif
 
-LIBDEFLATEAPI u32
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
 libdeflate_adler32(u32 adler, const void *buffer, size_t size)
 {
 	if (buffer == NULL) /* return initial value */

lib/aligned_malloc.h

@@ -1,13 +0,0 @@
-/*
- * aligned_malloc.c - aligned memory allocation
- */
-
-#ifndef LIB_ALIGNED_MALLOC_H
-#define LIB_ALIGNED_MALLOC_H
-
-#include "lib_common.h"
-
-extern void *aligned_malloc(size_t alignment, size_t size);
-extern void aligned_free(void *ptr);
-
-#endif /* LIB_ALIGNED_MALLOC_H */

lib/arm/adler32_impl.h

@@ -25,6 +25,9 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_ARM_ADLER32_IMPL_H
+#define LIB_ARM_ADLER32_IMPL_H
+
 #include "cpu_features.h"
 
 /* NEON implementation */
@@ -118,3 +121,5 @@ arch_select_adler32_func(void)
 	return NULL;
 }
 #endif /* DISPATCH */
+
+#endif /* LIB_ARM_ADLER32_IMPL_H */

lib/arm/cpu_features.c

@@ -26,19 +26,25 @@
  */
 
 /*
- * ARM processors don't have a standard way for unprivileged programs to detect
- * processor features.  But, on Linux we can read the AT_HWCAP and AT_HWCAP2
- * values from /proc/self/auxv.
- *
- * Ideally we'd use the C library function getauxval(), but it's not guaranteed
- * to be available: it was only added to glibc in 2.16, and in Android it was
- * added to API level 18 for ARM and level 21 for AArch64.
+ * ARM CPUs don't have a standard way for unprivileged programs to detect CPU
+ * features.  But an OS-specific way can be used when available.
  */
 
+#include "../cpu_features_common.h" /* must be included first */
 #include "cpu_features.h"
 
 #if ARM_CPU_FEATURES_ENABLED
 
+#ifdef __linux__
+/*
+ * On Linux, arm32 and arm64 CPU features can be detected by reading the
+ * AT_HWCAP and AT_HWCAP2 values from /proc/self/auxv.
+ *
+ * Ideally we'd use the C library function getauxval(), but it's not guaranteed
+ * to be available: it was only added to glibc in 2.16, and in Android it was
+ * added to API level 18 for arm32 and level 21 for arm64.
+ */
+
 #include <errno.h>
 #include <fcntl.h>
 #include <string.h>
@@ -47,8 +53,6 @@
 #define AT_HWCAP	16
 #define AT_HWCAP2	26
 
-volatile u32 _cpu_features = 0;
-
 static void scan_auxv(unsigned long *hwcap, unsigned long *hwcap2)
 {
 	int fd;
@@ -91,7 +95,7 @@ out:
 	close(fd);
 }
 
-void setup_cpu_features(void)
+static u32 get_arm_cpu_features(void)
 {
 	u32 features = 0;
 	unsigned long hwcap = 0;
@@ -105,13 +109,70 @@ void setup_cpu_features(void)
 		features |= ARM_CPU_FEATURE_NEON;
 	if (hwcap2 & (1 << 1))	/* HWCAP2_PMULL */
 		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap2 & (1 << 4))	/* HWCAP2_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
 #else
 	STATIC_ASSERT(sizeof(long) == 8);
 	if (hwcap & (1 << 1))	/* HWCAP_ASIMD */
 		features |= ARM_CPU_FEATURE_NEON;
 	if (hwcap & (1 << 4))	/* HWCAP_PMULL */
 		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap & (1 << 7))	/* HWCAP_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
 #endif
+	return features;
+}
+
+#elif defined(__APPLE__)
+/* On Apple platforms, arm64 CPU features can be detected via sysctlbyname(). */
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+
+static const struct {
+	const char *name;
+	u32 feature;
+} feature_sysctls[] = {
+	{ "hw.optional.neon",		ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.AdvSIMD",	ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.arm.FEAT_PMULL",	ARM_CPU_FEATURE_PMULL },
+	{ "hw.optional.armv8_crc32",	ARM_CPU_FEATURE_CRC32 },
+};
+
+static u32 get_arm_cpu_features(void)
+{
+	u32 features = 0;
+	size_t i;
+
+	for (i = 0; i < ARRAY_LEN(feature_sysctls); i++) {
+		const char *name = feature_sysctls[i].name;
+		u32 val = 0;
+		size_t valsize = sizeof(val);
+
+		if (sysctlbyname(name, &val, &valsize, NULL, 0) == 0 &&
+		    valsize == sizeof(val) && val == 1)
+			features |= feature_sysctls[i].feature;
+	}
+	return features;
+}
+#else
+#error "unhandled case"
+#endif
+
+static const struct cpu_feature arm_cpu_feature_table[] = {
+	{ARM_CPU_FEATURE_NEON,		"neon"},
+	{ARM_CPU_FEATURE_PMULL,		"pmull"},
+	{ARM_CPU_FEATURE_CRC32,		"crc32"},
+};
+
+volatile u32 _cpu_features = 0;
+
+void setup_cpu_features(void)
+{
+	u32 features = get_arm_cpu_features();
+
+	disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
+					 ARRAY_LEN(arm_cpu_feature_table));
 
 	_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
 }

lib/arm/cpu_features.h

@@ -8,7 +8,10 @@
 #include "../lib_common.h"
 
 #if (defined(__arm__) || defined(__aarch64__)) && \
-	defined(__linux__) && COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+	(defined(__linux__) || \
+	 (defined(__aarch64__) && defined(__APPLE__))) && \
+	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE && \
+	!defined(FREESTANDING)
 #  define ARM_CPU_FEATURES_ENABLED 1
 #else
 #  define ARM_CPU_FEATURES_ENABLED 0
@@ -18,12 +21,13 @@
 
 #define ARM_CPU_FEATURE_NEON		0x00000001
 #define ARM_CPU_FEATURE_PMULL		0x00000002
+#define ARM_CPU_FEATURE_CRC32		0x00000004
 
 #define ARM_CPU_FEATURES_KNOWN		0x80000000
 
 extern volatile u32 _cpu_features;
 
-extern void setup_cpu_features(void);
+void setup_cpu_features(void);
 
 static inline u32 get_cpu_features(void)
 {

lib/arm/crc32_impl.h

@@ -26,8 +26,89 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_ARM_CRC32_IMPL_H
+#define LIB_ARM_CRC32_IMPL_H
+
 #include "cpu_features.h"
 
+/* Implementation using ARM CRC32 instructions */
+#undef DISPATCH_ARM
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRC32) || \
+     (ARM_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS))
+#  ifdef __ARM_FEATURE_CRC32
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_arm
+#  else
+#    ifdef __arm__
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("armv8-a,crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("arch=armv8-a+crc")))
+#      endif
+#    else
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("+crc")))
+#      endif
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_ARM	1
+#  endif
+
+/*
+ * gcc's (as of 10.1) version of arm_acle.h for arm32, and clang's (as of
+ * 10.0.1) version of arm_acle.h for both arm32 and arm64, have a bug where they
+ * only define the CRC32 functions like __crc32b() when __ARM_FEATURE_CRC32 is
+ * defined.  That prevents them from being used via __attribute__((target)) when
+ * the main target doesn't have CRC32 support enabled.  The actual built-ins
+ * like __builtin_arm_crc32b() are available and work, however; it's just the
+ * wrappers in arm_acle.h like __crc32b() that erroneously don't get defined.
+ * Work around this by manually defining __ARM_FEATURE_CRC32.
+ */
+#ifndef __ARM_FEATURE_CRC32
+#  define __ARM_FEATURE_CRC32	1
+#endif
+#include <arm_acle.h>
+
+static u32 ATTRIBUTES
+crc32_arm(u32 remainder, const u8 *p, size_t size)
+{
+	while (size != 0 && (uintptr_t)p & 7) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	while (size >= 64) {
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 0)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 1)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 2)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 3)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 4)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 5)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 6)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 7)));
+		p += 64;
+		size -= 64;
+	}
+
+	while (size >= 8) {
+		remainder = __crc32d(remainder, le64_bswap(*(u64 *)p));
+		p += 8;
+		size -= 8;
+	}
+
+	while (size != 0) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	return remainder;
+}
+#undef ATTRIBUTES
+#endif /* Implementation using ARM CRC32 instructions */
+
 /*
  * CRC-32 folding with ARM Crypto extension-PMULL
  *
@@ -35,14 +116,12 @@
  * See x86/crc32_pclmul_template.h for an explanation.
  */
 #undef DISPATCH_PMULL
-#if (defined(__ARM_FEATURE_CRYPTO) ||	\
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRYPTO) ||	\
      (ARM_CPU_FEATURES_ENABLED &&	\
       COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS)) && \
       /* not yet tested on big endian, probably needs changes to work there */ \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) && \
-      /* clang as of v5.0.1 doesn't allow pmull intrinsics in 32-bit mode, even
-       * when compiling with -mfpu=crypto-neon-fp-armv8 */ \
-    !(defined(__clang__) && defined(__arm__))
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
 #  define FUNCNAME		crc32_pmull
 #  define FUNCNAME_ALIGNED	crc32_pmull_aligned
 #  ifdef __ARM_FEATURE_CRYPTO
@@ -157,6 +236,10 @@ arch_select_crc32_func(void)
 {
 	u32 features = get_cpu_features();
 
+#ifdef DISPATCH_ARM
+	if (features & ARM_CPU_FEATURE_CRC32)
+		return crc32_arm;
+#endif
 #ifdef DISPATCH_PMULL
 	if (features & ARM_CPU_FEATURE_PMULL)
 		return crc32_pmull;
@@ -164,3 +247,5 @@ arch_select_crc32_func(void)
 	return NULL;
 }
 #endif /* DISPATCH */
+
+#endif /* LIB_ARM_CRC32_IMPL_H */

lib/arm/matchfinder_impl.h

@@ -25,69 +25,62 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_ARM_MATCHFINDER_IMPL_H
+#define LIB_ARM_MATCHFINDER_IMPL_H
+
 #ifdef __ARM_NEON
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
-#  endif
 #  include <arm_neon.h>
-static forceinline bool
+static forceinline void
 matchfinder_init_neon(mf_pos_t *data, size_t size)
 {
-	int16x8_t v, *p;
-	size_t n;
-
-	if (size % (sizeof(int16x8_t) * 4) != 0)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
 		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
 		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
 		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
 	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
 	do {
 		p[0] = v;
 		p[1] = v;
 		p[2] = v;
 		p[3] = v;
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
-#undef arch_matchfinder_init
-#define arch_matchfinder_init matchfinder_init_neon
+#define matchfinder_init matchfinder_init_neon
 
-static forceinline bool
+static forceinline void
 matchfinder_rebase_neon(mf_pos_t *data, size_t size)
 {
-	int16x8_t v, *p;
-	size_t n;
-
-	if (size % (sizeof(int16x8_t) * 4) != 0)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
 		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
 		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
 		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
 		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
 	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
 	do {
 		p[0] = vqaddq_s16(p[0], v);
 		p[1] = vqaddq_s16(p[1], v);
 		p[2] = vqaddq_s16(p[2], v);
 		p[3] = vqaddq_s16(p[3], v);
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
-#undef arch_matchfinder_rebase
-#define arch_matchfinder_rebase matchfinder_rebase_neon
+#define matchfinder_rebase matchfinder_rebase_neon
 
 #endif /* __ARM_NEON */
+
+#endif /* LIB_ARM_MATCHFINDER_IMPL_H */

lib/bt_matchfinder.h

@@ -1,8 +1,6 @@
 /*
  * bt_matchfinder.h - Lempel-Ziv matchfinding with a hash table of binary trees
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -64,6 +62,8 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_BT_MATCHFINDER_H
+#define LIB_BT_MATCHFINDER_H
 
 #include "matchfinder_common.h"
 
@@ -71,9 +71,9 @@
 #define BT_MATCHFINDER_HASH3_WAYS  2
 #define BT_MATCHFINDER_HASH4_ORDER 16
 
-#define BT_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
-	 (1UL << BT_MATCHFINDER_HASH4_ORDER))
+#define BT_MATCHFINDER_TOTAL_HASH_SIZE		\
+	(((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
+	  (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 /* Representation of a match found by the bt_matchfinder  */
 struct lz_match {
@@ -99,24 +99,24 @@ struct bt_matchfinder {
 	 * 'child_tab[pos * 2]' and 'child_tab[pos * 2 + 1]', respectively.  */
 	mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE];
 
-}
-#ifdef _aligned_attribute
-_aligned_attribute(MATCHFINDER_ALIGNMENT)
-#endif
-;
+} MATCHFINDER_ALIGNED;
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void
 bt_matchfinder_init(struct bt_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 bt_matchfinder_slide_window(struct bt_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct bt_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 static forceinline mf_pos_t *
@@ -132,7 +132,7 @@ bt_right_child(struct bt_matchfinder *mf, s32 node)
 }
 
 /* The minimum permissible value of 'max_len' for bt_matchfinder_get_matches()
- * and bt_matchfinder_skip_position().  There must be sufficiently many bytes
+ * and bt_matchfinder_skip_byte().  There must be sufficiently many bytes
  * remaining to load a 32-bit integer from the *next* position.  */
 #define BT_MATCHFINDER_REQUIRED_NBYTES	5
 
@@ -146,7 +146,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 				const u32 nice_len,
 				const u32 max_search_depth,
 				u32 * const restrict next_hashes,
-				u32 * const restrict best_len_ret,
 				struct lz_match * restrict lz_matchptr,
 				const bool record_matches)
 {
@@ -212,7 +211,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 	if (cur_node <= cutoff) {
 		*pending_lt_ptr = MATCHFINDER_INITVAL;
 		*pending_gt_ptr = MATCHFINDER_INITVAL;
-		*best_len_ret = best_len;
 		return lz_matchptr;
 	}
 
@@ -235,7 +233,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 				if (len >= nice_len) {
 					*pending_lt_ptr = *bt_left_child(mf, cur_node);
 					*pending_gt_ptr = *bt_right_child(mf, cur_node);
-					*best_len_ret = best_len;
 					return lz_matchptr;
 				}
 			}
@@ -260,7 +257,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 		if (cur_node <= cutoff || !--depth_remaining) {
 			*pending_lt_ptr = MATCHFINDER_INITVAL;
 			*pending_gt_ptr = MATCHFINDER_INITVAL;
-			*best_len_ret = best_len;
 			return lz_matchptr;
 		}
 	}
@@ -289,12 +285,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
  *	The precomputed hash codes for the sequence beginning at @in_next.
  *	These will be used and then updated with the precomputed hashcodes for
  *	the sequence beginning at @in_next + 1.
- * @best_len_ret
- *	If a match of length >= 4 was found, then the length of the longest such
- *	match is written here; otherwise 3 is written here.  (Note: this is
- *	redundant with the 'struct lz_match' array, but this is easier for the
- *	compiler to optimize when inlined and the caller immediately does a
- *	check against 'best_len'.)
  * @lz_matchptr
  *	An array in which this function will record the matches.  The recorded
  *	matches will be sorted by strictly increasing length and (non-strictly)
@@ -312,7 +302,6 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
 			   u32 nice_len,
 			   u32 max_search_depth,
 			   u32 next_hashes[2],
-			   u32 *best_len_ret,
 			   struct lz_match *lz_matchptr)
 {
 	return bt_matchfinder_advance_one_byte(mf,
@@ -322,7 +311,6 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
 					       nice_len,
 					       max_search_depth,
 					       next_hashes,
-					       best_len_ret,
 					       lz_matchptr,
 					       true);
 }
@@ -334,14 +322,13 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
  * must do hashing and tree re-rooting.
  */
 static forceinline void
-bt_matchfinder_skip_position(struct bt_matchfinder *mf,
-			     const u8 *in_base,
-			     ptrdiff_t cur_pos,
-			     u32 nice_len,
-			     u32 max_search_depth,
-			     u32 next_hashes[2])
+bt_matchfinder_skip_byte(struct bt_matchfinder *mf,
+			 const u8 *in_base,
+			 ptrdiff_t cur_pos,
+			 u32 nice_len,
+			 u32 max_search_depth,
+			 u32 next_hashes[2])
 {
-	u32 best_len;
 	bt_matchfinder_advance_one_byte(mf,
 					in_base,
 					cur_pos,
@@ -349,7 +336,8 @@ bt_matchfinder_skip_position(struct bt_matchfinder *mf,
 					nice_len,
 					max_search_depth,
 					next_hashes,
-					&best_len,
 					NULL,
 					false);
 }
+
+#endif /* LIB_BT_MATCHFINDER_H */

lib/cpu_features_common.h

@@ -0,0 +1,88 @@
+/*
+ * cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
+ *
+ * Copyright 2020 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_CPU_FEATURES_COMMON_H
+#define LIB_CPU_FEATURES_COMMON_H
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+#  define _GNU_SOURCE 1 /* for strdup() and strtok_r() */
+#  include <stdio.h>
+#  include <stdlib.h>
+#  include <string.h>
+#endif
+
+#include "lib_common.h"
+
+struct cpu_feature {
+	u32 bit;
+	const char *name;
+};
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+	char *env_value, *strbuf, *p, *saveptr = NULL;
+	size_t i;
+
+	env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
+	if (!env_value)
+		return;
+	strbuf = strdup(env_value);
+	if (!strbuf)
+		abort();
+	p = strtok_r(strbuf, ",", &saveptr);
+	while (p) {
+		for (i = 0; i < feature_table_length; i++) {
+			if (strcmp(p, feature_table[i].name) == 0) {
+				*features &= ~feature_table[i].bit;
+				break;
+			}
+		}
+		if (i == feature_table_length) {
+			fprintf(stderr,
+				"unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
+				p);
+			abort();
+		}
+		p = strtok_r(NULL, ",", &saveptr);
+	}
+	free(strbuf);
+}
+#else /* TEST_SUPPORT__DO_NOT_USE */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+}
+#endif /* !TEST_SUPPORT__DO_NOT_USE */
+
+#endif /* LIB_CPU_FEATURES_COMMON_H */

lib/crc32.c

@@ -304,7 +304,7 @@ static u32 dispatch(u32 remainder, const u8 *buffer, size_t size)
 #  define crc32_impl DEFAULT_IMPL /* only one implementation, use it */
 #endif
 
-LIBDEFLATEAPI u32
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
 libdeflate_crc32(u32 remainder, const void *buffer, size_t size)
 {
 	if (buffer == NULL) /* return initial value */

lib/decompress_template.h

@@ -43,7 +43,7 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 	const u8 * const in_end = in_next + in_nbytes;
 	bitbuf_t bitbuf = 0;
 	unsigned bitsleft = 0;
-	size_t overrun_count = 0;
+	size_t overread_count = 0;
 	unsigned i;
 	unsigned is_final_block;
 	unsigned block_type;
@@ -89,19 +89,13 @@ next_block:
 		STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == ((1 << 4) - 1) + 4);
 		num_explicit_precode_lens = POP_BITS(4) + 4;
 
+		d->static_codes_loaded = false;
+
 		/* Read the precode codeword lengths.  */
 		STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
-		if (CAN_ENSURE(DEFLATE_NUM_PRECODE_SYMS * 3)) {
-
-			ENSURE_BITS(DEFLATE_NUM_PRECODE_SYMS * 3);
-
-			for (i = 0; i < num_explicit_precode_lens; i++)
-				d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-		} else {
-			for (i = 0; i < num_explicit_precode_lens; i++) {
-				ENSURE_BITS(3);
-				d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-			}
+		for (i = 0; i < num_explicit_precode_lens; i++) {
+			ENSURE_BITS(3);
+			d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
 		}
 
 		for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
@@ -217,9 +211,20 @@ next_block:
 	} else {
 		SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
 
-		/* Static Huffman block: set the static Huffman codeword
-		 * lengths.  Then the remainder is the same as decompressing a
-		 * dynamic Huffman block.  */
+		/*
+		 * Static Huffman block: build the decode tables for the static
+		 * codes.  Skip doing so if the tables are already set up from
+		 * an earlier static block; this speeds up decompression of
+		 * degenerate input of many empty or very short static blocks.
+		 *
+		 * Afterwards, the remainder is the same as decompressing a
+		 * dynamic Huffman block.
+		 */
+
+		if (d->static_codes_loaded)
+			goto have_decode_tables;
+
+		d->static_codes_loaded = true;
 
 		STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
 		STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
@@ -238,19 +243,21 @@ next_block:
 
 		num_litlen_syms = 288;
 		num_offset_syms = 32;
-
 	}
 
 	/* Decompressing a Huffman block (either dynamic or static)  */
 
 	SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
 	SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
+have_decode_tables:
 
 	/* The main DEFLATE decode loop  */
 	for (;;) {
 		u32 entry;
 		u32 length;
 		u32 offset;
+		const u8 *src;
+		u8 *dst;
 
 		/* Decode a litlen symbol.  */
 		ENSURE_BITS(DEFLATE_MAX_LITLEN_CODEWORD_LEN);
@@ -323,65 +330,66 @@ next_block:
 		 * output buffer.  */
 		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
 
-		/* Copy the match: 'length' bytes at 'out_next - offset' to
-		 * 'out_next'.  */
+		/*
+		 * Copy the match: 'length' bytes at 'out_next - offset' to
+		 * 'out_next', possibly overlapping.  If the match doesn't end
+		 * too close to the end of the buffer and offset >= WORDBYTES ||
+		 * offset == 1, take a fast path which copies a word at a time
+		 * -- potentially more than the length of the match, but that's
+		 * fine as long as we check for enough extra space.
+		 *
+		 * The remaining cases are not performance-critical so are
+		 * handled by a simple byte-by-byte copy.
+		 */
+
+		src = out_next - offset;
+		dst = out_next;
+		out_next += length;
 
 		if (UNALIGNED_ACCESS_IS_FAST &&
-		    length <= (3 * WORDBYTES) &&
-		    offset >= WORDBYTES &&
-		    length + (3 * WORDBYTES) <= out_end - out_next)
-		{
-			/* Fast case: short length, no overlaps if we copy one
-			 * word at a time, and we aren't getting too close to
-			 * the end of the output array.  */
-			copy_word_unaligned(out_next - offset + (0 * WORDBYTES),
-					    out_next + (0 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (1 * WORDBYTES),
-					    out_next + (1 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (2 * WORDBYTES),
-					    out_next + (2 * WORDBYTES));
-		} else {
-			const u8 *src = out_next - offset;
-			u8 *dst = out_next;
-			u8 *end = out_next + length;
-
-			if (UNALIGNED_ACCESS_IS_FAST &&
-			    likely(out_end - end >= WORDBYTES - 1)) {
-				if (offset >= WORDBYTES) {
+		    /* max overrun is writing 3 words for a min length match */
+		    likely(out_end - out_next >=
+			   3 * WORDBYTES - DEFLATE_MIN_MATCH_LEN)) {
+			if (offset >= WORDBYTES) { /* words don't overlap? */
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				do {
 					copy_word_unaligned(src, dst);
 					src += WORDBYTES;
 					dst += WORDBYTES;
-					if (dst < end) {
-						do {
-							copy_word_unaligned(src, dst);
-							src += WORDBYTES;
-							dst += WORDBYTES;
-						} while (dst < end);
-					}
-				} else if (offset == 1) {
-					machine_word_t v = repeat_byte(*(dst - 1));
-					do {
-						store_word_unaligned(v, dst);
-						src += WORDBYTES;
-						dst += WORDBYTES;
-					} while (dst < end);
-				} else {
-					*dst++ = *src++;
-					*dst++ = *src++;
-					do {
-						*dst++ = *src++;
-					} while (dst < end);
-				}
+				} while (dst < out_next);
+			} else if (offset == 1) {
+				/* RLE encoding of previous byte, common if the
+				 * data contains many repeated bytes */
+				machine_word_t v = repeat_byte(*src);
+
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				do {
+					store_word_unaligned(v, dst);
+					dst += WORDBYTES;
+				} while (dst < out_next);
 			} else {
 				*dst++ = *src++;
 				*dst++ = *src++;
 				do {
 					*dst++ = *src++;
-				} while (dst < end);
+				} while (dst < out_next);
 			}
+		} else {
+			STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
+			*dst++ = *src++;
+			*dst++ = *src++;
+			do {
+				*dst++ = *src++;
+			} while (dst < out_next);
 		}
-
-		out_next += length;
 	}
 
 block_done:

lib/deflate_compress.h

@@ -3,12 +3,13 @@
 
 #include "lib_common.h"
 
-/* DEFLATE compression is private to deflate_compress.c, but we do need to be
- * able to query the compression level for zlib and gzip header generation.  */
+/*
+ * DEFLATE compression is private to deflate_compress.c, but we do need to be
+ * able to query the compression level for zlib and gzip header generation.
+ */
 
 struct libdeflate_compressor;
 
-extern unsigned int
-deflate_get_compression_level(struct libdeflate_compressor *c);
+unsigned int deflate_get_compression_level(struct libdeflate_compressor *c);
 
 #endif /* LIB_DEFLATE_COMPRESS_H */

lib/deflate_constants.h

@@ -14,11 +14,11 @@
 #define DEFLATE_MIN_MATCH_LEN			3
 #define DEFLATE_MAX_MATCH_LEN			258
 
-/* Minimum and maximum supported match offsets (in bytes)  */
-#define DEFLATE_MIN_MATCH_OFFSET		1
+/* Maximum supported match offset (in bytes) */
 #define DEFLATE_MAX_MATCH_OFFSET		32768
 
-#define DEFLATE_MAX_WINDOW_SIZE			32768
+/* log2 of DEFLATE_MAX_MATCH_OFFSET */
+#define DEFLATE_WINDOW_ORDER			15
 
 /* Number of symbols in each Huffman code.  Note: for the literal/length
  * and offset codes, these are actually the maximum values; a given block
@@ -33,7 +33,7 @@
 /* Division of symbols in the literal/length code  */
 #define DEFLATE_NUM_LITERALS			256
 #define DEFLATE_END_OF_BLOCK			256
-#define DEFLATE_NUM_LEN_SYMS			31
+#define DEFLATE_FIRST_LEN_SYM			257
 
 /* Maximum codeword length, in bits, within each Huffman code  */
 #define DEFLATE_MAX_PRE_CODEWORD_LEN		7
@@ -56,11 +56,4 @@
 #define DEFLATE_MAX_EXTRA_LENGTH_BITS		5
 #define DEFLATE_MAX_EXTRA_OFFSET_BITS		14
 
-/* The maximum number of bits in which a match can be represented.  This
- * is the absolute worst case, which assumes the longest possible Huffman
- * codewords and the maximum numbers of extra bits.  */
-#define DEFLATE_MAX_MATCH_BITS	\
-	(DEFLATE_MAX_LITLEN_CODEWORD_LEN + DEFLATE_MAX_EXTRA_LENGTH_BITS + \
-	DEFLATE_MAX_OFFSET_CODEWORD_LEN + DEFLATE_MAX_EXTRA_OFFSET_BITS)
-
 #endif /* LIB_DEFLATE_CONSTANTS_H */

lib/deflate_decompress.c

@@ -44,8 +44,7 @@
  *   instructions and use it automatically at runtime when supported.
  */
 
-#include <stdlib.h>
-#include <string.h>
+#include <limits.h>
 
 #include "deflate_constants.h"
 #include "unaligned.h"
@@ -87,7 +86,7 @@
  * required for the corresponding Huffman code, including the main table and all
  * subtables.  Each number depends on three parameters:
  *
- *	(1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMBOLS)
+ *	(1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMS)
  *	(2) the number of main table bits (the TABLEBITS numbers defined above)
  *	(3) the maximum allowed codeword length (DEFLATE_MAX_*_CODEWORD_LEN)
  *
@@ -99,11 +98,6 @@
 #define LITLEN_ENOUGH		1334	/* enough 288 10 15	*/
 #define OFFSET_ENOUGH		402	/* enough 32 8 15	*/
 
-/*
- * Type for codeword lengths.
- */
-typedef u8 len_t;
-
 /*
  * The main DEFLATE decompressor structure.  Since this implementation only
  * supports full buffer decompression, this structure does not store the entire
@@ -122,12 +116,12 @@ struct libdeflate_decompressor {
 	 */
 
 	union {
-		len_t precode_lens[DEFLATE_NUM_PRECODE_SYMS];
+		u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];
 
 		struct {
-			len_t lens[DEFLATE_NUM_LITLEN_SYMS +
-				   DEFLATE_NUM_OFFSET_SYMS +
-				   DEFLATE_MAX_LENS_OVERRUN];
+			u8 lens[DEFLATE_NUM_LITLEN_SYMS +
+				DEFLATE_NUM_OFFSET_SYMS +
+				DEFLATE_MAX_LENS_OVERRUN];
 
 			u32 precode_decode_table[PRECODE_ENOUGH];
 		} l;
@@ -137,8 +131,10 @@ struct libdeflate_decompressor {
 
 	u32 offset_decode_table[OFFSET_ENOUGH];
 
-	u16 working_space[2 * (DEFLATE_MAX_CODEWORD_LEN + 1) +
-			  DEFLATE_MAX_NUM_SYMS];
+	/* used only during build_decode_table() */
+	u16 sorted_syms[DEFLATE_MAX_NUM_SYMS];
+
+	bool static_codes_loaded;
 };
 
 /*****************************************************************************
@@ -174,18 +170,18 @@ typedef machine_word_t bitbuf_t;
 
 /*
  * Number of bits the bitbuffer variable can hold.
+ *
+ * This is one less than the obvious value because of the optimized arithmetic
+ * in FILL_BITS_WORDWISE() that leaves 'bitsleft' in the range
+ * [WORDBITS - 8, WORDBITS - 1] rather than [WORDBITS - 7, WORDBITS].
  */
-#define BITBUF_NBITS	(8 * sizeof(bitbuf_t))
+#define BITBUF_NBITS	(8 * sizeof(bitbuf_t) - 1)
 
 /*
- * The maximum number of bits that can be requested to be in the bitbuffer
- * variable.  This is the maximum value of 'n' that can be passed
- * ENSURE_BITS(n).
- *
- * This not equal to BITBUF_NBITS because we never read less than one byte at a
- * time.  If the bitbuffer variable contains more than (BITBUF_NBITS - 8) bits,
- * then we can't read another byte without first consuming some bits.  So the
- * maximum count we can ensure is (BITBUF_NBITS - 7).
+ * The maximum number of bits that can be ensured in the bitbuffer variable,
+ * i.e. the maximum value of 'n' that can be passed ENSURE_BITS(n).  The decoder
+ * only reads whole bytes from memory, so this is the lowest value of 'bitsleft'
+ * at which another byte cannot be read without first consuming some bits.
  */
 #define MAX_ENSURE	(BITBUF_NBITS - 7)
 
@@ -203,41 +199,63 @@ typedef machine_word_t bitbuf_t;
  *
  * If we would overread the input buffer, we just don't read anything, leaving
  * the bits zeroed but marking them filled.  This simplifies the decompressor
- * because it removes the need to distinguish between real overreads and
- * overreads that occur only because of the decompressor's own lookahead.
+ * because it removes the need to always be able to distinguish between real
+ * overreads and overreads caused only by the decompressor's own lookahead.
  *
- * The disadvantage is that real overreads are not detected immediately.
- * However, this is safe because the decompressor is still guaranteed to make
- * forward progress when presented never-ending 0 bits.  In an existing block
- * output will be getting generated, whereas new blocks can only be uncompressed
- * (since the type code for uncompressed blocks is 0), for which we check for
- * previous overread.  But even if we didn't check, uncompressed blocks would
- * fail to validate because LEN would not equal ~NLEN.  So the decompressor will
- * eventually either detect that the output buffer is full, or detect invalid
- * input, or finish the final block.
+ * We do still keep track of the number of bytes that have been overread, for
+ * two reasons.  First, it allows us to determine the exact number of bytes that
+ * were consumed once the stream ends or an uncompressed block is reached.
+ * Second, it allows us to stop early if the overread amount gets so large (more
+ * than sizeof bitbuf) that it can only be caused by a real overread.  (The
+ * second part is arguably unneeded, since libdeflate is buffer-based; given
+ * infinite zeroes, it will eventually either completely fill the output buffer
+ * or return an error.  However, we do it to be slightly more friendly to the
+ * not-recommended use case of decompressing with an unknown output size.)
  */
 #define FILL_BITS_BYTEWISE()					\
 do {								\
-	if (likely(in_next != in_end))				\
+	if (likely(in_next != in_end)) {			\
 		bitbuf |= (bitbuf_t)*in_next++ << bitsleft;	\
-	else							\
-		overrun_count++;				\
+	} else {						\
+		overread_count++;				\
+		SAFETY_CHECK(overread_count <= sizeof(bitbuf));	\
+	}							\
 	bitsleft += 8;						\
 } while (bitsleft <= BITBUF_NBITS - 8)
 
 /*
- * Fill the bitbuffer variable by reading the next word from the input buffer.
- * This can be significantly faster than FILL_BITS_BYTEWISE().  However, for
- * this to work correctly, the word must be interpreted in little-endian format.
- * In addition, the memory access may be unaligned.  Therefore, this method is
- * most efficient on little-endian architectures that support fast unaligned
- * access, such as x86 and x86_64.
+ * Fill the bitbuffer variable by reading the next word from the input buffer
+ * and branchlessly updating 'in_next' and 'bitsleft' based on how many bits
+ * were filled.  This can be significantly faster than FILL_BITS_BYTEWISE().
+ * However, for this to work correctly, the word must be interpreted in
+ * little-endian format.  In addition, the memory access may be unaligned.
+ * Therefore, this method is most efficient on little-endian architectures that
+ * support fast unaligned access, such as x86 and x86_64.
+ *
+ * For faster updating of 'bitsleft', we consider the bitbuffer size in bits to
+ * be 1 less than the word size and therefore be all 1 bits.  Then the number of
+ * bits filled is the value of the 0 bits in position >= 3 when changed to 1.
+ * E.g. if words are 64 bits and bitsleft = 16 = b010000 then we refill b101000
+ * = 40 bits = 5 bytes.  This uses only 4 operations to update 'in_next' and
+ * 'bitsleft': one each of +, ^, >>, and |.  (Not counting operations the
+ * compiler optimizes out.)  In contrast, the alternative of:
+ *
+ *	in_next += (BITBUF_NBITS - bitsleft) >> 3;
+ *	bitsleft += (BITBUF_NBITS - bitsleft) & ~7;
+ *
+ * (where BITBUF_NBITS would be WORDBITS rather than WORDBITS - 1) would on
+ * average refill an extra bit, but uses 5 operations: two +, and one each of
+ * -, >>, and &.  Also the - and & must be completed before 'bitsleft' can be
+ * updated, while the current solution updates 'bitsleft' with no dependencies.
  */
 #define FILL_BITS_WORDWISE()					\
 do {								\
+	/* BITBUF_NBITS must be all 1's in binary, see above */	\
+	STATIC_ASSERT((BITBUF_NBITS & (BITBUF_NBITS + 1)) == 0);\
+								\
 	bitbuf |= get_unaligned_leword(in_next) << bitsleft;	\
-	in_next += (BITBUF_NBITS - bitsleft) >> 3;		\
-	bitsleft += (BITBUF_NBITS - bitsleft) & ~7;		\
+	in_next += (bitsleft ^ BITBUF_NBITS) >> 3;		\
+	bitsleft |= BITBUF_NBITS & ~7;				\
 } while (0)
 
 /*
@@ -286,16 +304,16 @@ if (!HAVE_BITS(n)) {						\
  */
 #define ALIGN_INPUT()							\
 do {									\
-	SAFETY_CHECK(overrun_count <= (bitsleft >> 3));			\
-	in_next -= (bitsleft >> 3) - overrun_count;			\
-	overrun_count = 0;						\
+	SAFETY_CHECK(overread_count <= (bitsleft >> 3));		\
+	in_next -= (bitsleft >> 3) - overread_count;			\
+	overread_count = 0;						\
 	bitbuf = 0;							\
 	bitsleft = 0;							\
 } while(0)
 
 /*
  * Read a 16-bit value from the input.  This must have been preceded by a call
- * to ALIGN_INPUT(), and the caller must have already checked for overrun.
+ * to ALIGN_INPUT(), and the caller must have already checked for overread.
  */
 #define READ_U16() (tmp16 = get_unaligned_le16(in_next), in_next += 2, tmp16)
 
@@ -313,7 +331,7 @@ do {									\
  * pointer", meaning that the entry references a subtable that must be indexed
  * using more bits of the compressed data to decode the symbol.
  *
- * Each decode table (a main table along with with its subtables, if any) is
+ * Each decode table (a main table along with its subtables, if any) is
  * associated with a Huffman code.  Logically, the result of a decode table
  * lookup is a symbol from the alphabet from which the corresponding Huffman
  * code was constructed.  A symbol with codeword length n <= TABLEBITS is
@@ -362,19 +380,28 @@ do {									\
 /* Shift to extract the decode result from a decode table entry.  */
 #define HUFFDEC_RESULT_SHIFT		8
 
+/* Shift a decode result into its position in the decode table entry.  */
+#define HUFFDEC_RESULT_ENTRY(result)	((u32)(result) << HUFFDEC_RESULT_SHIFT)
+
 /* The decode result for each precode symbol.  There is no special optimization
  * for the precode; the decode result is simply the symbol value.  */
 static const u32 precode_decode_results[DEFLATE_NUM_PRECODE_SYMS] = {
-	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+#define ENTRY(presym)	HUFFDEC_RESULT_ENTRY(presym)
+	ENTRY(0)   , ENTRY(1)   , ENTRY(2)   , ENTRY(3)   ,
+	ENTRY(4)   , ENTRY(5)   , ENTRY(6)   , ENTRY(7)   ,
+	ENTRY(8)   , ENTRY(9)   , ENTRY(10)  , ENTRY(11)  ,
+	ENTRY(12)  , ENTRY(13)  , ENTRY(14)  , ENTRY(15)  ,
+	ENTRY(16)  , ENTRY(17)  , ENTRY(18)  ,
+#undef ENTRY
 };
 
 /* The decode result for each litlen symbol.  For literals, this is the literal
  * value itself and the HUFFDEC_LITERAL flag.  For lengths, this is the length
  * base and the number of extra length bits.  */
 static const u32 litlen_decode_results[DEFLATE_NUM_LITLEN_SYMS] = {
-#define ENTRY(literal)	((HUFFDEC_LITERAL >> HUFFDEC_RESULT_SHIFT) | (literal))
 
 	/* Literals  */
+#define ENTRY(literal)	(HUFFDEC_LITERAL | HUFFDEC_RESULT_ENTRY(literal))
 	ENTRY(0)   , ENTRY(1)   , ENTRY(2)   , ENTRY(3)   ,
 	ENTRY(4)   , ENTRY(5)   , ENTRY(6)   , ENTRY(7)   ,
 	ENTRY(8)   , ENTRY(9)   , ENTRY(10)  , ENTRY(11)  ,
@@ -445,8 +472,8 @@ static const u32 litlen_decode_results[DEFLATE_NUM_LITLEN_SYMS] = {
 #define HUFFDEC_LENGTH_BASE_SHIFT	8
 #define HUFFDEC_END_OF_BLOCK_LENGTH	0
 
-#define ENTRY(length_base, num_extra_bits) \
-	(((u32)(length_base) << HUFFDEC_LENGTH_BASE_SHIFT) | (num_extra_bits))
+#define ENTRY(length_base, num_extra_bits)	HUFFDEC_RESULT_ENTRY(	\
+	((u32)(length_base) << HUFFDEC_LENGTH_BASE_SHIFT) | (num_extra_bits))
 
 	/* End of block  */
 	ENTRY(HUFFDEC_END_OF_BLOCK_LENGTH, 0),
@@ -470,8 +497,9 @@ static const u32 offset_decode_results[DEFLATE_NUM_OFFSET_SYMS] = {
 #define HUFFDEC_EXTRA_OFFSET_BITS_SHIFT 16
 #define HUFFDEC_OFFSET_BASE_MASK (((u32)1 << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT) - 1)
 
-#define ENTRY(offset_base, num_extra_bits) \
-	((offset_base) | ((u32)(num_extra_bits) << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT))
+#define ENTRY(offset_base, num_extra_bits)	HUFFDEC_RESULT_ENTRY(	\
+		((u32)(num_extra_bits) << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT) | \
+		(offset_base))
 	ENTRY(1     , 0)  , ENTRY(2     , 0)  , ENTRY(3     , 0)  , ENTRY(4     , 0)  ,
 	ENTRY(5     , 1)  , ENTRY(7     , 1)  , ENTRY(9     , 2)  , ENTRY(13    , 2) ,
 	ENTRY(17    , 3)  , ENTRY(25    , 3)  , ENTRY(33    , 4)  , ENTRY(49    , 4)  ,
@@ -483,13 +511,6 @@ static const u32 offset_decode_results[DEFLATE_NUM_OFFSET_SYMS] = {
 #undef ENTRY
 };
 
-/* Construct a decode table entry from a decode result and codeword length.  */
-static forceinline u32
-make_decode_table_entry(u32 result, u32 length)
-{
-	return (result << HUFFDEC_RESULT_SHIFT) | length;
-}
-
 /*
  * Build a table for fast decoding of symbols from a Huffman code.  As input,
  * this function takes the codeword length of each symbol which may be used in
@@ -521,209 +542,272 @@ make_decode_table_entry(u32 result, u32 length)
  *	The log base-2 of the number of main table entries to use.
  * @max_codeword_len
  *	The maximum allowed codeword length for this Huffman code.
- * @working_space
- *	A temporary array of length '2 * (@max_codeword_len + 1) + @num_syms'.
+ *	Must be <= DEFLATE_MAX_CODEWORD_LEN.
+ * @sorted_syms
+ *	A temporary array of length @num_syms.
  *
  * Returns %true if successful; %false if the codeword lengths do not form a
  * valid Huffman code.
  */
 static bool
 build_decode_table(u32 decode_table[],
-		   const len_t lens[],
+		   const u8 lens[],
 		   const unsigned num_syms,
 		   const u32 decode_results[],
 		   const unsigned table_bits,
 		   const unsigned max_codeword_len,
-		   u16 working_space[])
+		   u16 *sorted_syms)
 {
-	u16 * const len_counts = &working_space[0];
-	u16 * const offsets = &working_space[1 * (max_codeword_len + 1)];
-	u16 * const sorted_syms = &working_space[2 * (max_codeword_len + 1)];
-	unsigned len;
-	unsigned sym;
-	s32 remainder;
-	unsigned sym_idx;
-	unsigned codeword_len;
-	unsigned codeword_reversed = 0;
-	unsigned cur_codeword_prefix = -1;
-	unsigned cur_table_start = 0;
-	unsigned cur_table_bits = table_bits;
-	unsigned num_dropped_bits = 0;
-	const unsigned table_mask = (1U << table_bits) - 1;
+	unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
+	unsigned offsets[DEFLATE_MAX_CODEWORD_LEN + 1];
+	unsigned sym;		/* current symbol */
+	unsigned codeword;	/* current codeword, bit-reversed */
+	unsigned len;		/* current codeword length in bits */
+	unsigned count;		/* num codewords remaining with this length */
+	u32 codespace_used;	/* codespace used out of '2^max_codeword_len' */
+	unsigned cur_table_end; /* end index of current table */
+	unsigned subtable_prefix; /* codeword prefix of current subtable */
+	unsigned subtable_start;  /* start index of current subtable */
+	unsigned subtable_bits;   /* log2 of current subtable length */
 
-	/* Count how many symbols have each codeword length, including 0.  */
+	/* Count how many codewords have each length, including 0. */
 	for (len = 0; len <= max_codeword_len; len++)
 		len_counts[len] = 0;
 	for (sym = 0; sym < num_syms; sym++)
 		len_counts[lens[sym]]++;
 
-	/* Sort the symbols primarily by increasing codeword length and
-	 * secondarily by increasing symbol value.  */
+	/*
+	 * Sort the symbols primarily by increasing codeword length and
+	 * secondarily by increasing symbol value; or equivalently by their
+	 * codewords in lexicographic order, since a canonical code is assumed.
+	 *
+	 * For efficiency, also compute 'codespace_used' in the same pass over
+	 * 'len_counts[]' used to build 'offsets[]' for sorting.
+	 */
+
+	/* Ensure that 'codespace_used' cannot overflow. */
+	STATIC_ASSERT(sizeof(codespace_used) == 4);
+	STATIC_ASSERT(UINT32_MAX / (1U << (DEFLATE_MAX_CODEWORD_LEN - 1)) >=
+		      DEFLATE_MAX_NUM_SYMS);
 
-	/* Initialize 'offsets' so that offsets[len] is the number of codewords
-	 * shorter than 'len' bits, including length 0.  */
 	offsets[0] = 0;
-	for (len = 0; len < max_codeword_len; len++)
+	offsets[1] = len_counts[0];
+	codespace_used = 0;
+	for (len = 1; len < max_codeword_len; len++) {
 		offsets[len + 1] = offsets[len] + len_counts[len];
+		codespace_used = (codespace_used << 1) + len_counts[len];
+	}
+	codespace_used = (codespace_used << 1) + len_counts[len];
 
-	/* Use the 'offsets' array to sort the symbols.  */
 	for (sym = 0; sym < num_syms; sym++)
 		sorted_syms[offsets[lens[sym]]++] = sym;
 
-	/* It is already guaranteed that all lengths are <= max_codeword_len,
-	 * but it cannot be assumed they form a complete prefix code.  A
-	 * codeword of length n should require a proportion of the codespace
-	 * equaling (1/2)^n.  The code is complete if and only if, by this
-	 * measure, the codespace is exactly filled by the lengths.  */
-	remainder = 1;
-	for (len = 1; len <= max_codeword_len; len++) {
-		remainder <<= 1;
-		remainder -= len_counts[len];
-		if (unlikely(remainder < 0)) {
-			/* The lengths overflow the codespace; that is, the code
-			 * is over-subscribed.  */
-			return false;
-		}
-	}
+	sorted_syms += offsets[0]; /* Skip unused symbols */
 
-	if (unlikely(remainder != 0)) {
-		/* The lengths do not fill the codespace; that is, they form an
-		 * incomplete code.  */
+	/* lens[] is done being used, so we can write to decode_table[] now. */
 
-		/* Initialize the table entries to default values.  When
-		 * decompressing a well-formed stream, these default values will
-		 * never be used.  But since a malformed stream might contain
-		 * any bits at all, these entries need to be set anyway.  */
-		u32 entry = make_decode_table_entry(decode_results[0], 1);
-		for (sym = 0; sym < (1U << table_bits); sym++)
-			decode_table[sym] = entry;
+	/*
+	 * Check whether the lengths form a complete code (exactly fills the
+	 * codespace), an incomplete code (doesn't fill the codespace), or an
+	 * overfull code (overflows the codespace).  A codeword of length 'n'
+	 * uses proportion '1/(2^n)' of the codespace.  An overfull code is
+	 * nonsensical, so is considered invalid.  An incomplete code is
+	 * considered valid only in two specific cases; see below.
+	 */
 
-		/* A completely empty code is permitted.  */
-		if (remainder == (1U << max_codeword_len))
-			return true;
+	/* overfull code? */
+	if (unlikely(codespace_used > (1U << max_codeword_len)))
+		return false;
 
-		/* The code is nonempty and incomplete.  Proceed only if there
-		 * is a single used symbol and its codeword has length 1.  The
-		 * DEFLATE RFC is somewhat unclear regarding this case.  What
-		 * zlib's decompressor does is permit this case for
-		 * literal/length and offset codes and assume the codeword is 0
-		 * rather than 1.  We do the same except we allow this case for
-		 * precodes too.  */
-		if (remainder != (1U << (max_codeword_len - 1)) ||
-		    len_counts[1] != 1)
-			return false;
-	}
-
-	/* Generate the decode table entries.  Since we process codewords from
-	 * shortest to longest, the main portion of the decode table is filled
-	 * first; then the subtables are filled.  Note that it's already been
-	 * verified that the code is nonempty and not over-subscribed.  */
-
-	/* Start with the smallest codeword length and the smallest-valued
-	 * symbol which has that codeword length.  */
-	sym_idx = offsets[0];
-	codeword_len = 1;
-	while (len_counts[codeword_len] == 0)
-		codeword_len++;
-
-	for (;;) {  /* For each used symbol and its codeword...  */
-		unsigned sym;
+	/* incomplete code? */
+	if (unlikely(codespace_used < (1U << max_codeword_len))) {
 		u32 entry;
 		unsigned i;
-		unsigned end;
-		unsigned increment;
+
+		if (codespace_used == 0) {
+			/*
+			 * An empty code is allowed.  This can happen for the
+			 * offset code in DEFLATE, since a dynamic Huffman block
+			 * need not contain any matches.
+			 */
+
+			/* sym=0, len=1 (arbitrary) */
+			entry = decode_results[0] | 1;
+		} else {
+			/*
+			 * Allow codes with a single used symbol, with codeword
+			 * length 1.  The DEFLATE RFC is unclear regarding this
+			 * case.  What zlib's decompressor does is permit this
+			 * for the litlen and offset codes and assume the
+			 * codeword is '0' rather than '1'.  We do the same
+			 * except we allow this for precodes too, since there's
+			 * no convincing reason to treat the codes differently.
+			 * We also assign both codewords '0' and '1' to the
+			 * symbol to avoid having to handle '1' specially.
+			 */
+			if (codespace_used != (1U << (max_codeword_len - 1)) ||
+			    len_counts[1] != 1)
+				return false;
+			entry = decode_results[*sorted_syms] | 1;
+		}
+		/*
+		 * Note: the decode table still must be fully initialized, in
+		 * case the stream is malformed and contains bits from the part
+		 * of the codespace the incomplete code doesn't use.
+		 */
+		for (i = 0; i < (1U << table_bits); i++)
+			decode_table[i] = entry;
+		return true;
+	}
+
+	/*
+	 * The lengths form a complete code.  Now, enumerate the codewords in
+	 * lexicographic order and fill the decode table entries for each one.
+	 *
+	 * First, process all codewords with len <= table_bits.  Each one gets
+	 * '2^(table_bits-len)' direct entries in the table.
+	 *
+	 * Since DEFLATE uses bit-reversed codewords, these entries aren't
+	 * consecutive but rather are spaced '2^len' entries apart.  This makes
+	 * filling them naively somewhat awkward and inefficient, since strided
+	 * stores are less cache-friendly and preclude the use of word or
+	 * vector-at-a-time stores to fill multiple entries per instruction.
+	 *
+	 * To optimize this, we incrementally double the table size.  When
+	 * processing codewords with length 'len', the table is treated as
+	 * having only '2^len' entries, so each codeword uses just one entry.
+	 * Then, each time 'len' is incremented, the table size is doubled and
+	 * the first half is copied to the second half.  This significantly
+	 * improves performance over naively doing strided stores.
+	 *
+	 * Note that some entries copied for each table doubling may not have
+	 * been initialized yet, but it doesn't matter since they're guaranteed
+	 * to be initialized later (because the Huffman code is complete).
+	 */
+	codeword = 0;
+	len = 1;
+	while ((count = len_counts[len]) == 0)
+		len++;
+	cur_table_end = 1U << len;
+	while (len <= table_bits) {
+		/* Process all 'count' codewords with length 'len' bits. */
+		do {
+			unsigned bit;
+
+			/* Fill the first entry for the current codeword. */
+			decode_table[codeword] =
+				decode_results[*sorted_syms++] | len;
+
+			if (codeword == cur_table_end - 1) {
+				/* Last codeword (all 1's) */
+				for (; len < table_bits; len++) {
+					memcpy(&decode_table[cur_table_end],
+					       decode_table,
+					       cur_table_end *
+						sizeof(decode_table[0]));
+					cur_table_end <<= 1;
+				}
+				return true;
+			}
+			/*
+			 * To advance to the lexicographically next codeword in
+			 * the canonical code, the codeword must be incremented,
+			 * then 0's must be appended to the codeword as needed
+			 * to match the next codeword's length.
+			 *
+			 * Since the codeword is bit-reversed, appending 0's is
+			 * a no-op.  However, incrementing it is nontrivial.  To
+			 * do so efficiently, use the 'bsr' instruction to find
+			 * the last (highest order) 0 bit in the codeword, set
+			 * it, and clear any later (higher order) 1 bits.  But
+			 * 'bsr' actually finds the highest order 1 bit, so to
+			 * use it first flip all bits in the codeword by XOR'ing
+			 * it with (1U << len) - 1 == cur_table_end - 1.
+			 */
+			bit = 1U << bsr32(codeword ^ (cur_table_end - 1));
+			codeword &= bit - 1;
+			codeword |= bit;
+		} while (--count);
+
+		/* Advance to the next codeword length. */
+		do {
+			if (++len <= table_bits) {
+				memcpy(&decode_table[cur_table_end],
+				       decode_table,
+				       cur_table_end * sizeof(decode_table[0]));
+				cur_table_end <<= 1;
+			}
+		} while ((count = len_counts[len]) == 0);
+	}
+
+	/* Process codewords with len > table_bits.  These require subtables. */
+	cur_table_end = 1U << table_bits;
+	subtable_prefix = -1;
+	subtable_start = 0;
+	for (;;) {
+		u32 entry;
+		unsigned i;
+		unsigned stride;
 		unsigned bit;
 
-		/* Get the next symbol.  */
-		sym = sorted_syms[sym_idx];
-
-		/* Start a new subtable if the codeword is long enough to
-		 * require a subtable, *and* the first 'table_bits' bits of the
-		 * codeword don't match the prefix for the previous subtable if
-		 * any.  */
-		if (codeword_len > table_bits &&
-		    (codeword_reversed & table_mask) != cur_codeword_prefix) {
-
-			cur_codeword_prefix = (codeword_reversed & table_mask);
-
-			cur_table_start += 1U << cur_table_bits;
-
-			/* Calculate the subtable length.  If the codeword
-			 * length exceeds 'table_bits' by n, the subtable needs
-			 * at least 2**n entries.  But it may need more; if
-			 * there are fewer than 2**n codewords of length
-			 * 'table_bits + n' remaining, then n will need to be
-			 * incremented to bring in longer codewords until the
-			 * subtable can be filled completely.  Note that it
-			 * always will, eventually, be possible to fill the
-			 * subtable, since the only case where we may have an
-			 * incomplete code is a single codeword of length 1,
-			 * and that never requires any subtables.  */
-			cur_table_bits = codeword_len - table_bits;
-			remainder = (s32)1 << cur_table_bits;
-			for (;;) {
-				remainder -= len_counts[table_bits +
-							cur_table_bits];
-				if (remainder <= 0)
-					break;
-				cur_table_bits++;
-				remainder <<= 1;
+		/*
+		 * Start a new subtable if the first 'table_bits' bits of the
+		 * codeword don't match the prefix of the current subtable.
+		 */
+		if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) {
+			subtable_prefix = (codeword & ((1U << table_bits) - 1));
+			subtable_start = cur_table_end;
+			/*
+			 * Calculate the subtable length.  If the codeword has
+			 * length 'table_bits + n', then the subtable needs
+			 * '2^n' entries.  But it may need more; if fewer than
+			 * '2^n' codewords of length 'table_bits + n' remain,
+			 * then the length will need to be incremented to bring
+			 * in longer codewords until the subtable can be
+			 * completely filled.  Note that because the Huffman
+			 * code is complete, it will always be possible to fill
+			 * the subtable eventually.
+			 */
+			subtable_bits = len - table_bits;
+			codespace_used = count;
+			while (codespace_used < (1U << subtable_bits)) {
+				subtable_bits++;
+				codespace_used = (codespace_used << 1) +
+					len_counts[table_bits + subtable_bits];
 			}
+			cur_table_end = subtable_start + (1U << subtable_bits);
 
-			/* Create the entry that points from the main table to
+			/*
+			 * Create the entry that points from the main table to
 			 * the subtable.  This entry contains the index of the
 			 * start of the subtable and the number of bits with
 			 * which the subtable is indexed (the log base 2 of the
-			 * number of entries it contains).  */
-			decode_table[cur_codeword_prefix] =
+			 * number of entries it contains).
+			 */
+			decode_table[subtable_prefix] =
 				HUFFDEC_SUBTABLE_POINTER |
-				make_decode_table_entry(cur_table_start,
-							cur_table_bits);
-
-			/* Now that we're filling a subtable, we need to drop
-			 * the first 'table_bits' bits of the codewords.  */
-			num_dropped_bits = table_bits;
+				HUFFDEC_RESULT_ENTRY(subtable_start) |
+				subtable_bits;
 		}
 
-		/* Create the decode table entry, which packs the decode result
-		 * and the codeword length (minus 'table_bits' for subtables)
-		 * together.  */
-		entry = make_decode_table_entry(decode_results[sym],
-						codeword_len - num_dropped_bits);
-
-		/* Fill in as many copies of the decode table entry as are
-		 * needed.  The number of entries to fill is a power of 2 and
-		 * depends on the codeword length; it could be as few as 1 or as
-		 * large as half the size of the table.  Since the codewords are
-		 * bit-reversed, the indices to fill are those with the codeword
-		 * in its low bits; it's the high bits that vary.  */
-		i = cur_table_start + (codeword_reversed >> num_dropped_bits);
-		end = cur_table_start + (1U << cur_table_bits);
-		increment = 1U << (codeword_len - num_dropped_bits);
+		/* Fill the subtable entries for the current codeword. */
+		entry = decode_results[*sorted_syms++] | (len - table_bits);
+		i = subtable_start + (codeword >> table_bits);
+		stride = 1U << (len - table_bits);
 		do {
 			decode_table[i] = entry;
-			i += increment;
-		} while (i < end);
+			i += stride;
+		} while (i < cur_table_end);
 
-		/* Advance to the next codeword by incrementing it.  But since
-		 * our codewords are bit-reversed, we must manipulate the bits
-		 * ourselves rather than simply adding 1.  */
-		bit = 1U << (codeword_len - 1);
-		while (codeword_reversed & bit)
-			bit >>= 1;
-		codeword_reversed &= bit - 1;
-		codeword_reversed |= bit;
-
-		/* Advance to the next symbol.  This will either increase the
-		 * codeword length, or keep the same codeword length but
-		 * increase the symbol value.  Note: since we are using
-		 * bit-reversed codewords, we don't need to explicitly append
-		 * zeroes to the codeword when the codeword length increases. */
-		if (++sym_idx == num_syms)
+		/* Advance to the next codeword. */
+		if (codeword == (1U << len) - 1) /* last codeword (all 1's)? */
 			return true;
-		len_counts[codeword_len]--;
-		while (len_counts[codeword_len] == 0)
-			codeword_len++;
+		bit = 1U << bsr32(codeword ^ ((1U << len) - 1));
+		codeword &= bit - 1;
+		codeword |= bit;
+		count--;
+		while (count == 0)
+			count = len_counts[++len];
 	}
 }
 
@@ -740,7 +824,7 @@ build_precode_decode_table(struct libdeflate_decompressor *d)
 				  precode_decode_results,
 				  PRECODE_TABLEBITS,
 				  DEFLATE_MAX_PRE_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 /* Build the decode table for the literal/length code.  */
@@ -757,7 +841,7 @@ build_litlen_decode_table(struct libdeflate_decompressor *d,
 				  litlen_decode_results,
 				  LITLEN_TABLEBITS,
 				  DEFLATE_MAX_LITLEN_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 /* Build the decode table for the offset code.  */
@@ -774,7 +858,7 @@ build_offset_decode_table(struct libdeflate_decompressor *d,
 				  offset_decode_results,
 				  OFFSET_TABLEBITS,
 				  DEFLATE_MAX_OFFSET_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 static forceinline machine_word_t
@@ -858,7 +942,7 @@ dispatch(struct libdeflate_decompressor * restrict d,
  * handles calling the appropriate implementation depending on the CPU features
  * at runtime.
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor * restrict d,
 				 const void * restrict in, size_t in_nbytes,
 				 void * restrict out, size_t out_nbytes_avail,
@@ -869,7 +953,7 @@ libdeflate_deflate_decompress_ex(struct libdeflate_decompressor * restrict d,
 			       actual_in_nbytes_ret, actual_out_nbytes_ret);
 }
 
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_deflate_decompress(struct libdeflate_decompressor * restrict d,
 			      const void * restrict in, size_t in_nbytes,
 			      void * restrict out, size_t out_nbytes_avail,
@@ -880,14 +964,34 @@ libdeflate_deflate_decompress(struct libdeflate_decompressor * restrict d,
 						NULL, actual_out_nbytes_ret);
 }
 
-LIBDEFLATEAPI struct libdeflate_decompressor *
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
 libdeflate_alloc_decompressor(void)
 {
-	return malloc(sizeof(struct libdeflate_decompressor));
+	/*
+	 * Note that only certain parts of the decompressor actually must be
+	 * initialized here:
+	 *
+	 * - 'static_codes_loaded' must be initialized to false.
+	 *
+	 * - The first half of the main portion of each decode table must be
+	 *   initialized to any value, to avoid reading from uninitialized
+	 *   memory during table expansion in build_decode_table().  (Although,
+	 *   this is really just to avoid warnings with dynamic tools like
+	 *   valgrind, since build_decode_table() is guaranteed to initialize
+	 *   all entries eventually anyway.)
+	 *
+	 * But for simplicity, we currently just zero the whole decompressor.
+	 */
+	struct libdeflate_decompressor *d = libdeflate_malloc(sizeof(*d));
+
+	if (d == NULL)
+		return NULL;
+	memset(d, 0, sizeof(*d));
+	return d;
 }
 
-LIBDEFLATEAPI void
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
 libdeflate_free_decompressor(struct libdeflate_decompressor *d)
 {
-	free(d);
+	libdeflate_free(d);
 }

lib/gzip_compress.c

@@ -1,8 +1,6 @@
 /*
  * gzip_compress.c - compress with a gzip wrapper
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -33,9 +31,9 @@
 
 #include "libdeflate.h"
 
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_gzip_compress(struct libdeflate_compressor *c,
-			 const void *in, size_t in_size,
+			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
@@ -61,32 +59,32 @@ libdeflate_gzip_compress(struct libdeflate_compressor *c,
 	xfl = 0;
 	compression_level = deflate_get_compression_level(c);
 	if (compression_level < 2)
-		xfl |= GZIP_XFL_FASTEST_COMRESSION;
+		xfl |= GZIP_XFL_FASTEST_COMPRESSION;
 	else if (compression_level >= 8)
-		xfl |= GZIP_XFL_SLOWEST_COMRESSION;
+		xfl |= GZIP_XFL_SLOWEST_COMPRESSION;
 	*out_next++ = xfl;
 	/* OS */
 	*out_next++ = GZIP_OS_UNKNOWN;	/* OS  */
 
 	/* Compressed data  */
-	deflate_size = libdeflate_deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - GZIP_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* CRC32 */
-	put_unaligned_le32(libdeflate_crc32(0, in, in_size), out_next);
+	put_unaligned_le32(libdeflate_crc32(0, in, in_nbytes), out_next);
 	out_next += 4;
 
 	/* ISIZE */
-	put_unaligned_le32((u32)in_size, out_next);
+	put_unaligned_le32((u32)in_nbytes, out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_gzip_compress_bound(struct libdeflate_compressor *c,
 			       size_t in_nbytes)
 {

lib/gzip_constants.h

@@ -23,8 +23,8 @@
 
 #define GZIP_MTIME_UNAVAILABLE	0
 
-#define GZIP_XFL_SLOWEST_COMRESSION	0x02
-#define GZIP_XFL_FASTEST_COMRESSION	0x04
+#define GZIP_XFL_SLOWEST_COMPRESSION	0x02
+#define GZIP_XFL_FASTEST_COMPRESSION	0x04
 
 #define GZIP_OS_FAT		0
 #define GZIP_OS_AMIGA		1

lib/gzip_decompress.c

@@ -1,8 +1,6 @@
 /*
  * gzip_decompress.c - decompress with a gzip wrapper
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -32,7 +30,7 @@
 
 #include "libdeflate.h"
 
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -136,7 +134,7 @@ libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
 	return LIBDEFLATE_SUCCESS;
 }
 
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_gzip_decompress(struct libdeflate_decompressor *d,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,

lib/hc_matchfinder.h

@@ -1,8 +1,6 @@
 /*
  * hc_matchfinder.h - Lempel-Ziv matchfinding with a hash table of linked lists
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -75,7 +73,7 @@
  * chain for length 3+ matches, the algorithm just checks for one close length 3
  * match, then focuses on finding length 4+ matches.
  *
- * The longest_match() and skip_positions() functions are inlined into the
+ * The longest_match() and skip_bytes() functions are inlined into the
  * compressors that use them.  This isn't just about saving the overhead of a
  * function call.  These functions are intended to be called from the inner
  * loops of compressors, where giving the compiler more control over register
@@ -106,14 +104,17 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_HC_MATCHFINDER_H
+#define LIB_HC_MATCHFINDER_H
+
 #include "matchfinder_common.h"
 
 #define HC_MATCHFINDER_HASH3_ORDER	15
 #define HC_MATCHFINDER_HASH4_ORDER	16
 
-#define HC_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
-	 (1UL << HC_MATCHFINDER_HASH4_ORDER))
+#define HC_MATCHFINDER_TOTAL_HASH_SIZE			\
+	(((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
+	  (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 struct hc_matchfinder {
 
@@ -128,24 +129,24 @@ struct hc_matchfinder {
 	 * the node for the sequence with position 'pos' is 'next_tab[pos]'.  */
 	mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE];
 
-}
-#ifdef _aligned_attribute
-  _aligned_attribute(MATCHFINDER_ALIGNMENT)
-#endif
-;
+} MATCHFINDER_ALIGNED;
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void
 hc_matchfinder_init(struct hc_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 hc_matchfinder_slide_window(struct hc_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct hc_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 /*
@@ -157,9 +158,9 @@ hc_matchfinder_slide_window(struct hc_matchfinder *mf)
  *	Location of a pointer which points to the place in the input data the
  *	matchfinder currently stores positions relative to.  This may be updated
  *	by this function.
- * @cur_pos
- *	The current position in the input buffer relative to @in_base (the
- *	position of the sequence being matched against).
+ * @in_next
+ *	Pointer to the next position in the input buffer, i.e. the sequence
+ *	being matched against.
  * @best_len
  *	Require a match longer than this length.
  * @max_len
@@ -346,26 +347,24 @@ out:
  *	Location of a pointer which points to the place in the input data the
  *	matchfinder currently stores positions relative to.  This may be updated
  *	by this function.
- * @cur_pos
- *	The current position in the input buffer relative to @in_base.
- * @end_pos
- *	The end position of the input buffer, relative to @in_base.
+ * @in_next
+ *	Pointer to the next position in the input buffer.
+ * @in_end
+ *	Pointer to the end of the input buffer.
+ * @count
+ *	The number of bytes to advance.  Must be > 0.
  * @next_hashes
  *	The precomputed hash codes for the sequence beginning at @in_next.
  *	These will be used and then updated with the precomputed hashcodes for
  *	the sequence beginning at @in_next + @count.
- * @count
- *	The number of bytes to advance.  Must be > 0.
- *
- * Returns @in_next + @count.
  */
-static forceinline const u8 *
-hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
-			      const u8 ** const restrict in_base_p,
-			      const u8 *in_next,
-			      const u8 * const in_end,
-			      const u32 count,
-			      u32 * const restrict next_hashes)
+static forceinline void
+hc_matchfinder_skip_bytes(struct hc_matchfinder * const restrict mf,
+			  const u8 ** const restrict in_base_p,
+			  const u8 *in_next,
+			  const u8 * const in_end,
+			  const u32 count,
+			  u32 * const restrict next_hashes)
 {
 	u32 cur_pos;
 	u32 hash3, hash4;
@@ -373,7 +372,7 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 	u32 remaining = count;
 
 	if (unlikely(count + 5 > in_end - in_next))
-		return &in_next[count];
+		return;
 
 	cur_pos = in_next - *in_base_p;
 	hash3 = next_hashes[0];
@@ -398,6 +397,6 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 	prefetchw(&mf->hash4_tab[hash4]);
 	next_hashes[0] = hash3;
 	next_hashes[1] = hash4;
-
-	return in_next;
 }
+
+#endif /* LIB_HC_MATCHFINDER_H */

lib/ht_matchfinder.h

@@ -0,0 +1,234 @@
+/*
+ * ht_matchfinder.h - Lempel-Ziv matchfinding with a hash table
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ---------------------------------------------------------------------------
+ *
+ * This is a Hash Table (ht) matchfinder.
+ *
+ * This is a variant of the Hash Chains (hc) matchfinder that is optimized for
+ * very fast compression.  The ht_matchfinder stores the hash chains inline in
+ * the hash table, whereas the hc_matchfinder stores them in a separate array.
+ * Storing the hash chains inline is the faster method when max_search_depth
+ * (the maximum chain length) is very small.  It is not appropriate when
+ * max_search_depth is larger, as then it uses too much memory.
+ *
+ * Due to its focus on speed, the ht_matchfinder doesn't support length 3
+ * matches.  It also doesn't allow max_search_depth to vary at runtime; it is
+ * fixed at build time as HT_MATCHFINDER_BUCKET_SIZE.
+ *
+ * See hc_matchfinder.h for more information.
+ */
+
+#ifndef LIB_HT_MATCHFINDER_H
+#define LIB_HT_MATCHFINDER_H
+
+#include "matchfinder_common.h"
+
+#define HT_MATCHFINDER_HASH_ORDER	15
+#define HT_MATCHFINDER_BUCKET_SIZE	2
+
+#define HT_MATCHFINDER_MIN_MATCH_LEN	4
+/* Minimum value of max_len for ht_matchfinder_longest_match() */
+#define HT_MATCHFINDER_REQUIRED_NBYTES	5
+
+struct ht_matchfinder {
+	mf_pos_t hash_tab[1UL << HT_MATCHFINDER_HASH_ORDER]
+			 [HT_MATCHFINDER_BUCKET_SIZE];
+} MATCHFINDER_ALIGNED;
+
+static forceinline void
+ht_matchfinder_init(struct ht_matchfinder *mf)
+{
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, sizeof(*mf));
+}
+
+static forceinline void
+ht_matchfinder_slide_window(struct ht_matchfinder *mf)
+{
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
+}
+
+/* Note: max_len must be >= HT_MATCHFINDER_REQUIRED_NBYTES */
+static forceinline u32
+ht_matchfinder_longest_match(struct ht_matchfinder * const restrict mf,
+			     const u8 ** const restrict in_base_p,
+			     const u8 * const restrict in_next,
+			     const u32 max_len,
+			     const u32 nice_len,
+			     u32 * const restrict next_hash,
+			     u32 * const restrict offset_ret)
+{
+	u32 best_len = 0;
+	const u8 *best_matchptr = in_next;
+	u32 cur_pos = in_next - *in_base_p;
+	const u8 *in_base;
+	mf_pos_t cutoff;
+	u32 hash;
+	u32 seq;
+	mf_pos_t cur_node;
+	const u8 *matchptr;
+#if HT_MATCHFINDER_BUCKET_SIZE > 1
+	mf_pos_t to_insert;
+	u32 len;
+#endif
+#if HT_MATCHFINDER_BUCKET_SIZE > 2
+	int i;
+#endif
+
+	/* This is assumed throughout this function. */
+	STATIC_ASSERT(HT_MATCHFINDER_MIN_MATCH_LEN == 4);
+
+	if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
+		ht_matchfinder_slide_window(mf);
+		*in_base_p += MATCHFINDER_WINDOW_SIZE;
+		cur_pos = 0;
+	}
+	in_base = *in_base_p;
+	cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
+
+	hash = *next_hash;
+	STATIC_ASSERT(HT_MATCHFINDER_REQUIRED_NBYTES == 5);
+	*next_hash = lz_hash(get_unaligned_le32(in_next + 1),
+			     HT_MATCHFINDER_HASH_ORDER);
+	seq = load_u32_unaligned(in_next);
+	prefetchw(&mf->hash_tab[*next_hash]);
+#if HT_MATCHFINDER_BUCKET_SIZE == 1
+	/* Hand-unrolled version for BUCKET_SIZE == 1 */
+	cur_node = mf->hash_tab[hash][0];
+	mf->hash_tab[hash][0] = cur_pos;
+	if (cur_node <= cutoff)
+		goto out;
+	matchptr = &in_base[cur_node];
+	if (load_u32_unaligned(matchptr) == seq) {
+		best_len = lz_extend(in_next, matchptr, 4, max_len);
+		best_matchptr = matchptr;
+	}
+#elif HT_MATCHFINDER_BUCKET_SIZE == 2
+	/*
+	 * Hand-unrolled version for BUCKET_SIZE == 2.  The logic here also
+	 * differs slightly in that it copies the first entry to the second even
+	 * if nice_len is reached on the first, as this can be slightly faster.
+	 */
+	cur_node = mf->hash_tab[hash][0];
+	mf->hash_tab[hash][0] = cur_pos;
+	if (cur_node <= cutoff)
+		goto out;
+	matchptr = &in_base[cur_node];
+
+	to_insert = cur_node;
+	cur_node = mf->hash_tab[hash][1];
+	mf->hash_tab[hash][1] = to_insert;
+
+	if (load_u32_unaligned(matchptr) == seq) {
+		best_len = lz_extend(in_next, matchptr, 4, max_len);
+		best_matchptr = matchptr;
+		if (cur_node <= cutoff || best_len >= nice_len)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq &&
+		    load_u32_unaligned(matchptr + best_len - 3) ==
+		    load_u32_unaligned(in_next + best_len - 3)) {
+			len = lz_extend(in_next, matchptr, 4, max_len);
+			if (len > best_len) {
+				best_len = len;
+				best_matchptr = matchptr;
+			}
+		}
+	} else {
+		if (cur_node <= cutoff)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq) {
+			best_len = lz_extend(in_next, matchptr, 4, max_len);
+			best_matchptr = matchptr;
+		}
+	}
+#else
+	/* Generic version for HT_MATCHFINDER_BUCKET_SIZE > 2 */
+	to_insert = cur_pos;
+	for (i = 0; i < HT_MATCHFINDER_BUCKET_SIZE; i++) {
+		cur_node = mf->hash_tab[hash][i];
+		mf->hash_tab[hash][i] = to_insert;
+		if (cur_node <= cutoff)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq) {
+			len = lz_extend(in_next, matchptr, 4, max_len);
+			if (len > best_len) {
+				best_len = len;
+				best_matchptr = matchptr;
+				if (best_len >= nice_len)
+					goto out;
+			}
+		}
+		to_insert = cur_node;
+	}
+#endif
+out:
+	*offset_ret = in_next - best_matchptr;
+	return best_len;
+}
+
+static forceinline void
+ht_matchfinder_skip_bytes(struct ht_matchfinder * const restrict mf,
+			  const u8 ** const restrict in_base_p,
+			  const u8 *in_next,
+			  const u8 * const in_end,
+			  const u32 count,
+			  u32 * const restrict next_hash)
+{
+	s32 cur_pos = in_next - *in_base_p;
+	u32 hash;
+	u32 remaining = count;
+	int i;
+
+	if (unlikely(count + HT_MATCHFINDER_REQUIRED_NBYTES > in_end - in_next))
+		return;
+
+	if (cur_pos + count - 1 >= MATCHFINDER_WINDOW_SIZE) {
+		ht_matchfinder_slide_window(mf);
+		*in_base_p += MATCHFINDER_WINDOW_SIZE;
+		cur_pos -= MATCHFINDER_WINDOW_SIZE;
+	}
+
+	hash = *next_hash;
+	do {
+		for (i = HT_MATCHFINDER_BUCKET_SIZE - 1; i > 0; i--)
+			mf->hash_tab[hash][i] = mf->hash_tab[hash][i - 1];
+		mf->hash_tab[hash][0] = cur_pos;
+
+		hash = lz_hash(get_unaligned_le32(++in_next),
+			       HT_MATCHFINDER_HASH_ORDER);
+		cur_pos++;
+	} while (--remaining);
+
+	prefetchw(&mf->hash_tab[hash]);
+	*next_hash = hash;
+}
+
+#endif /* LIB_HT_MATCHFINDER_H */

lib/lib_common.h

@@ -12,12 +12,12 @@
 
 #define BUILDING_LIBDEFLATE
 
-#include "common_defs.h"
+#include "../common/common_defs.h"
 
 /*
- * Prefix with "_libdeflate_" all global symbols which are not part of the API.
- * This avoids exposing overly generic names when libdeflate is built as a
- * static library.
+ * Prefix with "_libdeflate_" all global symbols which are not part of the API
+ * and don't already have a "libdeflate" prefix.  This avoids exposing overly
+ * generic names when libdeflate is built as a static library.
  *
  * Note that the chosen prefix is not really important and can be changed
  * without breaking library users.  It was just chosen so that the resulting
@@ -26,10 +26,42 @@
  * shared library, since these symbols are not exported.
  */
 #define SYM_FIXUP(sym)			_libdeflate_##sym
-#define aligned_malloc			SYM_FIXUP(aligned_malloc)
-#define aligned_free			SYM_FIXUP(aligned_free)
 #define deflate_get_compression_level	SYM_FIXUP(deflate_get_compression_level)
 #define _cpu_features			SYM_FIXUP(_cpu_features)
 #define setup_cpu_features		SYM_FIXUP(setup_cpu_features)
 
+void *libdeflate_malloc(size_t size);
+void libdeflate_free(void *ptr);
+
+void *libdeflate_aligned_malloc(size_t alignment, size_t size);
+void libdeflate_aligned_free(void *ptr);
+
+#ifdef FREESTANDING
+/*
+ * With -ffreestanding, <string.h> may be missing, and we must provide
+ * implementations of memset(), memcpy(), memmove(), and memcmp().
+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+ *
+ * Also, -ffreestanding disables interpreting calls to these functions as
+ * built-ins.  E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
+ * not be optimized to a single load instruction.  For performance reasons we
+ * don't want that.  So, declare these functions as macros that expand to the
+ * corresponding built-ins.  This approach is recommended in the gcc man page.
+ * We still need the actual function definitions in case gcc calls them.
+ */
+void *memset(void *s, int c, size_t n);
+#define memset(s, c, n)		__builtin_memset((s), (c), (n))
+
+void *memcpy(void *dest, const void *src, size_t n);
+#define memcpy(dest, src, n)	__builtin_memcpy((dest), (src), (n))
+
+void *memmove(void *dest, const void *src, size_t n);
+#define memmove(dest, src, n)	__builtin_memmove((dest), (src), (n))
+
+int memcmp(const void *s1, const void *s2, size_t n);
+#define memcmp(s1, s2, n)	__builtin_memcmp((s1), (s2), (n))
+#else
+#include <string.h>
+#endif
+
 #endif /* LIB_LIB_COMMON_H */

lib/matchfinder_common.h

@@ -18,17 +18,24 @@ typedef s16 mf_pos_t;
 
 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
 
-#define MATCHFINDER_ALIGNMENT 8
-
-#define arch_matchfinder_init(data, size)	false
-#define arch_matchfinder_rebase(data, size)	false
+/*
+ * Required alignment of the matchfinder buffer pointer and size.  The values
+ * here come from the AVX-2 implementation, which is the worst case.
+ */
+#define MATCHFINDER_MEM_ALIGNMENT	32
+#define MATCHFINDER_SIZE_ALIGNMENT	128
 
+#undef matchfinder_init
+#undef matchfinder_rebase
 #ifdef _aligned_attribute
+#  define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
 #  if defined(__arm__) || defined(__aarch64__)
 #    include "arm/matchfinder_impl.h"
 #  elif defined(__i386__) || defined(__x86_64__)
 #    include "x86/matchfinder_impl.h"
 #  endif
+#else
+#  define MATCHFINDER_ALIGNED
 #endif
 
 /*
@@ -36,19 +43,20 @@ typedef s16 mf_pos_t;
  *
  * Essentially, this is an optimized memset().
  *
- * 'data' must be aligned to a MATCHFINDER_ALIGNMENT boundary.
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_init
 static forceinline void
-matchfinder_init(mf_pos_t *data, size_t num_entries)
+matchfinder_init(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-	if (arch_matchfinder_init(data, num_entries * sizeof(data[0])))
-		return;
-
 	for (i = 0; i < num_entries; i++)
 		data[i] = MATCHFINDER_INITVAL;
 }
+#endif
 
 /*
  * Slide the matchfinder by WINDOW_SIZE bytes.
@@ -70,15 +78,17 @@ matchfinder_init(mf_pos_t *data, size_t num_entries)
  * hash bucket.  Note that in the latter table, the links are 1-ary in the case
  * of "hash chains", and 2-ary in the case of "binary trees".  In either case,
  * the links need to be rebased in the same way.
+ *
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_rebase
 static forceinline void
-matchfinder_rebase(mf_pos_t *data, size_t num_entries)
+matchfinder_rebase(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-	if (arch_matchfinder_rebase(data, num_entries * sizeof(data[0])))
-		return;
-
 	if (MATCHFINDER_WINDOW_SIZE == 32768) {
 		/* Branchless version for 32768 byte windows.  If the value was
 		 * already negative, clear all bits except the sign bit; this
@@ -101,6 +111,7 @@ matchfinder_rebase(mf_pos_t *data, size_t num_entries)
 			data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
 	}
 }
+#endif
 
 /*
  * The hash function: given a sequence prefix held in the low-order bits of a

lib/unaligned.h

@@ -7,13 +7,39 @@
 
 #include "lib_common.h"
 
+/***** Unaligned loads and stores without endianness conversion *****/
+
 /*
- * Naming note:
+ * memcpy() is portable, and it usually gets optimized appropriately by modern
+ * compilers.  I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled
+ * to a load or store instruction, not to an actual function call.
  *
- * {load,store}_*_unaligned() deal with raw bytes without endianness conversion.
- * {get,put}_unaligned_*() deal with a specific endianness.
+ * We no longer use the "packed struct" approach, as that is nonstandard, has
+ * unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366).  However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
  */
 
+#define DEFINE_UNALIGNED_TYPE(type)				\
+static forceinline type						\
+load_##type##_unaligned(const void *p)				\
+{								\
+	type v;							\
+	memcpy(&v, p, sizeof(v));				\
+	return v;						\
+}								\
+								\
+static forceinline void						\
+store_##type##_unaligned(type v, void *p)			\
+{								\
+	memcpy(p, &v, sizeof(v));				\
+}
+
 DEFINE_UNALIGNED_TYPE(u16)
 DEFINE_UNALIGNED_TYPE(u32)
 DEFINE_UNALIGNED_TYPE(u64)
@@ -22,7 +48,7 @@ DEFINE_UNALIGNED_TYPE(machine_word_t)
 #define load_word_unaligned	load_machine_word_t_unaligned
 #define store_word_unaligned	store_machine_word_t_unaligned
 
-/***** Unaligned loads  *****/
+/***** Unaligned loads with endianness conversion *****/
 
 static forceinline u16
 get_unaligned_le16(const u8 *p)
@@ -84,7 +110,7 @@ get_unaligned_leword(const u8 *p)
 		return get_unaligned_le64(p);
 }
 
-/***** Unaligned stores  *****/
+/***** Unaligned stores with endianness conversion *****/
 
 static forceinline void
 put_unaligned_le16(u16 v, u8 *p)

lib/utils.c

@@ -0,0 +1,142 @@
+/*
+ * utils.c - utility functions for libdeflate
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+#include "libdeflate.h"
+
+#ifdef FREESTANDING
+#  define malloc NULL
+#  define free NULL
+#else
+#  include <stdlib.h>
+#endif
+
+static void *(*libdeflate_malloc_func)(size_t) = malloc;
+static void (*libdeflate_free_func)(void *) = free;
+
+void *
+libdeflate_malloc(size_t size)
+{
+	return (*libdeflate_malloc_func)(size);
+}
+
+void
+libdeflate_free(void *ptr)
+{
+	(*libdeflate_free_func)(ptr);
+}
+
+void *
+libdeflate_aligned_malloc(size_t alignment, size_t size)
+{
+	void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size);
+	if (ptr) {
+		void *orig_ptr = ptr;
+		ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
+		((void **)ptr)[-1] = orig_ptr;
+	}
+	return ptr;
+}
+
+void
+libdeflate_aligned_free(void *ptr)
+{
+	if (ptr)
+		libdeflate_free(((void **)ptr)[-1]);
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *))
+{
+	libdeflate_malloc_func = malloc_func;
+	libdeflate_free_func = free_func;
+}
+
+/*
+ * Implementations of libc functions for freestanding library builds.
+ * Normal library builds don't use these.  Not optimized yet; usually the
+ * compiler expands these functions and doesn't actually call them anyway.
+ */
+#ifdef FREESTANDING
+#undef memset
+void * __attribute__((weak))
+memset(void *s, int c, size_t n)
+{
+	u8 *p = s;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		p[i] = c;
+	return s;
+}
+
+#undef memcpy
+void * __attribute__((weak))
+memcpy(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		d[i] = s[i];
+	return dest;
+}
+
+#undef memmove
+void * __attribute__((weak))
+memmove(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	if (d <= s)
+		return memcpy(d, s, n);
+
+	for (i = n; i > 0; i--)
+		d[i - 1] = s[i - 1];
+	return dest;
+}
+
+#undef memcmp
+int __attribute__((weak))
+memcmp(const void *s1, const void *s2, size_t n)
+{
+	const u8 *p1 = s1;
+	const u8 *p2 = s2;
+	size_t i;
+
+	for (i = 0; i < n; i++) {
+		if (p1[i] != p2[i])
+			return (int)p1[i] - (int)p2[i];
+	}
+	return 0;
+}
+#endif /* FREESTANDING */

lib/x86/adler32_impl.h

@@ -25,9 +25,137 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_X86_ADLER32_IMPL_H
+#define LIB_X86_ADLER32_IMPL_H
+
 #include "cpu_features.h"
 
-/* AVX2 implementation */
+/*
+ * The following macros horizontally sum the s1 counters and add them to the
+ * real s1, and likewise for s2.  They do this via a series of reductions, each
+ * of which halves the vector length, until just one counter remains.
+ *
+ * The s1 reductions don't depend on the s2 reductions and vice versa, so for
+ * efficiency they are interleaved.  Also, every other s1 counter is 0 due to
+ * the 'psadbw' instruction (_mm_sad_epu8) summing groups of 8 bytes rather than
+ * 4; hence, one of the s1 reductions is skipped when going from 128 => 32 bits.
+ */
+
+#define ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_last = (v_s1), s2_last = (v_s2);			    \
+									    \
+	/* 128 => 32 bits */						    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x31);	    \
+	s1_last += (__v4su)_mm_shuffle_epi32((__m128i)s1_last, 0x02);	    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x02);	    \
+									    \
+	*(s1) += (u32)_mm_cvtsi128_si32((__m128i)s1_last);		    \
+	*(s2) += (u32)_mm_cvtsi128_si32((__m128i)s2_last);		    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_128bit, s2_128bit;					    \
+									    \
+	/* 256 => 128 bits */						    \
+	s1_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 1);   \
+	s2_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 1);   \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_128((s1), (s2), s1_128bit, s2_128bit);	    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v8su s1_256bit, s2_256bit;					    \
+									    \
+	/* 512 => 256 bits */						    \
+	s1_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 1);  \
+	s2_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 1);  \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_256((s1), (s2), s1_256bit, s2_256bit);	    \
+}
+
+/* AVX-512BW implementation: like the AVX2 one, but does 64 bytes at a time */
+#undef DISPATCH_AVX512BW
+#if !defined(DEFAULT_IMPL) &&						\
+    /*
+     * clang before v3.9 is missing some AVX-512BW intrinsics including
+     * _mm512_sad_epu8(), a.k.a. __builtin_ia32_psadbw512.  So just make using
+     * AVX-512BW, even when __AVX512BW__ is defined, conditional on
+     * COMPILER_SUPPORTS_AVX512BW_TARGET where we check for that builtin.
+     */									\
+    COMPILER_SUPPORTS_AVX512BW_TARGET &&				\
+    (defined(__AVX512BW__) || (X86_CPU_FEATURES_ENABLED &&		\
+			       COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_avx512bw
+#  define FUNCNAME_CHUNK	adler32_avx512bw_chunk
+#  define IMPL_ALIGNMENT	64
+#  define IMPL_SEGMENT_SIZE	64
+#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
+#  ifdef __AVX512BW__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_avx512bw
+#  else
+#    define ATTRIBUTES		__attribute__((target("avx512bw")))
+#    define DISPATCH		1
+#    define DISPATCH_AVX512BW	1
+#  endif
+#  include <immintrin.h>
+static forceinline ATTRIBUTES void
+adler32_avx512bw_chunk(const __m512i *p, const __m512i *const end,
+		       u32 *s1, u32 *s2)
+{
+	const __m512i zeroes = _mm512_setzero_si512();
+	const __v64qi multipliers = (__v64qi){
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v32hi ones = (__v32hi)_mm512_set1_epi16(1);
+	__v16si v_s1 = (__v16si)zeroes;
+	__v16si v_s1_sums = (__v16si)zeroes;
+	__v16si v_s2 = (__v16si)zeroes;
+
+	do {
+		/* Load the next 64-byte segment */
+		__m512i bytes = *p++;
+
+		/* Multiply the bytes by 64...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v32hi sums = (__v32hi)_mm512_maddubs_epi16(
+						bytes, (__m512i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 64 to the end. */
+		v_s1_sums += v_s1;
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v16si)_mm512_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 64...1 to the corresponding s2 counter */
+		v_s2 += (__v16si)_mm512_madd_epi16((__m512i)sums,
+						   (__m512i)ones);
+	} while (p != end);
+
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (64) */
+	v_s2 += (__v16si)_mm512_slli_epi32((__m512i)v_s1_sums, 6);
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* AVX-512BW implementation */
+
+/* AVX2 implementation: like the AVX-512BW one, but does 32 bytes at a time */
 #undef DISPATCH_AVX2
 #if !defined(DEFAULT_IMPL) &&	\
 	(defined(__AVX2__) || (X86_CPU_FEATURES_ENABLED &&	\
@@ -50,32 +178,43 @@ static forceinline ATTRIBUTES void
 adler32_avx2_chunk(const __m256i *p, const __m256i *const end, u32 *s1, u32 *s2)
 {
 	const __m256i zeroes = _mm256_setzero_si256();
-	const __v32qi multipliers = (__v32qi) { 32, 31, 30, 29, 28, 27, 26, 25,
-						24, 23, 22, 21, 20, 19, 18, 17,
-						16, 15, 14, 13, 12, 11, 10, 9,
-						8,  7,  6,  5,  4,  3,  2,  1 };
-	const __v16hi ones = (__v16hi)_mm256_set1_epi16(1);
-	__v8si v_s1 = (__v8si)zeroes;
-	__v8si v_s1_sums = (__v8si)zeroes;
-	__v8si v_s2 = (__v8si)zeroes;
+	const __v32qu multipliers = (__v32qu){
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v16hu ones = (__v16hu)_mm256_set1_epi16(1);
+	__v8su v_s1 = (__v8su)zeroes;
+	__v8su v_s1_sums = (__v8su)zeroes;
+	__v8su v_s2 = (__v8su)zeroes;
 
 	do {
+		/* Load the next 32-byte segment */
 		__m256i bytes = *p++;
-		__v16hi sums = (__v16hi)_mm256_maddubs_epi16(
+
+		/* Multiply the bytes by 32...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v16hu sums = (__v16hu)_mm256_maddubs_epi16(
 						bytes, (__m256i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 32 to the end. */
 		v_s1_sums += v_s1;
-		v_s1 += (__v8si)_mm256_sad_epu8(bytes, zeroes);
-		v_s2 += (__v8si)_mm256_madd_epi16((__m256i)sums, (__m256i)ones);
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v8su)_mm256_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 32...1 to the corresponding s2 counter */
+		v_s2 += (__v8su)_mm256_madd_epi16((__m256i)sums, (__m256i)ones);
 	} while (p != end);
 
-	v_s1 = (__v8si)_mm256_hadd_epi32((__m256i)v_s1, zeroes);
-	v_s1 = (__v8si)_mm256_hadd_epi32((__m256i)v_s1, zeroes);
-	*s1 += (u32)v_s1[0] + (u32)v_s1[4];
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (32) */
+	v_s2 += (__v8su)_mm256_slli_epi32((__m256i)v_s1_sums, 5);
 
-	v_s2 += (__v8si)_mm256_slli_epi32((__m256i)v_s1_sums, 5);
-	v_s2 = (__v8si)_mm256_hadd_epi32((__m256i)v_s2, zeroes);
-	v_s2 = (__v8si)_mm256_hadd_epi32((__m256i)v_s2, zeroes);
-	*s2 += (u32)v_s2[0] + (u32)v_s2[4];
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2);
 }
 #  include "../adler32_vec_template.h"
 #endif /* AVX2 implementation */
@@ -110,20 +249,20 @@ adler32_sse2_chunk(const __m128i *p, const __m128i *const end, u32 *s1, u32 *s2)
 	const __m128i zeroes = _mm_setzero_si128();
 
 	/* s1 counters: 32-bit, sum of bytes */
-	__v4si v_s1 = (__v4si)zeroes;
+	__v4su v_s1 = (__v4su)zeroes;
 
 	/* s2 counters: 32-bit, sum of s1 values */
-	__v4si v_s2 = (__v4si)zeroes;
+	__v4su v_s2 = (__v4su)zeroes;
 
 	/*
 	 * Thirty-two 16-bit counters for byte sums.  Each accumulates the bytes
 	 * that eventually need to be multiplied by a number 32...1 for addition
 	 * into s2.
 	 */
-	__v8hi v_byte_sums_a = (__v8hi)zeroes;
-	__v8hi v_byte_sums_b = (__v8hi)zeroes;
-	__v8hi v_byte_sums_c = (__v8hi)zeroes;
-	__v8hi v_byte_sums_d = (__v8hi)zeroes;
+	__v8hu v_byte_sums_a = (__v8hu)zeroes;
+	__v8hu v_byte_sums_b = (__v8hu)zeroes;
+	__v8hu v_byte_sums_c = (__v8hu)zeroes;
+	__v8hu v_byte_sums_d = (__v8hu)zeroes;
 
 	do {
 		/* Load the next 32 bytes */
@@ -142,39 +281,33 @@ adler32_sse2_chunk(const __m128i *p, const __m128i *const end, u32 *s1, u32 *s2)
 		 * the bytes horizontally with 8 bytes per sum.  Then add the
 		 * sums to the s1 counters.
 		 */
-		v_s1 += (__v4si)_mm_sad_epu8(bytes1, zeroes);
-		v_s1 += (__v4si)_mm_sad_epu8(bytes2, zeroes);
+		v_s1 += (__v4su)_mm_sad_epu8(bytes1, zeroes);
+		v_s1 += (__v4su)_mm_sad_epu8(bytes2, zeroes);
 
 		/*
 		 * Also accumulate the bytes into 32 separate counters that have
 		 * 16-bit precision.
 		 */
-		v_byte_sums_a += (__v8hi)_mm_unpacklo_epi8(bytes1, zeroes);
-		v_byte_sums_b += (__v8hi)_mm_unpackhi_epi8(bytes1, zeroes);
-		v_byte_sums_c += (__v8hi)_mm_unpacklo_epi8(bytes2, zeroes);
-		v_byte_sums_d += (__v8hi)_mm_unpackhi_epi8(bytes2, zeroes);
+		v_byte_sums_a += (__v8hu)_mm_unpacklo_epi8(bytes1, zeroes);
+		v_byte_sums_b += (__v8hu)_mm_unpackhi_epi8(bytes1, zeroes);
+		v_byte_sums_c += (__v8hu)_mm_unpacklo_epi8(bytes2, zeroes);
+		v_byte_sums_d += (__v8hu)_mm_unpackhi_epi8(bytes2, zeroes);
 
 	} while (p != end);
 
 	/* Finish calculating the s2 counters */
-	v_s2 = (__v4si)_mm_slli_epi32((__m128i)v_s2, 5);
-	v_s2 += (__v4si)_mm_madd_epi16((__m128i)v_byte_sums_a,
-				       (__m128i)(__v8hi){ 32, 31, 30, 29, 28, 27, 26, 25 });
-	v_s2 += (__v4si)_mm_madd_epi16((__m128i)v_byte_sums_b,
-				       (__m128i)(__v8hi){ 24, 23, 22, 21, 20, 19, 18, 17 });
-	v_s2 += (__v4si)_mm_madd_epi16((__m128i)v_byte_sums_c,
-				       (__m128i)(__v8hi){ 16, 15, 14, 13, 12, 11, 10, 9 });
-	v_s2 += (__v4si)_mm_madd_epi16((__m128i)v_byte_sums_d,
-				       (__m128i)(__v8hi){ 8,  7,  6,  5,  4,  3,  2,  1 });
+	v_s2 = (__v4su)_mm_slli_epi32((__m128i)v_s2, 5);
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_a,
+				       (__m128i)(__v8hu){ 32, 31, 30, 29, 28, 27, 26, 25 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_b,
+				       (__m128i)(__v8hu){ 24, 23, 22, 21, 20, 19, 18, 17 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_c,
+				       (__m128i)(__v8hu){ 16, 15, 14, 13, 12, 11, 10, 9 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_d,
+				       (__m128i)(__v8hu){ 8,  7,  6,  5,  4,  3,  2,  1 });
 
-	/* Now accumulate what we computed into the real s1 and s2 */
-	v_s1 += (__v4si)_mm_shuffle_epi32((__m128i)v_s1, 0x31);
-	v_s1 += (__v4si)_mm_shuffle_epi32((__m128i)v_s1, 0x02);
-	*s1 += _mm_cvtsi128_si32((__m128i)v_s1);
-
-	v_s2 += (__v4si)_mm_shuffle_epi32((__m128i)v_s2, 0x31);
-	v_s2 += (__v4si)_mm_shuffle_epi32((__m128i)v_s2, 0x02);
-	*s2 += _mm_cvtsi128_si32((__m128i)v_s2);
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2);
 }
 #  include "../adler32_vec_template.h"
 #endif /* SSE2 implementation */
@@ -185,6 +318,10 @@ arch_select_adler32_func(void)
 {
 	u32 features = get_cpu_features();
 
+#ifdef DISPATCH_AVX512BW
+	if (features & X86_CPU_FEATURE_AVX512BW)
+		return adler32_avx512bw;
+#endif
 #ifdef DISPATCH_AVX2
 	if (features & X86_CPU_FEATURE_AVX2)
 		return adler32_avx2;
@@ -196,3 +333,5 @@ arch_select_adler32_func(void)
 	return NULL;
 }
 #endif /* DISPATCH */
+
+#endif /* LIB_X86_ADLER32_IMPL_H */

lib/x86/cpu_features.c

@@ -25,6 +25,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include "../cpu_features_common.h" /* must be included first */
 #include "cpu_features.h"
 
 #if X86_CPU_FEATURES_ENABLED
@@ -34,7 +35,7 @@ volatile u32 _cpu_features = 0;
 /* With old GCC versions we have to manually save and restore the x86_32 PIC
  * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
 #if defined(__i386__) && defined(__PIC__)
-#  define EBX_CONSTRAINT "=r"
+#  define EBX_CONSTRAINT "=&r"
 #else
 #  define EBX_CONSTRAINT "=b"
 #endif
@@ -63,7 +64,26 @@ read_xcr(u32 index)
 	return ((u64)edx << 32) | eax;
 }
 
-#define IS_SET(reg, bit) ((reg) & ((u32)1 << (bit)))
+#undef BIT
+#define BIT(nr)			(1UL << (nr))
+
+#define XCR0_BIT_SSE		BIT(1)
+#define XCR0_BIT_AVX		BIT(2)
+#define XCR0_BIT_OPMASK		BIT(5)
+#define XCR0_BIT_ZMM_HI256	BIT(6)
+#define XCR0_BIT_HI16_ZMM	BIT(7)
+
+#define IS_SET(reg, nr)		((reg) & BIT(nr))
+#define IS_ALL_SET(reg, mask)	(((reg) & (mask)) == (mask))
+
+static const struct cpu_feature x86_cpu_feature_table[] = {
+	{X86_CPU_FEATURE_SSE2,		"sse2"},
+	{X86_CPU_FEATURE_PCLMUL,	"pclmul"},
+	{X86_CPU_FEATURE_AVX,		"avx"},
+	{X86_CPU_FEATURE_AVX2,		"avx2"},
+	{X86_CPU_FEATURE_BMI2,		"bmi2"},
+	{X86_CPU_FEATURE_AVX512BW,	"avx512bw"},
+};
 
 /* Initialize _cpu_features with bits for interesting processor features. */
 void setup_cpu_features(void)
@@ -72,7 +92,8 @@ void setup_cpu_features(void)
 	u32 dummy1, dummy2, dummy3, dummy4;
 	u32 max_function;
 	u32 features_1, features_2, features_3, features_4;
-	bool os_saves_ymm_regs = false;
+	bool os_avx_support = false;
+	bool os_avx512_support = false;
 
 	/* Get maximum supported function  */
 	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
@@ -86,13 +107,24 @@ void setup_cpu_features(void)
 		features |= X86_CPU_FEATURE_SSE2;
 
 	if (IS_SET(features_2, 1))
-		features |= X86_CPU_FEATURE_PCLMULQDQ;
+		features |= X86_CPU_FEATURE_PCLMUL;
 
-	if (IS_SET(features_2, 27)) /* OSXSAVE set?  */
-		if ((read_xcr(0) & 0x6) == 0x6)
-			os_saves_ymm_regs = true;
+	if (IS_SET(features_2, 27)) { /* OSXSAVE set? */
+		u64 xcr0 = read_xcr(0);
 
-	if (os_saves_ymm_regs && IS_SET(features_2, 28))
+		os_avx_support = IS_ALL_SET(xcr0,
+					    XCR0_BIT_SSE |
+					    XCR0_BIT_AVX);
+
+		os_avx512_support = IS_ALL_SET(xcr0,
+					       XCR0_BIT_SSE |
+					       XCR0_BIT_AVX |
+					       XCR0_BIT_OPMASK |
+					       XCR0_BIT_ZMM_HI256 |
+					       XCR0_BIT_HI16_ZMM);
+	}
+
+	if (os_avx_support && IS_SET(features_2, 28))
 		features |= X86_CPU_FEATURE_AVX;
 
 	if (max_function < 7)
@@ -101,13 +133,19 @@ void setup_cpu_features(void)
 	/* Extended feature flags  */
 	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
 
-	if (os_saves_ymm_regs && IS_SET(features_3, 5))
+	if (os_avx_support && IS_SET(features_3, 5))
 		features |= X86_CPU_FEATURE_AVX2;
 
 	if (IS_SET(features_3, 8))
 		features |= X86_CPU_FEATURE_BMI2;
 
+	if (os_avx512_support && IS_SET(features_3, 30))
+		features |= X86_CPU_FEATURE_AVX512BW;
+
 out:
+	disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
+					 ARRAY_LEN(x86_cpu_feature_table));
+
 	_cpu_features = features | X86_CPU_FEATURES_KNOWN;
 }
 

lib/x86/cpu_features.h

@@ -17,16 +17,17 @@
 #if X86_CPU_FEATURES_ENABLED
 
 #define X86_CPU_FEATURE_SSE2		0x00000001
-#define X86_CPU_FEATURE_PCLMULQDQ	0x00000002
+#define X86_CPU_FEATURE_PCLMUL		0x00000002
 #define X86_CPU_FEATURE_AVX		0x00000004
 #define X86_CPU_FEATURE_AVX2		0x00000008
 #define X86_CPU_FEATURE_BMI2		0x00000010
+#define X86_CPU_FEATURE_AVX512BW	0x00000020
 
 #define X86_CPU_FEATURES_KNOWN		0x80000000
 
 extern volatile u32 _cpu_features;
 
-extern void setup_cpu_features(void);
+void setup_cpu_features(void);
 
 static inline u32 get_cpu_features(void)
 {

lib/x86/crc32_impl.h

@@ -25,6 +25,9 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_X86_CRC32_IMPL_H
+#define LIB_X86_CRC32_IMPL_H
+
 #include "cpu_features.h"
 
 /*
@@ -74,14 +77,16 @@ arch_select_crc32_func(void)
 	u32 features = get_cpu_features();
 
 #ifdef DISPATCH_PCLMUL_AVX
-	if ((features & X86_CPU_FEATURE_PCLMULQDQ) &&
+	if ((features & X86_CPU_FEATURE_PCLMUL) &&
 	    (features & X86_CPU_FEATURE_AVX))
 		return crc32_pclmul_avx;
 #endif
 #ifdef DISPATCH_PCLMUL
-	if (features & X86_CPU_FEATURE_PCLMULQDQ)
+	if (features & X86_CPU_FEATURE_PCLMUL)
 		return crc32_pclmul;
 #endif
 	return NULL;
 }
 #endif /* DISPATCH */
+
+#endif /* LIB_X86_CRC32_IMPL_H */

lib/x86/crc32_pclmul_template.h

@@ -25,7 +25,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <wmmintrin.h>
+#include <immintrin.h>
 
 /*
  * CRC-32 folding with PCLMULQDQ.
@@ -92,8 +92,8 @@
  * When <= 512 bits remain in the message, we finish up by folding across
  * smaller distances.  This works similarly; the distance D is just different,
  * so different constant multipliers must be used.  Finally, once the remaining
- * message is just 64 bits, it is is reduced to the CRC-32 using Barrett
- * reduction (explained later).
+ * message is just 64 bits, it is reduced to the CRC-32 using Barrett reduction
+ * (explained later).
  *
  * For more information see the original paper from Intel:
  *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"

lib/x86/decompress_impl.h

@@ -1,3 +1,6 @@
+#ifndef LIB_X86_DECOMPRESS_IMPL_H
+#define LIB_X86_DECOMPRESS_IMPL_H
+
 #include "cpu_features.h"
 
 /* Include the BMI2-optimized version? */
@@ -24,3 +27,5 @@ arch_select_decompress_func(void)
 	return NULL;
 }
 #endif /* DISPATCH */
+
+#endif /* LIB_X86_DECOMPRESS_IMPL_H */

lib/x86/matchfinder_impl.h

@@ -25,48 +25,42 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef LIB_X86_MATCHFINDER_IMPL_H
+#define LIB_X86_MATCHFINDER_IMPL_H
+
 #ifdef __AVX2__
-#  if MATCHFINDER_ALIGNMENT < 32
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 32
-#  endif
 #  include <immintrin.h>
-static forceinline bool
+static forceinline void
 matchfinder_init_avx2(mf_pos_t *data, size_t size)
 {
-	__m256i v, *p;
-	size_t n;
-
-	if (size % (sizeof(__m256i) * 4) != 0)
-		return false;
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
 
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
 	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
+
 	do {
 		p[0] = v;
 		p[1] = v;
 		p[2] = v;
 		p[3] = v;
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
+#define matchfinder_init matchfinder_init_avx2
 
-static forceinline bool
+static forceinline void
 matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
 {
-	__m256i v, *p;
-	size_t n;
-
-	if (size % (sizeof(__m256i) * 4) != 0)
-		return false;
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
 
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
 	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
+
 	do {
 		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
 		p[0] = _mm256_adds_epi16(p[0], v);
@@ -74,53 +68,44 @@ matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
 		p[2] = _mm256_adds_epi16(p[2], v);
 		p[3] = _mm256_adds_epi16(p[3], v);
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
-#endif /* __AVX2__ */
+#define matchfinder_rebase matchfinder_rebase_avx2
 
-#ifdef __SSE2__
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
-#  endif
+#elif defined(__SSE2__)
 #  include <emmintrin.h>
-static forceinline bool
+static forceinline void
 matchfinder_init_sse2(mf_pos_t *data, size_t size)
 {
-	__m128i v, *p;
-	size_t n;
-
-	if (size % (sizeof(__m128i) * 4) != 0)
-		return false;
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
 
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
 	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
+
 	do {
 		p[0] = v;
 		p[1] = v;
 		p[2] = v;
 		p[3] = v;
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
+#define matchfinder_init matchfinder_init_sse2
 
-static forceinline bool
+static forceinline void
 matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
 {
-	__m128i v, *p;
-	size_t n;
-
-	if (size % (sizeof(__m128i) * 4) != 0)
-		return false;
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
 
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
 	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
+
 	do {
 		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
 		p[0] = _mm_adds_epi16(p[0], v);
@@ -128,37 +113,10 @@ matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
 		p[2] = _mm_adds_epi16(p[2], v);
 		p[3] = _mm_adds_epi16(p[3], v);
 		p += 4;
-	} while (--n);
-	return true;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
 }
+#define matchfinder_rebase matchfinder_rebase_sse2
 #endif /* __SSE2__ */
 
-#undef arch_matchfinder_init
-static forceinline bool
-arch_matchfinder_init(mf_pos_t *data, size_t size)
-{
-#ifdef __AVX2__
-	if (matchfinder_init_avx2(data, size))
-		return true;
-#endif
-#ifdef __SSE2__
-	if (matchfinder_init_sse2(data, size))
-		return true;
-#endif
-	return false;
-}
-
-#undef arch_matchfinder_rebase
-static forceinline bool
-arch_matchfinder_rebase(mf_pos_t *data, size_t size)
-{
-#ifdef __AVX2__
-	if (matchfinder_rebase_avx2(data, size))
-		return true;
-#endif
-#ifdef __SSE2__
-	if (matchfinder_rebase_sse2(data, size))
-		return true;
-#endif
-	return false;
-}
+#endif /* LIB_X86_MATCHFINDER_IMPL_H */

lib/zlib_compress.c

@@ -1,8 +1,6 @@
 /*
  * zlib_compress.c - compress with a zlib wrapper
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -33,9 +31,9 @@
 
 #include "libdeflate.h"
 
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_zlib_compress(struct libdeflate_compressor *c,
-			 const void *in, size_t in_size,
+			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
@@ -65,20 +63,20 @@ libdeflate_zlib_compress(struct libdeflate_compressor *c,
 	out_next += 2;
 
 	/* Compressed data  */
-	deflate_size = libdeflate_deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - ZLIB_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* ADLER32  */
-	put_unaligned_be32(libdeflate_adler32(1, in, in_size), out_next);
+	put_unaligned_be32(libdeflate_adler32(1, in, in_nbytes), out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_zlib_compress_bound(struct libdeflate_compressor *c,
 			       size_t in_nbytes)
 {

lib/zlib_decompress.c

@@ -1,8 +1,6 @@
 /*
  * zlib_decompress.c - decompress with a zlib wrapper
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
@@ -32,15 +30,17 @@
 
 #include "libdeflate.h"
 
-LIBDEFLATEAPI enum libdeflate_result
-libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
-			   const void *in, size_t in_nbytes,
-			   void *out, size_t out_nbytes_avail,
-			   size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret)
 {
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
 	u16 hdr;
+	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
 	enum libdeflate_result result;
 
@@ -68,10 +68,10 @@ libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
 		return LIBDEFLATE_BAD_DATA;
 
 	/* Compressed data  */
-	result = libdeflate_deflate_decompress(d, in_next,
+	result = libdeflate_deflate_decompress_ex(d, in_next,
 					in_end - ZLIB_FOOTER_SIZE - in_next,
 					out, out_nbytes_avail,
-					actual_out_nbytes_ret);
+					&actual_in_nbytes, actual_out_nbytes_ret);
 	if (result != LIBDEFLATE_SUCCESS)
 		return result;
 
@@ -80,12 +80,27 @@ libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
 	else
 		actual_out_nbytes = out_nbytes_avail;
 
-	in_next = in_end - ZLIB_FOOTER_SIZE;
+	in_next += actual_in_nbytes;
 
 	/* ADLER32  */
 	if (libdeflate_adler32(1, out, actual_out_nbytes) !=
 	    get_unaligned_be32(in_next))
 		return LIBDEFLATE_BAD_DATA;
+	in_next += 4;
+
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
 
 	return LIBDEFLATE_SUCCESS;
 }
+
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret)
+{
+	return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
+					     out, out_nbytes_avail,
+					     NULL, actual_out_nbytes_ret);
+}

libdeflate.h

@@ -10,27 +10,47 @@ extern "C" {
 #endif
 
 #define LIBDEFLATE_VERSION_MAJOR	1
-#define LIBDEFLATE_VERSION_MINOR	0
-#define LIBDEFLATE_VERSION_STRING	"1.0"
+#define LIBDEFLATE_VERSION_MINOR	10
+#define LIBDEFLATE_VERSION_STRING	"1.10"
 
 #include <stddef.h>
 #include <stdint.h>
 
 /*
  * On Windows, if you want to link to the DLL version of libdeflate, then
- * #define LIBDEFLATE_DLL.
+ * #define LIBDEFLATE_DLL.  Note that the calling convention is "stdcall".
  */
 #ifdef LIBDEFLATE_DLL
 #  ifdef BUILDING_LIBDEFLATE
-#    define LIBDEFLATEAPI LIBEXPORT
+#    define LIBDEFLATEEXPORT	LIBEXPORT
 #  elif defined(_WIN32) || defined(__CYGWIN__)
-#    define LIBDEFLATEAPI __declspec(dllimport)
+#    define LIBDEFLATEEXPORT	__declspec(dllimport)
 #  endif
 #endif
-#ifndef LIBDEFLATEAPI
-#  define LIBDEFLATEAPI
+#ifndef LIBDEFLATEEXPORT
+#  define LIBDEFLATEEXPORT
 #endif
 
+#if defined(_WIN32) && !defined(_WIN64) && defined(LIBDEFLATE_DLL)
+#  define LIBDEFLATEAPI_ABI	__stdcall
+#else
+#  define LIBDEFLATEAPI_ABI
+#endif
+
+#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \
+	defined(_WIN32) && !defined(_WIN64)
+    /*
+     * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4.
+     * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX
+     * code when called from an MSVC-compiled application.
+     */
+#  define LIBDEFLATEAPI_STACKALIGN	__attribute__((force_align_arg_pointer))
+#else
+#  define LIBDEFLATEAPI_STACKALIGN
+#endif
+
+#define LIBDEFLATEAPI	LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN
+
 /* ========================================================================== */
 /*                             Compression                                    */
 /* ========================================================================== */
@@ -41,8 +61,12 @@ struct libdeflate_compressor;
  * libdeflate_alloc_compressor() allocates a new compressor that supports
  * DEFLATE, zlib, and gzip compression.  'compression_level' is the compression
  * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
- * medium/default, 9 = slow, 12 = slowest).  The return value is a pointer to
- * the new compressor, or NULL if out of memory.
+ * medium/default, 9 = slow, 12 = slowest).  Level 0 is also supported and means
+ * "no compression", specifically "create a valid stream, but only emit
+ * uncompressed blocks" (this will expand the data slightly).
+ *
+ * The return value is a pointer to the new compressor, or NULL if out of memory
+ * or if the compression level is invalid (i.e. outside the range [0, 12]).
  *
  * Note: for compression, the sliding window size is defined at compilation time
  * to 32768, the largest size permissible in the DEFLATE format.  It cannot be
@@ -51,7 +75,7 @@ struct libdeflate_compressor;
  * A single compressor is not safe to use by multiple threads concurrently.
  * However, different threads may use different compressors concurrently.
  */
-LIBDEFLATEAPI struct libdeflate_compressor *
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
 libdeflate_alloc_compressor(int compression_level);
 
 /*
@@ -61,7 +85,7 @@ libdeflate_alloc_compressor(int compression_level);
  * bytes.  The return value is the compressed size in bytes, or 0 if the data
  * could not be compressed to 'out_nbytes_avail' bytes or fewer.
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
 			    const void *in, size_t in_nbytes,
 			    void *out, size_t out_nbytes_avail);
@@ -91,7 +115,7 @@ libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
  * libdeflate_deflate_compress() returns 0, indicating that the compressed data
  * did not fit into the provided output buffer.
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
 				  size_t in_nbytes);
 
@@ -99,7 +123,7 @@ libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
  * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper
  * format.
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail);
@@ -109,7 +133,7 @@ libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
  * compressed with libdeflate_zlib_compress() rather than with
  * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
 			       size_t in_nbytes);
 
@@ -117,7 +141,7 @@ libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
  * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper
  * format.
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail);
@@ -127,7 +151,7 @@ libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
  * compressed with libdeflate_gzip_compress() rather than with
  * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
 			       size_t in_nbytes);
 
@@ -136,7 +160,7 @@ libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
  * libdeflate_alloc_compressor().  If a NULL pointer is passed in, no action is
  * taken.
  */
-LIBDEFLATEAPI void
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
 
 /* ========================================================================== */
@@ -157,7 +181,7 @@ struct libdeflate_decompressor;
  * A single decompressor is not safe to use by multiple threads concurrently.
  * However, different threads may use different decompressors concurrently.
  */
-LIBDEFLATEAPI struct libdeflate_decompressor *
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
 libdeflate_alloc_decompressor(void);
 
 /*
@@ -213,7 +237,7 @@ enum libdeflate_result {
  *     not large enough but no other problems were encountered, or another
  *     nonzero result code if decompression failed for another reason.
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -225,7 +249,7 @@ libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
  * then the actual compressed size of the DEFLATE stream (aligned to the next
  * byte boundary) is written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
 				 const void *in, size_t in_nbytes,
 				 void *out, size_t out_nbytes_avail,
@@ -235,13 +259,31 @@ libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
 /*
  * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
  * instead of raw DEFLATE.
+ *
+ * Decompression will stop at the end of the zlib stream, even if it is shorter
+ * than 'in_nbytes'.  If you need to know exactly where the zlib stream ended,
+ * use libdeflate_zlib_decompress_ex().
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,
 			   size_t *actual_out_nbytes_ret);
 
+/*
+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first zlib-compressed stream in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret);
+
 /*
  * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
  * instead of raw DEFLATE.
@@ -250,7 +292,7 @@ libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
  * will be decompressed.  Use libdeflate_gzip_decompress_ex() if you need
  * multi-member support.
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,
@@ -263,7 +305,7 @@ libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
  * buffer was decompressed), then the actual number of input bytes consumed is
  * written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEAPI enum libdeflate_result
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -275,7 +317,7 @@ libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
  * libdeflate_alloc_decompressor().  If a NULL pointer is passed in, no action
  * is taken.
  */
-LIBDEFLATEAPI void
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
 
 /* ========================================================================== */
@@ -288,8 +330,8 @@ libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
  * required initial value for 'adler' is 1.  This value is also returned when
  * 'buffer' is specified as NULL.
  */
-LIBDEFLATEAPI uint32_t
-libdeflate_adler32(uint32_t adler32, const void *buffer, size_t len);
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
 
 
 /*
@@ -298,9 +340,25 @@ libdeflate_adler32(uint32_t adler32, const void *buffer, size_t len);
  * initial value for 'crc' is 0.  This value is also returned when 'buffer' is
  * specified as NULL.
  */
-LIBDEFLATEAPI uint32_t
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
 
+/* ========================================================================== */
+/*                           Custom memory allocator                          */
+/* ========================================================================== */
+
+/*
+ * Install a custom memory allocator which libdeflate will use for all memory
+ * allocations.  'malloc_func' is a function that must behave like malloc(), and
+ * 'free_func' is a function that must behave like free().
+ *
+ * There must not be any libdeflate_compressor or libdeflate_decompressor
+ * structures in existence when calling this function.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *));
+
 #ifdef __cplusplus
 }
 #endif

libdeflate.pc.in

@@ -0,0 +1,10 @@
+prefix=@PREFIX@
+exec_prefix=${prefix}
+includedir=@INCDIR@
+libdir=@LIBDIR@
+
+Name: libdeflate
+Description: Fast implementation of DEFLATE, zlib, and gzip
+Version: @VERSION@
+Libs: -L${libdir} -ldeflate
+Cflags: -I${includedir}

programs/benchmark.c

@@ -25,27 +25,25 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <zlib.h> /* for comparison purposes */
+#include "test_util.h"
 
-#include "prog_util.h"
+static const tchar *const optstring = T("0::1::2::3::4::5::6::7::8::9::C:D:eghs:VYZz");
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::C:D:ghs:VYZz");
-
-enum wrapper {
-	NO_WRAPPER,
-	ZLIB_WRAPPER,
-	GZIP_WRAPPER,
+enum format {
+	DEFLATE_FORMAT,
+	ZLIB_FORMAT,
+	GZIP_FORMAT,
 };
 
 struct compressor {
 	int level;
-	enum wrapper wrapper;
+	enum format format;
 	const struct engine *engine;
 	void *private;
 };
 
 struct decompressor {
-	enum wrapper wrapper;
+	enum format format;
 	const struct engine *engine;
 	void *private;
 };
@@ -54,6 +52,7 @@ struct engine {
 	const tchar *name;
 
 	bool (*init_compressor)(struct compressor *);
+	size_t (*compress_bound)(struct compressor *, size_t);
 	size_t (*compress)(struct compressor *, const void *, size_t,
 			   void *, size_t);
 	void (*destroy_compressor)(struct compressor *);
@@ -73,15 +72,28 @@ libdeflate_engine_init_compressor(struct compressor *c)
 	return c->private != NULL;
 }
 
+static size_t
+libdeflate_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	switch (c->format) {
+	case ZLIB_FORMAT:
+		return libdeflate_zlib_compress_bound(c->private, in_nbytes);
+	case GZIP_FORMAT:
+		return libdeflate_gzip_compress_bound(c->private, in_nbytes);
+	default:
+		return libdeflate_deflate_compress_bound(c->private, in_nbytes);
+	}
+}
+
 static size_t
 libdeflate_engine_compress(struct compressor *c, const void *in,
 			   size_t in_nbytes, void *out, size_t out_nbytes_avail)
 {
-	switch (c->wrapper) {
-	case ZLIB_WRAPPER:
+	switch (c->format) {
+	case ZLIB_FORMAT:
 		return libdeflate_zlib_compress(c->private, in, in_nbytes,
 						out, out_nbytes_avail);
-	case GZIP_WRAPPER:
+	case GZIP_FORMAT:
 		return libdeflate_gzip_compress(c->private, in, in_nbytes,
 						out, out_nbytes_avail);
 	default:
@@ -107,11 +119,11 @@ static bool
 libdeflate_engine_decompress(struct decompressor *d, const void *in,
 			     size_t in_nbytes, void *out, size_t out_nbytes)
 {
-	switch (d->wrapper) {
-	case ZLIB_WRAPPER:
+	switch (d->format) {
+	case ZLIB_FORMAT:
 		return !libdeflate_zlib_decompress(d->private, in, in_nbytes,
 						   out, out_nbytes, NULL);
-	case GZIP_WRAPPER:
+	case GZIP_FORMAT:
 		return !libdeflate_gzip_decompress(d->private, in, in_nbytes,
 						   out, out_nbytes, NULL);
 	default:
@@ -130,6 +142,7 @@ static const struct engine libdeflate_engine = {
 	.name			= T("libdeflate"),
 
 	.init_compressor	= libdeflate_engine_init_compressor,
+	.compress_bound		= libdeflate_engine_compress_bound,
 	.compress		= libdeflate_engine_compress,
 	.destroy_compressor	= libdeflate_engine_destroy_compressor,
 
@@ -141,13 +154,13 @@ static const struct engine libdeflate_engine = {
 /******************************************************************************/
 
 static int
-get_libz_window_bits(enum wrapper wrapper)
+get_libz_window_bits(enum format format)
 {
 	const int windowBits = 15;
-	switch (wrapper) {
-	case ZLIB_WRAPPER:
+	switch (format) {
+	case ZLIB_FORMAT:
 		return windowBits;
-	case GZIP_WRAPPER:
+	case GZIP_FORMAT:
 		return windowBits + 16;
 	default:
 		return -windowBits;
@@ -174,7 +187,7 @@ libz_engine_init_compressor(struct compressor *c)
 	z->zfree = NULL;
 	z->opaque = NULL;
 	if (deflateInit2(z, c->level, Z_DEFLATED,
-			 get_libz_window_bits(c->wrapper),
+			 get_libz_window_bits(c->format),
 			 8, Z_DEFAULT_STRATEGY) != Z_OK)
 	{
 		msg("unable to initialize deflater");
@@ -186,6 +199,12 @@ libz_engine_init_compressor(struct compressor *c)
 	return true;
 }
 
+static size_t
+libz_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return deflateBound(c->private, in_nbytes);
+}
+
 static size_t
 libz_engine_compress(struct compressor *c, const void *in, size_t in_nbytes,
 		     void *out, size_t out_nbytes_avail)
@@ -228,7 +247,7 @@ libz_engine_init_decompressor(struct decompressor *d)
 	z->zalloc = NULL;
 	z->zfree = NULL;
 	z->opaque = NULL;
-	if (inflateInit2(z, get_libz_window_bits(d->wrapper)) != Z_OK) {
+	if (inflateInit2(z, get_libz_window_bits(d->format)) != Z_OK) {
 		msg("unable to initialize inflater");
 		free(z);
 		return false;
@@ -267,6 +286,7 @@ static const struct engine libz_engine = {
 	.name			= T("libz"),
 
 	.init_compressor	= libz_engine_init_compressor,
+	.compress_bound		= libz_engine_compress_bound,
 	.compress		= libz_engine_compress,
 	.destroy_compressor	= libz_engine_destroy_compressor,
 
@@ -298,15 +318,21 @@ name_to_engine(const tchar *name)
 /******************************************************************************/
 
 static bool
-compressor_init(struct compressor *c, int level, enum wrapper wrapper,
+compressor_init(struct compressor *c, int level, enum format format,
 		const struct engine *engine)
 {
 	c->level = level;
-	c->wrapper = wrapper;
+	c->format = format;
 	c->engine = engine;
 	return engine->init_compressor(c);
 }
 
+static size_t
+compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return c->engine->compress_bound(c, in_nbytes);
+}
+
 static size_t
 do_compress(struct compressor *c, const void *in, size_t in_nbytes,
 	    void *out, size_t out_nbytes_avail)
@@ -317,14 +343,15 @@ do_compress(struct compressor *c, const void *in, size_t in_nbytes,
 static void
 compressor_destroy(struct compressor *c)
 {
-	c->engine->destroy_compressor(c);
+	if (c->engine != NULL)
+		c->engine->destroy_compressor(c);
 }
 
 static bool
-decompressor_init(struct decompressor *d, enum wrapper wrapper,
+decompressor_init(struct decompressor *d, enum format format,
 		  const struct engine *engine)
 {
-	d->wrapper = wrapper;
+	d->format = format;
 	d->engine = engine;
 	return engine->init_decompressor(d);
 }
@@ -339,7 +366,8 @@ do_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
 static void
 decompressor_destroy(struct decompressor *d)
 {
-	d->engine->destroy_decompressor(d);
+	if (d->engine != NULL)
+		d->engine->destroy_decompressor(d);
 }
 
 /******************************************************************************/
@@ -366,17 +394,19 @@ show_usage(FILE *fp)
 "Benchmark DEFLATE compression and decompression on the specified FILEs.\n"
 "\n"
 "Options:\n"
+"  -0        no compression\n"
 "  -1        fastest (worst) compression\n"
 "  -6        medium compression (default)\n"
 "  -12       slowest (best) compression\n"
 "  -C ENGINE compression engine\n"
 "  -D ENGINE decompression engine\n"
-"  -g        use gzip wrapper\n"
+"  -e        allow chunks to be expanded (implied by -0)\n"
+"  -g        use gzip format instead of raw DEFLATE\n"
 "  -h        print this help\n"
 "  -s SIZE   chunk size\n"
 "  -V        show version and legal information\n"
-"  -z        use zlib wrapper\n"
-"\n", program_invocation_name);
+"  -z        use zlib format instead of raw DEFLATE\n"
+"\n", prog_invocation_name);
 
 	show_available_engines(fp);
 }
@@ -400,6 +430,7 @@ show_version(void)
 static int
 do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 	     void *decompressed_buf, u32 chunk_size,
+	     bool allow_expansion, size_t compressed_buf_size,
 	     struct compressor *compressor,
 	     struct decompressor *decompressor)
 {
@@ -411,19 +442,31 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 	while ((ret = xread(in, original_buf, chunk_size)) > 0) {
 		u32 original_size = ret;
+		size_t out_nbytes_avail;
 		u32 compressed_size;
 		u64 start_time;
 		bool ok;
 
 		total_uncompressed_size += original_size;
 
+		if (allow_expansion) {
+			out_nbytes_avail = compress_bound(compressor,
+							  original_size);
+			if (out_nbytes_avail > compressed_buf_size) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
+		} else {
+			out_nbytes_avail = original_size - 1;
+		}
+
 		/* Compress the chunk of data. */
 		start_time = timer_ticks();
 		compressed_size = do_compress(compressor,
 					      original_buf,
 					      original_size,
 					      compressed_buf,
-					      original_size - 1);
+					      out_nbytes_avail);
 		total_compress_time += timer_ticks() - start_time;
 
 		if (compressed_size) {
@@ -453,7 +496,14 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 			total_compressed_size += compressed_size;
 		} else {
-			/* Compression did not make the chunk smaller. */
+			/*
+			 * The chunk would have compressed to more than
+			 * out_nbytes_avail bytes.
+			 */
+			if (allow_expansion) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
 			total_compressed_size += original_size;
 		}
 	}
@@ -492,23 +542,26 @@ tmain(int argc, tchar *argv[])
 {
 	u32 chunk_size = 1048576;
 	int level = 6;
-	enum wrapper wrapper = NO_WRAPPER;
+	enum format format = DEFLATE_FORMAT;
 	const struct engine *compress_engine = &DEFAULT_ENGINE;
 	const struct engine *decompress_engine = &DEFAULT_ENGINE;
+	bool allow_expansion = false;
+	struct compressor compressor = { 0 };
+	struct decompressor decompressor = { 0 };
+	size_t compressed_buf_size;
 	void *original_buf = NULL;
 	void *compressed_buf = NULL;
 	void *decompressed_buf = NULL;
-	struct compressor compressor;
-	struct decompressor decompressor;
 	tchar *default_file_list[] = { NULL };
 	int opt_char;
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
+		case '0':
 		case '1':
 		case '2':
 		case '3':
@@ -519,7 +572,7 @@ tmain(int argc, tchar *argv[])
 		case '8':
 		case '9':
 			level = parse_compression_level(opt_char, toptarg);
-			if (level == 0)
+			if (level < 0)
 				return 1;
 			break;
 		case 'C':
@@ -538,8 +591,11 @@ tmain(int argc, tchar *argv[])
 				return 1;
 			}
 			break;
+		case 'e':
+			allow_expansion = true;
+			break;
 		case 'g':
-			wrapper = GZIP_WRAPPER;
+			format = GZIP_FORMAT;
 			break;
 		case 'h':
 			show_usage(stdout);
@@ -561,7 +617,7 @@ tmain(int argc, tchar *argv[])
 			decompress_engine = &libz_engine;
 			break;
 		case 'z':
-			wrapper = ZLIB_WRAPPER;
+			format = ZLIB_FORMAT;
 			break;
 		default:
 			show_usage(stderr);
@@ -572,20 +628,28 @@ tmain(int argc, tchar *argv[])
 	argc -= toptind;
 	argv += toptind;
 
+	if (level == 0)
+		allow_expansion = true;
+
+	ret = -1;
+	if (!compressor_init(&compressor, level, format, compress_engine))
+		goto out;
+	if (!decompressor_init(&decompressor, format, decompress_engine))
+		goto out;
+
+	if (allow_expansion)
+		compressed_buf_size = compress_bound(&compressor, chunk_size);
+	else
+		compressed_buf_size = chunk_size - 1;
+
 	original_buf = xmalloc(chunk_size);
-	compressed_buf = xmalloc(chunk_size - 1);
+	compressed_buf = xmalloc(compressed_buf_size);
 	decompressed_buf = xmalloc(chunk_size);
 
 	ret = -1;
 	if (original_buf == NULL || compressed_buf == NULL ||
 	    decompressed_buf == NULL)
-		goto out0;
-
-	if (!compressor_init(&compressor, level, wrapper, compress_engine))
-		goto out0;
-
-	if (!decompressor_init(&decompressor, wrapper, decompress_engine))
-		goto out1;
+		goto out;
 
 	if (argc == 0) {
 		argv = default_file_list;
@@ -596,12 +660,11 @@ tmain(int argc, tchar *argv[])
 				argv[i] = NULL;
 	}
 
-	printf("Benchmarking DEFLATE compression:\n");
+	printf("Benchmarking %s compression:\n",
+	       format == DEFLATE_FORMAT ? "DEFLATE" :
+	       format == ZLIB_FORMAT ? "zlib" : "gzip");
 	printf("\tCompression level: %d\n", level);
 	printf("\tChunk size: %"PRIu32"\n", chunk_size);
-	printf("\tWrapper: %s\n",
-	       wrapper == NO_WRAPPER ? "None" :
-	       wrapper == ZLIB_WRAPPER ? "zlib" : "gzip");
 	printf("\tCompression engine: %"TS"\n", compress_engine->name);
 	printf("\tDecompression engine: %"TS"\n", decompress_engine->name);
 
@@ -610,25 +673,24 @@ tmain(int argc, tchar *argv[])
 
 		ret = xopen_for_read(argv[i], true, &in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 
 		printf("Processing %"TS"...\n", in.name);
 
 		ret = do_benchmark(&in, original_buf, compressed_buf,
-				   decompressed_buf, chunk_size, &compressor,
-				   &decompressor);
+				   decompressed_buf, chunk_size,
+				   allow_expansion, compressed_buf_size,
+				   &compressor, &decompressor);
 		xclose(&in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 	}
 	ret = 0;
-out2:
-	decompressor_destroy(&decompressor);
-out1:
-	compressor_destroy(&compressor);
-out0:
+out:
 	free(decompressed_buf);
 	free(compressed_buf);
 	free(original_buf);
+	decompressor_destroy(&decompressor);
+	compressor_destroy(&compressor);
 	return -ret;
 }

programs/checksum.c

@@ -25,9 +25,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <zlib.h>
-
-#include "prog_util.h"
+#include "test_util.h"
 
 static const tchar *const optstring = T("Ahs:tZ");
 
@@ -44,23 +42,35 @@ show_usage(FILE *fp)
 "  -s SIZE   chunk size\n"
 "  -t        show checksum speed, excluding I/O\n"
 "  -Z        use zlib implementation instead of libdeflate\n",
-	program_invocation_name);
+	prog_invocation_name);
+}
+
+typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
+
+static u32
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
 }
 
 static u32
-zlib_adler32(u32 adler, const void *buf, size_t len)
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
 {
 	return adler32(adler, buf, len);
 }
 
 static u32
-zlib_crc32(u32 crc, const void *buf, size_t len)
+crc32_zlib(u32 crc, const void *buf, size_t len)
 {
 	return crc32(crc, buf, len);
 }
 
-typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
-
 static int
 checksum_stream(struct file_stream *in, cksum_fn_t cksum, u32 *sum,
 		void *buf, size_t bufsize, u64 *size_ret, u64 *elapsed_ret)
@@ -105,7 +115,7 @@ tmain(int argc, tchar *argv[])
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
@@ -139,14 +149,14 @@ tmain(int argc, tchar *argv[])
 
 	if (use_adler32) {
 		if (use_zlib_impl)
-			cksum = zlib_adler32;
+			cksum = adler32_zlib;
 		else
-			cksum = libdeflate_adler32;
+			cksum = adler32_libdeflate;
 	} else {
 		if (use_zlib_impl)
-			cksum = zlib_crc32;
+			cksum = crc32_zlib;
 		else
-			cksum = libdeflate_crc32;
+			cksum = crc32_libdeflate;
 	}
 
 	buf = xmalloc(bufsize);

programs/gzip.c

@@ -38,22 +38,29 @@
 #  include <utime.h>
 #endif
 
+#define GZIP_MIN_HEADER_SIZE	10
+#define GZIP_FOOTER_SIZE	8
+#define GZIP_MIN_OVERHEAD	(GZIP_MIN_HEADER_SIZE + GZIP_FOOTER_SIZE)
+#define GZIP_ID1		0x1F
+#define GZIP_ID2		0x8B
+
 struct options {
 	bool to_stdout;
 	bool decompress;
 	bool force;
 	bool keep;
+	bool test;
 	int compression_level;
 	const tchar *suffix;
 };
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknS:V");
+static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknqS:tV");
 
 static void
 show_usage(FILE *fp)
 {
 	fprintf(fp,
-"Usage: %"TS" [-LEVEL] [-cdfhkV] [-S SUF] FILE...\n"
+"Usage: %"TS" [-LEVEL] [-cdfhkqtV] [-S SUF] FILE...\n"
 "Compress or decompress the specified FILEs.\n"
 "\n"
 "Options:\n"
@@ -62,12 +69,16 @@ show_usage(FILE *fp)
 "  -12       slowest (best) compression\n"
 "  -c        write to standard output\n"
 "  -d        decompress\n"
-"  -f        overwrite existing output files\n"
+"  -f        overwrite existing output files; (de)compress hard-linked files;\n"
+"            allow reading/writing compressed data from/to terminal;\n"
+"            with gunzip -c, pass through non-gzipped data\n"
 "  -h        print this help\n"
 "  -k        don't delete input files\n"
+"  -q        suppress warnings\n"
 "  -S SUF    use suffix SUF instead of .gz\n"
+"  -t        test file integrity\n"
 "  -V        show version and legal information\n",
-	program_invocation_name);
+	prog_invocation_name);
 }
 
 static void
@@ -87,14 +98,14 @@ show_version(void)
 static bool
 is_gunzip(void)
 {
-	if (tstrxcmp(program_invocation_name, T("gunzip")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip")) == 0)
 		return true;
-	if (tstrxcmp(program_invocation_name, T("libdeflate-gunzip")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip")) == 0)
 		return true;
 #ifdef _WIN32
-	if (tstrxcmp(program_invocation_name, T("gunzip.exe")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip.exe")) == 0)
 		return true;
-	if (tstrxcmp(program_invocation_name, T("libdeflate-gunzip.exe")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip.exe")) == 0)
 		return true;
 #endif
 	return false;
@@ -183,27 +194,56 @@ load_u32_gzip(const u8 *p)
 
 static int
 do_decompress(struct libdeflate_decompressor *decompressor,
-	      struct file_stream *in, struct file_stream *out)
+	      struct file_stream *in, struct file_stream *out,
+	      const struct options *options)
 {
 	const u8 *compressed_data = in->mmap_mem;
 	size_t compressed_size = in->mmap_size;
 	void *uncompressed_data = NULL;
 	size_t uncompressed_size;
+	size_t max_uncompressed_size;
 	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
 	enum libdeflate_result result;
 	int ret = 0;
 
-	if (compressed_size < sizeof(u32)) {
-	       msg("%"TS": not in gzip format", in->name);
-	       ret = -1;
-	       goto out;
+	if (compressed_size < GZIP_MIN_OVERHEAD ||
+	    compressed_data[0] != GZIP_ID1 ||
+	    compressed_data[1] != GZIP_ID2) {
+		if (options->force)
+			return full_write(out, compressed_data, compressed_size);
+		msg("%"TS": not in gzip format", in->name);
+		return -1;
 	}
 
+	/*
+	 * Use the ISIZE field as a hint for the decompressed data size.  It may
+	 * need to be increased later, however, because the file may contain
+	 * multiple gzip members and the particular ISIZE we happen to use may
+	 * not be the largest; or the real size may be >= 4 GiB, causing ISIZE
+	 * to overflow.  In any case, make sure to allocate at least one byte.
+	 */
 	uncompressed_size = load_u32_gzip(&compressed_data[compressed_size - 4]);
+	if (uncompressed_size == 0)
+		uncompressed_size = 1;
+
+	/*
+	 * DEFLATE cannot expand data more than 1032x, so there's no need to
+	 * ever allocate a buffer more than 1032 times larger than the
+	 * compressed data.  This is a fail-safe, albeit not a very good one, if
+	 * ISIZE becomes corrupted on a small file.  (The 1032x number comes
+	 * from each 2 bits generating a 258-byte match.  This is a hard upper
+	 * bound; the real upper bound is slightly smaller due to overhead.)
+	 */
+	if (compressed_size <= SIZE_MAX / 1032)
+		max_uncompressed_size = compressed_size * 1032;
+	else
+		max_uncompressed_size = SIZE_MAX;
 
 	do {
 		if (uncompressed_data == NULL) {
+			uncompressed_size = MIN(uncompressed_size,
+						max_uncompressed_size);
 			uncompressed_data = xmalloc(uncompressed_size);
 			if (uncompressed_data == NULL) {
 				msg("%"TS": file is probably too large to be "
@@ -222,6 +262,11 @@ do_decompress(struct libdeflate_decompressor *decompressor,
 						       &actual_out_nbytes);
 
 		if (result == LIBDEFLATE_INSUFFICIENT_SPACE) {
+			if (uncompressed_size >= max_uncompressed_size) {
+				msg("Bug in libdeflate_gzip_decompress_ex(): data expanded too much!");
+				ret = -1;
+				goto out;
+			}
 			if (uncompressed_size * 2 <= uncompressed_size) {
 				msg("%"TS": file corrupt or too large to be "
 				    "processed by this program", in->name);
@@ -244,14 +289,16 @@ do_decompress(struct libdeflate_decompressor *decompressor,
 		if (actual_in_nbytes == 0 ||
 		    actual_in_nbytes > compressed_size ||
 		    actual_out_nbytes > uncompressed_size) {
-			msg("Bug in libdeflate_gzip_decompress_ex()!");
+			msg("Bug in libdeflate_gzip_decompress_ex(): impossible actual_nbytes value!");
 			ret = -1;
 			goto out;
 		}
 
-		ret = full_write(out, uncompressed_data, actual_out_nbytes);
-		if (ret != 0)
-			goto out;
+		if (!options->test) {
+			ret = full_write(out, uncompressed_data, actual_out_nbytes);
+			if (ret != 0)
+				goto out;
+		}
 
 		compressed_data += actual_in_nbytes;
 		compressed_size -= actual_in_nbytes;
@@ -271,15 +318,15 @@ stat_file(struct file_stream *in, stat_t *stbuf, bool allow_hard_links)
 	}
 
 	if (!S_ISREG(stbuf->st_mode) && !in->is_standard_stream) {
-		msg("%"TS" is %s -- skipping",
-		    in->name, S_ISDIR(stbuf->st_mode) ? "a directory" :
-							"not a regular file");
+		warn("%"TS" is %s -- skipping",
+		     in->name, S_ISDIR(stbuf->st_mode) ? "a directory" :
+							 "not a regular file");
 		return -2;
 	}
 
 	if (stbuf->st_nlink > 1 && !allow_hard_links) {
-		msg("%"TS" has multiple hard links -- skipping "
-		    "(use -f to process anyway)", in->name);
+		warn("%"TS" has multiple hard links -- skipping (use -f to process anyway)",
+		     in->name);
 		return -2;
 	}
 
@@ -370,9 +417,8 @@ decompress_file(struct libdeflate_decompressor *decompressor, const tchar *path,
 				if (!options->to_stdout)
 					newpath = (tchar *)path;
 			} else if (!options->to_stdout) {
-				msg("\"%"TS"\" does not end with the %"TS" "
-				    "suffix -- skipping",
-				    path, options->suffix);
+				warn("\"%"TS"\" does not end with the %"TS" suffix -- skipping",
+				     path, options->suffix);
 				return -2;
 			}
 		} else if (!options->to_stdout) {
@@ -416,7 +462,7 @@ decompress_file(struct libdeflate_decompressor *decompressor, const tchar *path,
 	if (ret != 0)
 		goto out_close_out;
 
-	ret = do_decompress(decompressor, &in, &out);
+	ret = do_decompress(decompressor, &in, &out, options);
 	if (ret != 0)
 		goto out_close_out;
 
@@ -519,12 +565,13 @@ tmain(int argc, tchar *argv[])
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	options.to_stdout = false;
 	options.decompress = is_gunzip();
 	options.force = false;
 	options.keep = false;
+	options.test = false;
 	options.compression_level = 6;
 	options.suffix = T(".gz");
 
@@ -541,7 +588,7 @@ tmain(int argc, tchar *argv[])
 		case '9':
 			options.compression_level =
 				parse_compression_level(opt_char, toptarg);
-			if (options.compression_level == 0)
+			if (options.compression_level < 0)
 				return 1;
 			break;
 		case 'c':
@@ -567,6 +614,9 @@ tmain(int argc, tchar *argv[])
 			 *  option as a no-op.
 			 */
 			break;
+		case 'q':
+			suppress_warnings = true;
+			break;
 		case 'S':
 			options.suffix = toptarg;
 			if (options.suffix[0] == T('\0')) {
@@ -574,6 +624,17 @@ tmain(int argc, tchar *argv[])
 				return 1;
 			}
 			break;
+		case 't':
+			options.test = true;
+			options.decompress = true;
+			options.to_stdout = true;
+			/*
+			 * -t behaves just like the more commonly used -c
+			 * option, except that -t doesn't actually write
+			 * anything.  For ease of implementation, just pretend
+			 * that -c was specified too.
+			 */
+			break;
 		case 'V':
 			show_version();
 			return 0;
@@ -620,13 +681,17 @@ tmain(int argc, tchar *argv[])
 		libdeflate_free_compressor(c);
 	}
 
-	/*
-	 * If ret=0, there were no warnings or errors.  Exit with status 0.
-	 * If ret=2, there was at least one warning.  Exit with status 2.
-	 * Else, there was at least one error.  Exit with status 1.
-	 */
-	if (ret != 0 && ret != 2)
-		ret = 1;
-
-	return ret;
+	switch (ret) {
+	case 0:
+		/* No warnings or errors */
+		return 0;
+	case 2:
+		/* At least one warning, but no errors */
+		if (suppress_warnings)
+			return 0;
+		return 2;
+	default:
+		/* At least one error */
+		return 1;
+	}
 }

programs/prog_util.c

@@ -30,13 +30,11 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdarg.h>
-#include <time.h>
 #ifdef _WIN32
 #  include <windows.h>
 #else
 #  include <unistd.h>
 #  include <sys/mman.h>
-#  include <sys/time.h>
 #endif
 
 #ifndef O_BINARY
@@ -56,14 +54,17 @@
 #endif
 
 /* The invocation name of the program (filename component only) */
-const tchar *program_invocation_name;
+const tchar *prog_invocation_name;
+
+/* Whether to suppress warning messages or not */
+bool suppress_warnings;
 
 static void
 do_msg(const char *format, bool with_errno, va_list va)
 {
 	int saved_errno = errno;
 
-	fprintf(stderr, "%"TS": ", program_invocation_name);
+	fprintf(stderr, "%"TS": ", prog_invocation_name);
 	vfprintf(stderr, format, va);
 	if (with_errno)
 		fprintf(stderr, ": %s\n", strerror(saved_errno));
@@ -95,6 +96,20 @@ msg_errno(const char *format, ...)
 	va_end(va);
 }
 
+
+/* Same as msg(), but do nothing if 'suppress_warnings' has been set. */
+void
+warn(const char *format, ...)
+{
+	if (!suppress_warnings) {
+		va_list va;
+
+		va_start(va, format);
+		do_msg(format, false, va);
+		va_end(va);
+	}
+}
+
 /* malloc() wrapper */
 void *
 xmalloc(size_t size)
@@ -107,61 +122,6 @@ xmalloc(size_t size)
 	return p;
 }
 
-/*
- * Return the number of timer ticks that have elapsed since some unspecified
- * point fixed at the start of program execution
- */
-u64
-timer_ticks(void)
-{
-#ifdef _WIN32
-	LARGE_INTEGER count;
-	QueryPerformanceCounter(&count);
-	return count.QuadPart;
-#elif defined(HAVE_CLOCK_GETTIME)
-	struct timespec ts;
-	clock_gettime(CLOCK_MONOTONIC, &ts);
-	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
-#else
-	struct timeval tv;
-	gettimeofday(&tv, NULL);
-	return (1000000 * (u64)tv.tv_sec) + tv.tv_usec;
-#endif
-}
-
-/*
- * Return the number of timer ticks per second
- */
-static u64
-timer_frequency(void)
-{
-#ifdef _WIN32
-	LARGE_INTEGER freq;
-	QueryPerformanceFrequency(&freq);
-	return freq.QuadPart;
-#elif defined(HAVE_CLOCK_GETTIME)
-	return 1000000000;
-#else
-	return 1000000;
-#endif
-}
-
-/*
- * Convert a number of elapsed timer ticks to milliseconds
- */
-u64 timer_ticks_to_ms(u64 ticks)
-{
-	return ticks * 1000 / timer_frequency();
-}
-
-/*
- * Convert a byte count and a number of elapsed timer ticks to MB/s
- */
-u64 timer_MB_per_s(u64 bytes, u64 ticks)
-{
-	return bytes * timer_frequency() / ticks / 1000000;
-}
-
 /*
  * Retrieve a pointer to the filename component of the specified path.
  *
@@ -169,7 +129,7 @@ u64 timer_MB_per_s(u64 bytes, u64 ticks)
  * properly for directories, since a path to a directory might have trailing
  * slashes.
  */
-const tchar *
+static const tchar *
 get_filename(const tchar *path)
 {
 	const tchar *slash = tstrrchr(path, '/');
@@ -183,6 +143,17 @@ get_filename(const tchar *path)
 	return path;
 }
 
+void
+begin_program(tchar *argv[])
+{
+	prog_invocation_name = get_filename(argv[0]);
+
+#ifdef FREESTANDING
+	/* This allows testing freestanding library builds. */
+	libdeflate_set_memory_allocator(malloc, free);
+#endif
+}
+
 /* Create a copy of 'path' surrounded by double quotes */
 static tchar *
 quote_path(const tchar *path)
@@ -232,7 +203,7 @@ xopen_for_read(const tchar *path, bool symlink_ok, struct file_stream *strm)
 	}
 
 #if defined(HAVE_POSIX_FADVISE) && (O_SEQUENTIAL == 0)
-	posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+	(void)posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
 #endif
 
 	return 0;
@@ -272,14 +243,14 @@ retry:
 		}
 		if (!overwrite) {
 			if (!isatty(STDERR_FILENO) || !isatty(STDIN_FILENO)) {
-				msg("%"TS" already exists; use -f to overwrite",
-				    strm->name);
+				warn("%"TS" already exists; use -f to overwrite",
+				     strm->name);
 				ret = -2; /* warning only */
 				goto err;
 			}
 			fprintf(stderr, "%"TS": %"TS" already exists; "
 				"overwrite? (y/n) ",
-				program_invocation_name, strm->name);
+				prog_invocation_name, strm->name);
 			if (getchar() != 'y') {
 				msg("Not overwriting.");
 				goto err;
@@ -393,7 +364,7 @@ map_file_contents(struct file_stream *strm, u64 size)
 	}
 
 #ifdef HAVE_POSIX_MADVISE
-	posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL);
+	(void)posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL);
 #endif
 	strm->mmap_token = strm; /* anything that's not NULL */
 
@@ -479,27 +450,39 @@ xclose(struct file_stream *strm)
 
 /*
  * Parse the compression level given on the command line, returning the
- * compression level on success or 0 on error
+ * compression level on success or -1 on error
  */
 int
 parse_compression_level(tchar opt_char, const tchar *arg)
 {
-	unsigned long level = opt_char - '0';
-	const tchar *p;
+	int level;
 
 	if (arg == NULL)
 		arg = T("");
 
-	for (p = arg; *p >= '0' && *p <= '9'; p++)
-		level = (level * 10) + (*p - '0');
+	if (opt_char < '0' || opt_char > '9')
+		goto invalid;
+	level = opt_char - '0';
 
-	if (level < 1 || level > 12 || *p != '\0') {
-		msg("Invalid compression level: \"%"TC"%"TS"\".  "
-		    "Must be an integer in the range [1, 12].", opt_char, arg);
-		return 0;
+	if (arg[0] != '\0') {
+		if (arg[0] < '0' || arg[0] > '9')
+			goto invalid;
+		if (arg[1] != '\0')	/* Levels are at most 2 digits */
+			goto invalid;
+		if (level == 0)		/* Don't allow arguments like "-01" */
+			goto invalid;
+		level = (level * 10) + (arg[0] - '0');
 	}
 
+	if (level < 0 || level > 12)
+		goto invalid;
+
 	return level;
+
+invalid:
+	msg("Invalid compression level: \"%"TC"%"TS"\".  "
+	    "Must be an integer in the range [0, 12].", opt_char, arg);
+	return -1;
 }
 
 /* Allocate a new DEFLATE compressor */

programs/prog_util.h

@@ -38,8 +38,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#ifndef _WIN32
+#  include <sys/types.h>
+#endif
 
-#include "common_defs.h"
+#include "../common/common_defs.h"
 
 #ifdef __GNUC__
 # define _printf(str_idx, args_idx)	\
@@ -48,6 +51,27 @@
 # define _printf(str_idx, args_idx)
 #endif
 
+#ifdef _MSC_VER
+/*
+ * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
+ * inttypes.h.  Work around this by defining the PRI* macros ourselves.
+ */
+# define PRIu8  "hhu"
+# define PRIu16 "hu"
+# define PRIu32 "u"
+# define PRIu64 "llu"
+# define PRIi8  "hhi"
+# define PRIi16 "hi"
+# define PRIi32 "i"
+# define PRIi64 "lli"
+# define PRIx8  "hhx"
+# define PRIx16 "hx"
+# define PRIx32 "x"
+# define PRIx64 "llx"
+#else
+# include <inttypes.h>
+#endif
+
 #ifdef _WIN32
 
 /*
@@ -57,7 +81,7 @@
  */
 
 #include <wchar.h>
-extern int wmain(int argc, wchar_t **argv);
+int wmain(int argc, wchar_t **argv);
 #  define	tmain		wmain
 #  define	tchar		wchar_t
 #  define	_T(text)	L##text
@@ -68,7 +92,6 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		_wopen
 #  define	tstrchr		wcschr
 #  define	tstrcmp		wcscmp
-#  define	tstrcpy		wcscpy
 #  define	tstrlen		wcslen
 #  define	tstrrchr	wcsrchr
 #  define	tstrtoul	wcstoul
@@ -100,7 +123,6 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		open
 #  define	tstrchr		strchr
 #  define	tstrcmp		strcmp
-#  define	tstrcpy		strcpy
 #  define	tstrlen		strlen
 #  define	tstrrchr	strrchr
 #  define	tstrtoul	strtoul
@@ -114,18 +136,16 @@ extern int wmain(int argc, wchar_t **argv);
 
 #endif /* !_WIN32 */
 
-extern const tchar *program_invocation_name;
+extern const tchar *prog_invocation_name;
+extern bool suppress_warnings;
 
-extern void _printf(1, 2) msg(const char *fmt, ...);
-extern void _printf(1, 2) msg_errno(const char *fmt, ...);
+void _printf(1, 2) msg(const char *fmt, ...);
+void _printf(1, 2) msg_errno(const char *fmt, ...);
+void _printf(1, 2) warn(const char *fmt, ...);
 
-extern void *xmalloc(size_t size);
+void *xmalloc(size_t size);
 
-extern u64 timer_ticks(void);
-extern u64 timer_ticks_to_ms(u64 ticks);
-extern u64 timer_MB_per_s(u64 bytes, u64 ticks);
-
-extern const tchar *get_filename(const tchar *path);
+void begin_program(tchar *argv[]);
 
 struct file_stream {
 	int fd;
@@ -136,27 +156,26 @@ struct file_stream {
 	size_t mmap_size;
 };
 
-extern int xopen_for_read(const tchar *path, bool symlink_ok,
-			  struct file_stream *strm);
-extern int xopen_for_write(const tchar *path, bool force,
-			   struct file_stream *strm);
-extern int map_file_contents(struct file_stream *strm, u64 size);
+int xopen_for_read(const tchar *path, bool symlink_ok,
+		   struct file_stream *strm);
+int xopen_for_write(const tchar *path, bool force, struct file_stream *strm);
+int map_file_contents(struct file_stream *strm, u64 size);
 
-extern ssize_t xread(struct file_stream *strm, void *buf, size_t count);
-extern int full_write(struct file_stream *strm, const void *buf, size_t count);
+ssize_t xread(struct file_stream *strm, void *buf, size_t count);
+int full_write(struct file_stream *strm, const void *buf, size_t count);
 
-extern int xclose(struct file_stream *strm);
+int xclose(struct file_stream *strm);
 
-extern int parse_compression_level(tchar opt_char, const tchar *arg);
+int parse_compression_level(tchar opt_char, const tchar *arg);
 
-extern struct libdeflate_compressor *alloc_compressor(int level);
-extern struct libdeflate_decompressor *alloc_decompressor(void);
+struct libdeflate_compressor *alloc_compressor(int level);
+struct libdeflate_decompressor *alloc_decompressor(void);
 
 /* tgetopt.c */
 
 extern tchar *toptarg;
 extern int toptind, topterr, toptopt;
 
-extern int tgetopt(int argc, tchar *argv[], const tchar *optstring);
+int tgetopt(int argc, tchar *argv[], const tchar *optstring);
 
 #endif /* PROGRAMS_PROG_UTIL_H */

programs/test_checksums.c

@@ -7,32 +7,33 @@
 
 #include <stdlib.h>
 #include <time.h>
-#include <zlib.h>
 
-#include "prog_util.h"
+#include "test_util.h"
 
 static unsigned int rng_seed;
 
-static void
-assertion_failed(const char *file, int line)
-{
-	fprintf(stderr, "Assertion failed at %s:%d\n", file, line);
-	fprintf(stderr, "RNG seed was %u\n", rng_seed);
-	abort();
-}
-
-#define ASSERT(expr) if (!(expr)) assertion_failed(__FILE__, __LINE__);
-
 typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
 
 static u32
-zlib_adler32(u32 adler, const void *buf, size_t len)
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
+}
+
+static u32
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
 {
 	return adler32(adler, buf, len);
 }
 
 static u32
-zlib_crc32(u32 crc, const void *buf, size_t len)
+crc32_zlib(u32 crc, const void *buf, size_t len)
 {
 	return crc32(crc, buf, len);
 }
@@ -62,7 +63,7 @@ static void
 test_initial_values(cksum_fn_t cksum, u32 expected)
 {
 	ASSERT(cksum(0, NULL, 0) == expected);
-	if (cksum != zlib_adler32) /* broken */
+	if (cksum != adler32_zlib) /* broken */
 		ASSERT(cksum(0, NULL, 1) == expected);
 	ASSERT(cksum(0, NULL, 1234) == expected);
 	ASSERT(cksum(1234, NULL, 0) == expected);
@@ -111,47 +112,61 @@ static void
 test_crc32(const void *buffer, size_t size, u32 initial_value)
 {
 	test_checksums(buffer, size, "CRC-32",
-		       libdeflate_crc32, zlib_crc32, initial_value);
+		       crc32_libdeflate, crc32_zlib, initial_value);
 }
 
 static void
 test_adler32(const void *buffer, size_t size, u32 initial_value)
 {
 	test_checksums(buffer, size, "Adler-32",
-		       libdeflate_adler32, zlib_adler32, initial_value);
+		       adler32_libdeflate, adler32_zlib, initial_value);
 }
 
-static void test_random_buffers(u8 *buffer, size_t limit, u32 num_iter)
+static void test_random_buffers(u8 *buffer, u8 *guarded_buf_end,
+				size_t limit, u32 num_iter)
 {
 	for (u32 i = 0; i < num_iter; i++) {
 		size_t start = rand() % limit;
 		size_t len = rand() % (limit - start);
+		u32 a0 = select_initial_adler();
+		u32 c0 = select_initial_crc();
 
 		for (size_t j = start; j < start + len; j++)
 			buffer[j] = rand();
 
-		test_adler32(&buffer[start], len, select_initial_adler());
-		test_crc32(&buffer[start], len, select_initial_crc());
+		/* Test with chosen size and alignment */
+		test_adler32(&buffer[start], len, a0);
+		test_crc32(&buffer[start], len, c0);
+
+		/* Test with chosen size, with guard page after input buffer */
+		memcpy(guarded_buf_end - len, &buffer[start], len);
+		test_adler32(guarded_buf_end - len, len, a0);
+		test_crc32(guarded_buf_end - len, len, c0);
 	}
 }
 
 int
 tmain(int argc, tchar *argv[])
 {
-	u8 *buffer = malloc(32768);
+	u8 *buffer = xmalloc(32768);
+	u8 *guarded_buf_start, *guarded_buf_end;
+
+	begin_program(argv);
+
+	alloc_guarded_buffer(32768, &guarded_buf_start, &guarded_buf_end);
 
 	rng_seed = time(NULL);
 	srand(rng_seed);
 
-	test_initial_values(libdeflate_adler32, 1);
-	test_initial_values(zlib_adler32, 1);
-	test_initial_values(libdeflate_crc32, 0);
-	test_initial_values(zlib_crc32, 0);
+	test_initial_values(adler32_libdeflate, 1);
+	test_initial_values(adler32_zlib, 1);
+	test_initial_values(crc32_libdeflate, 0);
+	test_initial_values(crc32_zlib, 0);
 
 	/* Test different buffer sizes and alignments */
-	test_random_buffers(buffer, 256, 5000);
-	test_random_buffers(buffer, 1024, 500);
-	test_random_buffers(buffer, 32768, 50);
+	test_random_buffers(buffer, guarded_buf_end, 256, 5000);
+	test_random_buffers(buffer, guarded_buf_end, 1024, 500);
+	test_random_buffers(buffer, guarded_buf_end, 32768, 50);
 
 	/*
 	 * Test Adler-32 overflow cases.  For example, given all 0xFF bytes and
@@ -175,8 +190,7 @@ tmain(int argc, tchar *argv[])
 		buffer[rand() % 32768] = 0xFE;
 	}
 
-	printf("Adler-32 and CRC-32 checksum tests passed!\n");
-
 	free(buffer);
+	free_guarded_buffer(guarded_buf_start, guarded_buf_end);
 	return 0;
 }

programs/test_custom_malloc.c

@@ -0,0 +1,85 @@
+/*
+ * test_custom_malloc.c
+ *
+ * Test libdeflate_set_memory_allocator().
+ * Also test injecting allocation failures.
+ */
+
+#include "test_util.h"
+
+static int malloc_count = 0;
+static int free_count = 0;
+
+static void *do_malloc(size_t size)
+{
+	malloc_count++;
+	return malloc(size);
+}
+
+static void *do_fail_malloc(size_t size)
+{
+	malloc_count++;
+	return NULL;
+}
+
+static void do_free(void *ptr)
+{
+	free_count++;
+	free(ptr);
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	int level;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+
+	begin_program(argv);
+
+	/* Test that the custom allocator is actually used when requested. */
+
+	libdeflate_set_memory_allocator(do_malloc, do_free);
+	ASSERT(malloc_count == 0);
+	ASSERT(free_count == 0);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c != NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+		libdeflate_free_compressor(c);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 1);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+	libdeflate_free_decompressor(d);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 1);
+
+	/* As long as we're here, also test injecting allocation failures. */
+
+	libdeflate_set_memory_allocator(do_fail_malloc, do_free);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c == NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d == NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+
+	return 0;
+}

programs/test_incomplete_codes.c

@@ -0,0 +1,385 @@
+/*
+ * test_incomplete_codes.c
+ *
+ * Test that the decompressor accepts incomplete Huffman codes in certain
+ * specific cases.
+ */
+
+#include "test_util.h"
+
+static void
+verify_decompression_libdeflate(const u8 *in, size_t in_nbytes,
+				u8 *out, size_t out_nbytes_avail,
+				const u8 *expected_out,
+				size_t expected_out_nbytes)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	size_t actual_out_nbytes;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	res = libdeflate_deflate_decompress(d, in, in_nbytes,
+					    out, out_nbytes_avail,
+					    &actual_out_nbytes);
+	ASSERT(res == LIBDEFLATE_SUCCESS);
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	libdeflate_free_decompressor(d);
+}
+
+static void
+verify_decompression_zlib(const u8 *in, size_t in_nbytes,
+			  u8 *out, size_t out_nbytes_avail,
+			  const u8 *expected_out, size_t expected_out_nbytes)
+{
+	z_stream z;
+	int res;
+	size_t actual_out_nbytes;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	z.next_in = (void *)in;
+	z.avail_in = in_nbytes;
+	z.next_out = (void *)out;
+	z.avail_out = out_nbytes_avail;
+	res = inflate(&z, Z_FINISH);
+	ASSERT(res == Z_STREAM_END);
+	actual_out_nbytes = out_nbytes_avail - z.avail_out;
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	inflateEnd(&z);
+}
+
+static void
+verify_decompression(const u8 *in, size_t in_nbytes,
+		     u8 *out, size_t out_nbytes_avail,
+		     const u8 *expected_out, size_t expected_out_nbytes)
+{
+	verify_decompression_libdeflate(in, in_nbytes, out, out_nbytes_avail,
+					expected_out, expected_out_nbytes);
+	verify_decompression_zlib(in, in_nbytes, out, out_nbytes_avail,
+				  expected_out, expected_out_nbytes);
+
+}
+
+/* Test that an empty offset code is accepted. */
+static void
+test_empty_offset_code(void)
+{
+	static const u8 expected_out[] = { 'A', 'B', 'A', 'A' };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Generate a DEFLATE stream containing a "dynamic Huffman" block
+	 * containing literals, but no offsets; and having an empty offset code
+	 * (all codeword lengths set to 0).
+	 *
+	 * Litlen code:
+	 *	litlensym_A			freq=3 len=1 codeword= 0
+	 *	litlensym_B			freq=1 len=2 codeword=01
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..'A'-1]	= 0	presym_18
+	 *	['A']		= 1	presym_1
+	 *	['B']		= 2	presym_2
+	 *	['B'+1..255]	= 0	presym_18 presym_18
+	 *	[256]		= 2	presym_2
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=3 codeword=011
+	 *	presym_1	freq=1 len=3 codeword=111
+	 *	presym_2	freq=2 len=2 codeword= 01
+	 *	presym_18	freq=3 len=1 codeword=  0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_0: len=3 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_1: len=3 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 54, 7));	/* presym_18, 65 zeroes */
+	ASSERT(put_bits(&os, 0x7, 3));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 89, 7));	/* presym_18, 100 zeroes */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 78, 7));	/* presym_18, 89 zeroes */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 3));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_B */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that a litrunlen code containing only one symbol is accepted. */
+static void
+test_singleton_litrunlen_code(void)
+{
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..256]	= 0	presym_18 presym_18
+	 *	[256]		= 1	presym_1
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=2 codeword=01
+	 *	presym_1	freq=1 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 13; i++)	/* presym_{8,...,14}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		ASSERT(put_bits(&os, 0, 1) &&	/* presym_18, 128 zeroes */
+		       put_bits(&os, 117, 7));
+	}
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out), in, 0);
+}
+
+/* Test that an offset code containing only one symbol is accepted. */
+static void
+test_singleton_offset_code(void)
+{
+	static const u8 expected_out[] = { 255, 255, 255, 255 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_255			freq=1 len=1 codeword= 0
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=01
+	 *	litlensym_257 (len 3)		freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_0 (offset 0)		freq=1 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..254] = 0	presym_{18,18}
+	 *	[255]	 = 1	presym_1
+	 *	[256]	 = 1	presym_2
+	 *	[257]	 = 1	presym_2
+	 *	[258]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=2 codeword=01
+	 *	presym_2	freq=2 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	for (i = 0; i < 12; i++)	/* presym_{0,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 127 zeroes */
+	       put_bits(&os, 116, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+
+	/* Literal */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_0 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that an offset code containing only one symbol is accepted, even if that
+ * symbol is not symbol 0.  The codeword should be '0' in either case. */
+static void
+test_singleton_offset_code_notsymzero(void)
+{
+	static const u8 expected_out[] = { 254, 255, 254, 255, 254 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_254			len=2 codeword=00
+	 *	litlensym_255			len=2 codeword=10
+	 *	litlensym_256 (end-of-block)	len=2 codeword=01
+	 *	litlensym_257 (len 3)		len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_1 (offset 2)		len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..253] = 0	presym_{18,18}
+	 *	[254]	 = 2	presym_2
+	 *	[255]	 = 2	presym_2
+	 *	[256]	 = 2	presym_2
+	 *	[257]	 = 2	presym_2
+	 *	[258]	 = 0	presym_0
+	 *	[259]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_0	len=2 codeword=00
+	 *	presym_1	len=2 codeword=10
+	 *	presym_2	len=2 codeword=01
+	 *	presym_18	len=2 codeword=11
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 1, 5));	/* num_offset_syms: 1 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_18: len=2 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 126 zeroes */
+	       put_bits(&os, 115, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 2));	/* presym_0 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* presym_1 */
+
+	/* Literals */
+	ASSERT(put_bits(&os, 0x0, 2));	/* litlensym_254 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_1 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	begin_program(argv);
+
+	test_empty_offset_code();
+	test_singleton_litrunlen_code();
+	test_singleton_offset_code();
+	test_singleton_offset_code_notsymzero();
+
+	return 0;
+}

programs/test_litrunlen_overflow.c

@@ -0,0 +1,72 @@
+/*
+ * test_litrunlen_overflow.c
+ *
+ * Regression test for commit f2f0df727444 ("deflate_compress: fix corruption
+ * with long literal run").  Try to compress a file longer than 65535 bytes
+ * where no 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes
+ * apart, and the distribution of bytes remains constant throughout, and yet not
+ * all bytes are used so the data is still slightly compressible.  There will be
+ * no matches in this data, but the compressor should still output a compressed
+ * block, and this block should contain more than 65535 consecutive literals,
+ * which triggered the bug.
+ *
+ * Note: on random data, this situation is extremely unlikely if the compressor
+ * uses all matches it finds, since random data will on average have a 3-byte
+ * match every (256**3)/32768 = 512 bytes.
+ */
+
+#include "test_util.h"
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t data_size = 2 * 250 * 251;
+	u8 *orig_data, *compressed_data, *decompressed_data;
+	int i, stride, multiple, j = 0;
+	struct libdeflate_decompressor *d;
+	static const int levels[] = { 3, 6, 12 };
+
+	begin_program(argv);
+
+	orig_data = xmalloc(data_size);
+	compressed_data = xmalloc(data_size);
+	decompressed_data = xmalloc(data_size);
+
+	for (i = 0; i < 2; i++) {
+		for (stride = 1; stride < 251; stride++) {
+			for (multiple = 0; multiple < 251; multiple++)
+				orig_data[j++] = (stride * multiple) % 251;
+		}
+	}
+	ASSERT(j == data_size);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(levels); i++) {
+		struct libdeflate_compressor *c;
+		size_t csize;
+		enum libdeflate_result res;
+
+		c = libdeflate_alloc_compressor(levels[i]);
+		ASSERT(c != NULL);
+
+		csize = libdeflate_deflate_compress(c, orig_data, data_size,
+						    compressed_data, data_size);
+		ASSERT(csize > 0 && csize < data_size);
+
+		res = libdeflate_deflate_decompress(d, compressed_data, csize,
+						    decompressed_data,
+						    data_size, NULL);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(memcmp(orig_data, decompressed_data, data_size) == 0);
+
+		libdeflate_free_compressor(c);
+	}
+
+	libdeflate_free_decompressor(d);
+	free(orig_data);
+	free(compressed_data);
+	free(decompressed_data);
+	return 0;
+}

programs/test_overread.c

@@ -0,0 +1,95 @@
+/*
+ * test_overread.c
+ *
+ * Test that the decompressor doesn't produce an unbounded amount of output if
+ * it runs out of input, even when implicit zeroes appended to the input would
+ * continue producing output (as is the case when the input ends during a
+ * DYNAMIC_HUFFMAN block where a literal has an all-zeroes codeword).
+ *
+ * This is a regression test for commit 3f21ec9d6121 ("deflate_decompress: error
+ * out if overread count gets too large").
+ */
+
+#include "test_util.h"
+
+static void
+generate_test_input(struct output_bitstream *os)
+{
+	int i;
+
+	put_bits(os, 0, 1);	/* BFINAL: 0 */
+	put_bits(os, 2, 2);	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Write the Huffman codes.
+	 *
+	 * Litlen code:
+	 *	litlensym_0   (0)		len=1 codeword=0
+	 *	litlensym_256 (end-of-block)	len=1 codeword=1
+	 * Offset code:
+	 *	offsetsym_0 (unused)		len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0]	 = 1	presym_1
+	 *	[1..255] = 0	presym_{18,18}
+	 *	[256]	 = 1	presym_1
+	 *	[257]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	len=1 codeword=0
+	 *	presym_18	len=1 codeword=1
+	 */
+	put_bits(os, 0, 5);	/* num_litlen_syms: 0 + 257 */
+	put_bits(os, 0, 5);	/* num_offset_syms: 0 + 1 */
+	put_bits(os, 14, 4);	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	put_bits(os, 0, 3);		/* presym_16: len=0 */
+	put_bits(os, 0, 3);		/* presym_17: len=0 */
+	put_bits(os, 1, 3);		/* presym_18: len=1 */
+	for (i = 0; i < 14; i++)	/* presym_{0,...,14}: len=0 */
+		put_bits(os, 0, 3);
+	put_bits(os, 1, 3);		/* presym_1: len=1 */
+
+	/* Litlen and offset codeword lengths */
+	put_bits(os, 0, 1);		/* presym_1 */
+	put_bits(os, 1, 1);		/* presym_18 ... */
+	put_bits(os, 117, 7);		/* ... 11 + 117 zeroes */
+	put_bits(os, 1, 1);		/* presym_18 ... */
+	put_bits(os, 116, 7);		/* ... 11 + 116 zeroes */
+	put_bits(os, 0, 1);		/* presym_1 */
+	put_bits(os, 0, 1);		/* presym_1 */
+
+	/* Implicit zeroes would generate endless literals from here. */
+
+	ASSERT(flush_bits(os));
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 cdata[16];
+	u8 udata[256];
+	struct output_bitstream os =
+		{ .next = cdata, .end = cdata + sizeof(cdata) };
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	size_t actual_out_nbytes;
+
+	begin_program(argv);
+
+	generate_test_input(&os);
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	res = libdeflate_deflate_decompress(d, cdata, os.next - cdata,
+					    udata, sizeof(udata),
+					    &actual_out_nbytes);
+	/* Before the fix, the result was LIBDEFLATE_INSUFFICIENT_SPACE here. */
+	ASSERT(res == LIBDEFLATE_BAD_DATA);
+
+	libdeflate_free_decompressor(d);
+	return 0;
+}

programs/test_slow_decompression.c

@@ -0,0 +1,472 @@
+/*
+ * test_slow_decompression.c
+ *
+ * Test how quickly libdeflate decompresses degenerate/malicious compressed data
+ * streams that start new Huffman blocks extremely frequently.
+ */
+
+#include "test_util.h"
+
+/*
+ * Generate a DEFLATE stream containing all empty "static Huffman" blocks.
+ *
+ * libdeflate used to decompress this very slowly (~1000x slower than typical
+ * data), but now it's much faster (only ~2x slower than typical data) because
+ * now it skips rebuilding the decode tables for the static Huffman codes when
+ * they're already loaded into the decompressor.
+ */
+static void
+generate_empty_static_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (put_bits(&os, 0, 1) &&	/* BFINAL: 0 */
+	       put_bits(&os, 1, 2) &&	/* BTYPE: STATIC_HUFFMAN */
+	       put_bits(&os, 0, 7))	/* litlensym_256 (end-of-block) */
+		;
+}
+
+static bool
+generate_empty_dynamic_huffman_block(struct output_bitstream *os)
+{
+	int i;
+
+	if (!put_bits(os, 0, 1))	/* BFINAL: 0 */
+		return false;
+	if (!put_bits(os, 2, 2))	/* BTYPE: DYNAMIC_HUFFMAN */
+		return false;
+
+	/*
+	 * Write a minimal Huffman code, then the end-of-block symbol.
+	 *
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	offsetsym_0 (unused)		freq=0 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..255] = 0	presym_{18,18}
+	 *	[256]	 = 1	presym_1
+	 *	[257]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=1 codeword=0
+	 *	presym_18	freq=2 len=1 codeword=1
+	 */
+
+	if (!put_bits(os, 0, 5))	/* num_litlen_syms: 0 + 257 */
+		return false;
+	if (!put_bits(os, 0, 5))	/* num_offset_syms: 0 + 1 */
+		return false;
+	if (!put_bits(os, 14, 4))	/* num_explicit_precode_lens: 14 + 4 */
+		return false;
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++) {	/* presym_{16,17}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_18: len=1 */
+		return false;
+	for (i = 0; i < 14; i++) {	/* presym_{0,...,14}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_1: len=1 */
+		return false;
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		if (!put_bits(os, 1, 1) || /* presym_18, 128 zeroes */
+		    !put_bits(os, 117, 7))
+			return false;
+	}
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	/* Done writing the Huffman codes */
+
+	return put_bits(os, 0, 1);	/* litlensym_256 (end-of-block) */
+}
+
+/*
+ * Generate a DEFLATE stream containing all empty "dynamic Huffman" blocks.
+ *
+ * This is the worst known case currently, being ~100x slower to decompress than
+ * typical data.
+ */
+static void
+generate_empty_dynamic_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (generate_empty_dynamic_huffman_block(&os))
+		;
+}
+
+#define NUM_ITERATIONS	100
+
+static u64
+do_test_libdeflate(const char *input_type, const u8 *in, size_t in_nbytes,
+		   u8 *out, size_t out_nbytes_avail)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	u64 t;
+	int i;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		res = libdeflate_deflate_decompress(d, in, in_nbytes, out,
+						    out_nbytes_avail, NULL);
+		ASSERT(res == LIBDEFLATE_BAD_DATA ||
+		       res == LIBDEFLATE_INSUFFICIENT_SPACE);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, libdeflate]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	libdeflate_free_decompressor(d);
+	return t;
+}
+
+static u64
+do_test_zlib(const char *input_type, const u8 *in, size_t in_nbytes,
+	     u8 *out, size_t out_nbytes_avail)
+{
+	z_stream z;
+	int res;
+	u64 t;
+	int i;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		inflateReset(&z);
+		z.next_in = (void *)in;
+		z.avail_in = in_nbytes;
+		z.next_out = out;
+		z.avail_out = out_nbytes_avail;
+		res = inflate(&z, Z_FINISH);
+		ASSERT(res == Z_BUF_ERROR || res == Z_DATA_ERROR);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, zlib      ]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	inflateEnd(&z);
+	return t;
+}
+
+/*
+ * Test case from https://github.com/ebiggers/libdeflate/issues/33
+ * with the gzip header and footer removed to leave just the DEFLATE stream
+ */
+static const u8 orig_repro[3962] =
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\x04\xea\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x00\x01\x04\x00\x3f\x00\x00\x00\x00"
+	"\x28\xf7\xff\x00\xff\xff\xff\xff\x00\x00";
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 in[4096];
+	u8 out[10000];
+	u64 t, tz;
+
+	begin_program(argv);
+
+	begin_performance_test();
+
+	/* static huffman case */
+	generate_empty_static_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("static huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("static huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is faster than zlib in this case, e.g.
+	 *	[static huffman, libdeflate]: 215861 KB/s
+	 *	[static huffman, zlib      ]: 73651 KB/s
+	 */
+	putchar('\n');
+	ASSERT(t < tz);
+
+	/* dynamic huffman case */
+	generate_empty_dynamic_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("dynamic huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("dynamic huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is slower than zlib in this case, though not super bad.
+	 *	[dynamic huffman, libdeflate]: 6277 KB/s
+	 *	[dynamic huffman, zlib      ]: 10419 KB/s
+	 * FIXME: make it faster.
+	 */
+	putchar('\n');
+	ASSERT(t < 4 * tz);
+
+	/* original reproducer */
+	t = do_test_libdeflate("original repro", orig_repro, sizeof(orig_repro),
+			       out, sizeof(out));
+	tz = do_test_zlib("original repro", orig_repro, sizeof(orig_repro),
+			  out, sizeof(out));
+	ASSERT(t < tz);
+
+	return 0;
+}

programs/test_trailing_bytes.c

@@ -0,0 +1,152 @@
+/*
+ * test_trailing_bytes.c
+ *
+ * Test that decompression correctly stops at the end of the first DEFLATE,
+ * zlib, or gzip stream, and doesn't process any additional trailing bytes.
+ */
+
+#include "test_util.h"
+
+static const struct {
+	size_t (LIBDEFLATEAPI *compress)(
+			struct libdeflate_compressor *compressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_out_nbytes_ret);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress_ex)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_in_nbytes_ret,
+			size_t *actual_out_nbytes_ret);
+} codecs[] = {
+	{
+		.compress = libdeflate_deflate_compress,
+		.decompress = libdeflate_deflate_decompress,
+		.decompress_ex = libdeflate_deflate_decompress_ex,
+	}, {
+		.compress = libdeflate_zlib_compress,
+		.decompress = libdeflate_zlib_decompress,
+		.decompress_ex = libdeflate_zlib_decompress_ex,
+	}, {
+		.compress = libdeflate_gzip_compress,
+		.decompress = libdeflate_gzip_decompress,
+		.decompress_ex = libdeflate_gzip_decompress_ex,
+	}
+};
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t original_nbytes = 32768;
+	const size_t compressed_nbytes_total = 32768;
+	/*
+	 * Don't use the full buffer for compressed data, because we want to
+	 * test whether decompression can deal with additional trailing bytes.
+	 *
+	 * Note: we can't use a guarded buffer (i.e. a buffer where the byte
+	 * after compressed_nbytes is unmapped) because the decompressor may
+	 * read a few bytes beyond the end of the stream (but ultimately not
+	 * actually use those bytes) as long as they are within the buffer.
+	 */
+	const size_t compressed_nbytes_avail = 30000;
+	size_t i;
+	u8 *original;
+	u8 *compressed;
+	u8 *decompressed;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	size_t compressed_nbytes;
+	enum libdeflate_result res;
+	size_t actual_compressed_nbytes;
+	size_t actual_decompressed_nbytes;
+
+	begin_program(argv);
+
+	ASSERT(compressed_nbytes_avail < compressed_nbytes_total);
+
+	/* Prepare some dummy data to compress */
+	original = xmalloc(original_nbytes);
+	ASSERT(original != NULL);
+	for (i = 0; i < original_nbytes; i++)
+		original[i] = (i % 123) + (i % 1023);
+
+	compressed = xmalloc(compressed_nbytes_total);
+	ASSERT(compressed != NULL);
+	memset(compressed, 0, compressed_nbytes_total);
+
+	decompressed = xmalloc(original_nbytes);
+	ASSERT(decompressed != NULL);
+
+	c = libdeflate_alloc_compressor(6);
+	ASSERT(c != NULL);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(codecs); i++) {
+		compressed_nbytes = codecs[i].compress(c, original,
+						       original_nbytes,
+						       compressed,
+						       compressed_nbytes_avail);
+		ASSERT(compressed_nbytes > 0);
+		ASSERT(compressed_nbytes <= compressed_nbytes_avail);
+
+		/* Test decompress() of stream that fills the whole buffer */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed, compressed_nbytes,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream that fills the whole buffer */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed, compressed_nbytes,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress() of stream with trailing bytes */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed,
+					   compressed_nbytes_total,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream with trailing bytes */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed,
+					      compressed_nbytes_total,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+	}
+
+	free(original);
+	free(compressed);
+	free(decompressed);
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

programs/test_util.c

@@ -0,0 +1,243 @@
+/*
+ * test_util.c - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _WIN32
+/* for MAP_ANONYMOUS or MAP_ANON, which unfortunately aren't part of POSIX... */
+#  undef _POSIX_C_SOURCE
+#  ifdef __APPLE__
+#    define _DARWIN_C_SOURCE
+#  elif defined(__linux__)
+#    define _GNU_SOURCE
+#  endif
+#endif
+
+#include "test_util.h"
+
+#include <fcntl.h>
+#include <time.h>
+#ifdef _WIN32
+#  include <windows.h>
+#else
+#  include <unistd.h>
+#  include <sys/mman.h>
+#  include <sys/time.h>
+#endif
+
+#ifndef MAP_ANONYMOUS
+#  define MAP_ANONYMOUS MAP_ANON
+#endif
+
+/* Abort with an error message */
+_noreturn void
+assertion_failed(const char *expr, const char *file, int line)
+{
+	msg("Assertion failed: %s at %s:%d", expr, file, line);
+	abort();
+}
+
+void
+begin_performance_test(void)
+{
+	/* Skip performance tests by default, since they can be flaky. */
+	if (getenv("INCLUDE_PERF_TESTS") == NULL)
+		exit(0);
+}
+
+static size_t
+get_page_size(void)
+{
+#ifdef _WIN32
+	SYSTEM_INFO info;
+
+	GetSystemInfo(&info);
+	return info.dwPageSize;
+#else
+	return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+/* Allocate a buffer with guard pages */
+void
+alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret)
+{
+	const size_t pagesize = get_page_size();
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	u8 *base_addr;
+	u8 *start, *end;
+#ifdef _WIN32
+	DWORD oldProtect;
+#endif
+
+	*start_ret = NULL;
+	*end_ret = NULL;
+
+#ifdef _WIN32
+	/* Allocate buffer and guard pages with no access. */
+	base_addr = VirtualAlloc(NULL, (nr_pages + 2) * pagesize,
+				 MEM_COMMIT | MEM_RESERVE, PAGE_NOACCESS);
+	if (!base_addr) {
+		msg("Unable to allocate memory (VirtualAlloc): Windows error %u",
+		    (unsigned int)GetLastError());
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Grant read+write access to just the buffer. */
+	if (!VirtualProtect(start, end - start, PAGE_READWRITE, &oldProtect)) {
+		msg("Unable to protect memory (VirtualProtect): Windows error %u",
+		    (unsigned int)GetLastError());
+		VirtualFree(base_addr, 0, MEM_RELEASE);
+		ASSERT(0);
+	}
+#else
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	if (base_addr == (u8 *)MAP_FAILED) {
+		msg_errno("Unable to allocate memory (anonymous mmap)");
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+#endif
+	*start_ret = start;
+	*end_ret = end;
+}
+
+/* Free a buffer that was allocated by alloc_guarded_buffer() */
+void
+free_guarded_buffer(u8 *start, u8 *end)
+{
+	if (!start)
+		return;
+#ifdef _WIN32
+	VirtualFree(start - get_page_size(), 0, MEM_RELEASE);
+#else
+	munmap(start, end - start);
+#endif
+}
+
+/*
+ * Return the number of timer ticks that have elapsed since some unspecified
+ * point fixed at the start of program execution
+ */
+u64
+timer_ticks(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER count;
+
+	QueryPerformanceCounter(&count);
+	return count.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
+#else
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (1000000 * (u64)tv.tv_sec) + tv.tv_usec;
+#endif
+}
+
+/*
+ * Return the number of timer ticks per second
+ */
+static u64
+timer_frequency(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return freq.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	return 1000000000;
+#else
+	return 1000000;
+#endif
+}
+
+/*
+ * Convert a number of elapsed timer ticks to milliseconds
+ */
+u64 timer_ticks_to_ms(u64 ticks)
+{
+	return ticks * 1000 / timer_frequency();
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to MB/s
+ */
+u64 timer_MB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000000;
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to KB/s
+ */
+u64 timer_KB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000;
+}
+
+bool
+put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits)
+{
+	os->bitbuf |= bits << os->bitcount;
+	os->bitcount += num_bits;
+	while (os->bitcount >= 8) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	return true;
+}
+
+bool
+flush_bits(struct output_bitstream *os)
+{
+	while (os->bitcount > 0) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	os->bitcount = 0;
+	return true;
+}

programs/test_util.h

@@ -1,7 +1,5 @@
 /*
- * aligned_malloc.c - aligned memory allocation
- *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
+ * test_util.h - utility functions for test programs
  *
  * Copyright 2016 Eric Biggers
  *
@@ -27,31 +25,43 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/*
- * This file provides portable aligned memory allocation functions that only
- * use malloc() and free().  This avoids portability problems with
- * posix_memalign(), aligned_alloc(), etc.
- */
+#ifndef PROGRAMS_TEST_UTIL_H
+#define PROGRAMS_TEST_UTIL_H
 
-#include <stdlib.h>
+#include "prog_util.h"
 
-#include "aligned_malloc.h"
+#include <zlib.h> /* for comparison purposes */
 
-void *
-aligned_malloc(size_t alignment, size_t size)
-{
-	void *ptr = malloc(sizeof(void *) + alignment - 1 + size);
-	if (ptr) {
-		void *orig_ptr = ptr;
-		ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
-		((void **)ptr)[-1] = orig_ptr;
-	}
-	return ptr;
-}
+#ifdef __GNUC__
+# define _noreturn __attribute__((noreturn))
+#else
+# define _noreturn
+#endif
 
-void
-aligned_free(void *ptr)
-{
-	if (ptr)
-		free(((void **)ptr)[-1]);
-}
+void _noreturn
+assertion_failed(const char *expr, const char *file, int line);
+
+#define ASSERT(expr) { if (unlikely(!(expr))) \
+	assertion_failed(#expr, __FILE__, __LINE__); }
+
+void begin_performance_test(void);
+
+void alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret);
+void free_guarded_buffer(u8 *start, u8 *end);
+
+u64 timer_ticks(void);
+u64 timer_ticks_to_ms(u64 ticks);
+u64 timer_MB_per_s(u64 bytes, u64 ticks);
+u64 timer_KB_per_s(u64 bytes, u64 ticks);
+
+struct output_bitstream {
+	machine_word_t bitbuf;
+	int bitcount;
+	u8 *next;
+	u8 *end;
+};
+
+bool put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits);
+bool flush_bits(struct output_bitstream *os);
+
+#endif /* PROGRAMS_TEST_UTIL_H */

scripts/afl-fuzz/.gitignore

@@ -0,0 +1 @@
+*/fuzz

scripts/afl-fuzz/Makefile

@@ -1,7 +1,6 @@
 SRC := $(wildcard */*.c)
 EXE := $(SRC:.c=)
 
-CFLAGS := -O2 -s
 LDLIBS := -ldeflate
 LDFLAGS := -L../..
 CPPFLAGS := -I../..

scripts/afl-fuzz/deflate_compress/fuzz.c

@@ -12,15 +12,22 @@ int main(int argc, char **argv)
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
+	unsigned char level;
 	assert(fd >= 0);
 	ret = fstat(fd, &stbuf);
 	assert(!ret);
 
-	char in[stbuf.st_size];
+	if (stbuf.st_size == 0)
+		return 0;
+	ret = read(fd, &level, 1);
+	assert(ret == 1);
+	level %= 13;
+
+	char in[stbuf.st_size - 1];
 	ret = read(fd, in, sizeof in);
 	assert(ret == sizeof in);
 
-	c = libdeflate_alloc_compressor(6);
+	c = libdeflate_alloc_compressor(level);
 	d = libdeflate_alloc_decompressor();
 
 	char out[sizeof(in)];

scripts/afl-fuzz/fuzz.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+set -e -u -o pipefail
+
+cd "$(dirname "$0")"
+
+read -r -a AVAILABLE_TARGETS < <(echo */fuzz.c | sed 's@/fuzz.c@@g')
+
+usage()
+{
+	cat << EOF
+Usage: $0 [OPTION]... [TARGET]...
+
+Fuzz libdeflate with afl-fuzz.
+
+Options:
+   --asan          Enable AddressSanitizer
+   --no-resume     Don't resume existing afl-fuzz session; start a new one
+   --ubsan         Enable UndefinedBehaviorSanitizer
+
+Available targets: ${AVAILABLE_TARGETS[*]}
+EOF
+}
+
+die()
+{
+	echo "$*" 1>&2
+	exit 1
+}
+
+asan=false
+ubsan=false
+may_resume=true
+
+longopts_array=(
+asan
+help
+no-resume
+ubsan
+)
+longopts=$(echo "${longopts_array[@]}" | tr ' ' ',')
+
+if ! options=$(getopt -o "" -l "$longopts" -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+eval set -- "$options"
+while (( $# >= 0 )); do
+	case "$1" in
+	--asan)
+		asan=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--no-resume)
+		may_resume=false
+		;;
+	--ubsan)
+		ubsan=true
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Invalid option: \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+if $asan && $ubsan; then
+	die "--asan and --ubsan are mutually exclusive"
+fi
+
+if ! type -P afl-fuzz > /dev/null; then
+	die "afl-fuzz is not installed"
+fi
+
+if (( $# == 0 )); then
+	targets=("${AVAILABLE_TARGETS[@]}")
+else
+	for target; do
+		found=false
+		for t in "${AVAILABLE_TARGETS[@]}"; do
+			if [ "$target" = "$t" ]; then
+				found=true
+			fi
+		done
+		if ! $found; then
+			echo 1>&2 "Unknown target '$target'"
+			echo 1>&2 "Available targets: ${AVAILABLE_TARGETS[*]}"
+			exit 1
+		fi
+	done
+	targets=("$@")
+fi
+if (( ${#targets[@]} > 1 )) && ! type -P urxvt > /dev/null; then
+	die "urxvt is not installed"
+fi
+
+afl_opts=""
+if $asan; then
+	export AFL_USE_ASAN=1
+	export CFLAGS="-O2 -m32"
+	export CC=afl-clang
+	afl_opts+=" -m 800"
+elif $ubsan; then
+	export CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined"
+	export CC=afl-gcc
+else
+	export AFL_HARDEN=1
+	export CFLAGS="-O2"
+	export CC=afl-clang-fast
+fi
+
+sudo sh -c "echo core > /proc/sys/kernel/core_pattern"
+sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
+
+NPROC=$(getconf _NPROCESSORS_ONLN)
+
+make "-j$NPROC" -C ../../ libdeflate.a
+make "-j$NPROC" -B
+
+for dir in "${targets[@]}"; do
+	cp -vaT "$dir" "/tmp/$dir"
+	indir=/tmp/$dir/inputs
+	outdir=/tmp/$dir/outputs
+	if [ -e "$outdir" ]; then
+		if $may_resume; then
+			indir="-"
+		else
+			rm -rf "${outdir:?}"/*
+		fi
+	else
+		mkdir "$outdir"
+	fi
+	cmd="afl-fuzz -i $indir -o $outdir -T $dir $afl_opts -- /tmp/$dir/fuzz @@"
+	if (( ${#targets[@]} > 1 )); then
+		urxvt -e bash -c "$cmd" &
+	else
+		$cmd
+	fi
+done
+wait

scripts/android_build.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+API_LEVEL=28
+ARCH=arm64
+CFLAGS=
+ENABLE_CRC=false
+ENABLE_CRYPTO=false
+NDKDIR=$HOME/android-ndk-r21d
+
+usage() {
+	cat << EOF
+Usage: $0 [OPTION]... -- [MAKE_TARGET]...
+Build libdeflate for Android.
+
+  --api-level=LEVEL    Android API level to target (default: $API_LEVEL)
+  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
+  --enable-crc         Enable crc instructions
+  --enable-crypto      Enable crypto instructions
+  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
+EOF
+}
+if ! options=$(getopt -o '' \
+	-l 'api-level:,arch:,enable-crc,enable-crypto,help,ndkdir:' -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+
+eval set -- "$options"
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--api-level)
+		API_LEVEL="$2"
+		shift
+		;;
+	--arch)
+		ARCH="$2"
+		shift
+		;;
+	--enable-crc)
+		ENABLE_CRC=true
+		;;
+	--enable-crypto)
+		ENABLE_CRYPTO=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--ndkdir)
+		NDKDIR="$2"
+		shift
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Unknown option \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+BINDIR=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/
+
+case "$ARCH" in
+arm|arm32|aarch32)
+	CC=$BINDIR/armv7a-linux-androideabi$API_LEVEL-clang
+	if $ENABLE_CRC || $ENABLE_CRYPTO; then
+		CFLAGS="-march=armv8-a"
+		if $ENABLE_CRC; then
+			CFLAGS+=" -mcrc"
+		else
+			CFLAGS+=" -mnocrc"
+		fi
+		if $ENABLE_CRYPTO; then
+			CFLAGS+=" -mfpu=crypto-neon-fp-armv8"
+		else
+			CFLAGS+=" -mfpu=neon"
+		fi
+	fi
+	;;
+arm64|aarch64)
+	CC=$BINDIR/aarch64-linux-android$API_LEVEL-clang
+	features=""
+	if $ENABLE_CRC; then
+		features+="+crc"
+	fi
+	if $ENABLE_CRYPTO; then
+		features+="+crypto"
+	fi
+	if [ -n "$features" ]; then
+		CFLAGS="-march=armv8-a$features"
+	fi
+	;;
+*)
+	echo 1>&2 "Unknown architecture: \"$ARCH\""
+	usage 1>&2
+	exit 1
+esac
+
+cmd=(make "-j$(grep -c processor /proc/cpuinfo)" "CC=$CC" "CFLAGS=$CFLAGS" "$@")
+echo "${cmd[*]}"
+"${cmd[@]}"

scripts/android_tests.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Test libdeflate on a connected arm64 Android device.
+# Requires the Android NDK (release 19 or later) and adb.
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use NDKDIR if specified in environment, else use default value.
+: "${NDKDIR:=$HOME/android-ndk-r21d}"
+if [ ! -e "$NDKDIR" ]; then
+	cat 1>&2 << EOF
+Android NDK was not found in NDKDIR=$NDKDIR!  Set the
+environmental variable NDKDIR to the location of your Android NDK installation.
+EOF
+	exit 1
+fi
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+android_build_and_test() {
+	echo "Running Android tests with $*"
+
+	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" \
+		all test_programs > /dev/null
+	adb push "$TESTDATA" ./scripts/exec_tests.sh benchmark test_* \
+		/data/local/tmp/ > /dev/null
+
+	# Note: adb shell always returns 0, even if the shell command fails...
+	adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \
+		> "$TMPDIR/adb.out"
+	if ! grep -q "exec_tests finished successfully" "$TMPDIR/adb.out"; then
+		echo 1>&2 "Android test failure!  adb shell output:"
+		cat "$TMPDIR/adb.out"
+		exit 1
+	fi
+}
+
+for arch in arm32 arm64; do
+	android_build_and_test --arch=$arch
+	android_build_and_test --arch=$arch --enable-crc
+	android_build_and_test --arch=$arch --enable-crypto
+	android_build_and_test --arch=$arch --enable-crc --enable-crypto
+done
+echo "Android tests passed"

scripts/checksum_benchmarks.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+have_cpu_feature() {
+	local feature="$1"
+	local tag
+	case $ARCH in
+	arm*|aarch*)
+		tag="Features"
+		;;
+	*)
+		tag="flags"
+		;;
+	esac
+	grep -q "^$tag"$'[ \t]'"*:.*\<$feature\>" /proc/cpuinfo
+}
+
+make_and_test() {
+	# Build the checksum program and tests.  Set the special test support
+	# flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	make "$@" TEST_SUPPORT__DO_NOT_USE=1 checksum test_checksums > /dev/null
+
+	# Run the checksum tests, for good measure.  (This isn't actually part
+	# of the benchmarking.)
+	./test_checksums > /dev/null
+}
+
+__do_benchmark() {
+	local impl="$1" speed
+	shift
+	local flags=("$@")
+
+	speed=$(./checksum "${CKSUM_FLAGS[@]}" "${flags[@]}" -t "$FILE" | \
+		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
+	printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
+}
+
+do_benchmark() {
+	local impl="$1"
+
+	if [ "$impl" = zlib ]; then
+		__do_benchmark "$impl" "-Z"
+	else
+		make_and_test CFLAGS="${EXTRA_CFLAGS[*]}"
+		__do_benchmark "libdeflate, $impl"
+		if [ "$ARCH" = x86_64 ]; then
+			make_and_test CFLAGS="-m32 ${EXTRA_CFLAGS[*]}"
+			__do_benchmark "libdeflate, $impl, 32-bit"
+		fi
+	fi
+}
+
+sort_by_speed() {
+	awk '{print $NF, $0}' | sort -nr | cut -f2- -d' '
+}
+
+disable_cpu_feature() {
+	local name="$1"
+	shift
+	local extra_cflags=("$@")
+
+	LIBDEFLATE_DISABLE_CPU_FEATURES+=",$name"
+	EXTRA_CFLAGS+=("${extra_cflags[@]}")
+}
+
+cleanup() {
+	if $USING_TMPFILE; then
+		rm "$FILE"
+	fi
+}
+
+ARCH="$(uname -m)"
+USING_TMPFILE=false
+
+if (( $# > 1 )); then
+	echo "Usage: $0 [FILE]" 1>&2
+	exit 1
+fi
+
+trap cleanup EXIT
+
+if (( $# == 0 )); then
+	# Generate default test data file.
+	FILE=$(mktemp -t checksum_testdata.XXXXXXXXXX)
+	USING_TMPFILE=true
+	echo "Generating 100 MB test file: $FILE"
+	head -c 100000000 /dev/urandom > "$FILE"
+else
+	FILE="$1"
+fi
+
+cat << EOF
+Method                                       Speed (MB/s)
+------                                       ------------
+EOF
+
+# CRC-32
+CKSUM_NAME="CRC-32"
+CKSUM_FLAGS=()
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature pclmulqdq && have_cpu_feature avx; then
+		do_benchmark "PCLMUL/AVX"
+		disable_cpu_feature "avx" "-mno-avx"
+	fi
+	if have_cpu_feature pclmulqdq; then
+		do_benchmark "PCLMUL"
+		disable_cpu_feature "pclmul" "-mno-pclmul"
+	fi
+	;;
+arm*|aarch*)
+	if have_cpu_feature crc32; then
+		do_benchmark "ARM"
+		disable_cpu_feature "crc32" "-march=armv8-a+nocrc"
+	fi
+	if have_cpu_feature pmull; then
+		do_benchmark "PMULL"
+		disable_cpu_feature "pmull" "-march=armv8-a+nocrc+nocrypto"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed
+
+# Adler-32
+CKSUM_NAME="Adler-32"
+CKSUM_FLAGS=(-A)
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+echo
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature avx512bw; then
+		do_benchmark "AVX-512BW"
+		disable_cpu_feature "avx512bw" "-mno-avx512bw"
+	fi
+	if have_cpu_feature avx2; then
+		do_benchmark "AVX2"
+		disable_cpu_feature "avx2" "-mno-avx2"
+	fi
+	if have_cpu_feature sse2; then
+		do_benchmark "SSE2"
+		disable_cpu_feature "sse2" "-mno-sse2"
+	fi
+	;;
+arm*)
+	if have_cpu_feature neon; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-mfpu=vfpv3"
+	fi
+	;;
+aarch*)
+	if have_cpu_feature asimd; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-march=armv8-a+nosimd"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed

scripts/deflate_benchmarks.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+set -eu -o pipefail
+topdir="$(dirname "$0")/.."
+tmpfile=$(mktemp)
+trap 'rm -f $tmpfile' EXIT
+
+run_benchmark()
+{
+	local best_ctime=1000000000
+	local i
+
+	for i in $(seq "$NUM_ITERATIONS"); do
+		"$@" > "$tmpfile"
+		csize=$(awk '/Compressed/{print $4}' "$tmpfile")
+		ctime=$(awk '/Compression time/{print $3}' "$tmpfile")
+		if (( ctime <  best_ctime )); then
+			best_ctime=$ctime
+		fi
+		: "$i" # make shellcheck happy
+	done
+	CSIZE=$csize
+	CTIME=$best_ctime
+}
+
+multifile()
+{
+	local file results cmd best em
+
+	NUM_ITERATIONS=1
+
+	echo "File | zlib -6 | zlib -9 | libdeflate -6 | libdeflate -9 | libdeflate -12"
+	echo "-----|---------|---------|---------------|---------------|---------------"
+
+	for file in "$@"; do
+		echo -n "$(basename "$file")"
+		results=()
+		cmd=("$topdir/benchmark" -s"$(stat -c "%s" "$file")" "$file")
+		run_benchmark "${cmd[@]}" -Y -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -Y -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -9
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -12
+		results+=("$CSIZE")
+		best=2000000000
+		for result in "${results[@]}"; do
+			if (( result < best)); then
+				best=$result
+			fi
+		done
+		for result in "${results[@]}"; do
+			if (( result == best )); then
+				em="**"
+			else
+				em=""
+			fi
+			echo -n " | ${em}${result}${em}"
+		done
+		echo
+	done
+}
+
+single_file()
+{
+	local file=$1
+	local usize args
+	local include_old=false
+
+	usize=$(stat -c "%s" "$file")
+	: ${NUM_ITERATIONS:=3}
+
+	if [ -e "$topdir/benchmark-old" ]; then
+		include_old=true
+	fi
+	echo -n "Level | libdeflate (new) "
+	if $include_old; then
+		echo -n "| libdeflate (old) "
+	fi
+	echo "| zlib"
+	echo -n "------|------------------"
+	if $include_old; then
+		echo -n "|------------------"
+	fi
+	echo "|-----"
+	for level in {1..12}; do
+		echo -n "$level"
+		args=("$file" -s "$usize" "-$level")
+
+		run_benchmark "$topdir/benchmark" "${args[@]}"
+		echo -n " | $CSIZE / $CTIME"
+
+		if $include_old; then
+			run_benchmark "$topdir/benchmark-old" "${args[@]}"
+			echo -n " | $CSIZE / $CTIME"
+		fi
+
+		if (( level > 9 )); then
+			echo -n " | N/A"
+		else
+			run_benchmark "$topdir/benchmark" "${args[@]}" -Y
+			echo -n " | $CSIZE / $CTIME"
+		fi
+		echo
+	done
+}
+
+if (( $# > 1 )); then
+	multifile "$@"
+elif (( $# == 1 )); then
+	single_file "$@"
+else
+	echo 1>&2 "Usage: $0 FILE..."
+fi

scripts/detect.sh

@@ -1,24 +1,25 @@
 #!/bin/sh
 
-if [ -z "$CC" ]; then
-	CC=cc
-fi
+set -eu
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
 
 echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
 echo "#ifndef CONFIG_H"
 echo "#define CONFIG_H"
 
-tmpfile="$(mktemp -t libdeflate_config.XXXXXXXX)"
-trap "rm -f \"$tmpfile\"" EXIT
-
 program_compiles() {
-	echo "$1" > "$tmpfile"
-	$CC $CFLAGS -x c "$tmpfile" -o /dev/null > /dev/null 2>&1
+	echo "$1" | $CC $CFLAGS -Wno-error -x c - -o /dev/null > /dev/null 2>&1
 }
 
 check_function() {
-	funcname="$1"
-	macro="HAVE_$(echo $funcname | tr a-z A-Z)"
+	funcname=$1
+	macro="HAVE_$(echo "$funcname" | tr '[:lower:]' '[:upper:]')"
+
 	echo
 	echo "/* Is the $funcname() function available? */"
 	if program_compiles "int main() { $funcname(); }"; then

scripts/exec_tests.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# Helper script used by run_tests.sh and android_tests.sh,
+# not intended to be run directly
+#
+
+set -eu
+
+run_cmd() {
+	echo "$WRAPPER $*"
+	$WRAPPER "$@" > /dev/null
+}
+
+for prog in ./test_*; do
+	run_cmd "$prog"
+done
+
+for format in '' '-g' '-z'; do
+	for ref_impl in '' '-Y' '-Z'; do
+		run_cmd ./benchmark $format $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9; do
+	for ref_impl in '' '-Y'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9 12; do
+	for ref_impl in '' '-Z'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+
+echo "exec_tests finished successfully" # Needed for 'adb shell'

scripts/gen_crc32_table.c

@@ -1,8 +1,6 @@
 /*
  * gen_crc32_table.c - a program for CRC-32 table generation
  *
- * Originally public domain; changes after 2016-09-07 are copyrighted.
- *
  * Copyright 2016 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person

scripts/gen_default_litlen_costs.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+#
+# This script computes the default litlen symbol costs for the near-optimal
+# parser.
+
+from math import log2
+
+BIT_COST = 16 # Must match BIT_COST in deflate_compress.c
+NUM_LEN_SLOTS = 29
+
+print("""static const struct {
+	u8 used_lits_to_lit_cost[257];
+	u8 len_sym_cost;
+} default_litlen_costs[] = {""")
+MATCH_PROBS = [0.25, 0.50, 0.75]
+for i, match_prob in enumerate(MATCH_PROBS):
+    len_prob = match_prob / NUM_LEN_SLOTS
+    len_sym_cost = int(-log2(len_prob) * BIT_COST)
+    if i == 0:
+        print('\t{', end='')
+    print(f' /* match_prob = {match_prob} */')
+    print('\t\t.used_lits_to_lit_cost = {')
+
+    j = 0
+    for num_used_literals in range(0, 257):
+        if num_used_literals == 0:
+            num_used_literals = 1
+        lit_prob = (1 - match_prob) / num_used_literals
+        lit_cost = int(-log2(lit_prob) * BIT_COST)
+        if j == 0:
+            print('\t\t\t', end='')
+        if j == 7 or num_used_literals == 256:
+            print(f'{lit_cost},')
+            j = 0
+        else:
+            print(f'{lit_cost}, ', end='')
+            j += 1
+    print('\t\t},')
+    print(f'\t\t.len_sym_cost = {len_sym_cost},')
+    if i < len(MATCH_PROBS) - 1:
+        print('\t}, {', end='')
+    else:
+        print('\t},')
+print('};')

scripts/gen_offset_slot_map.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+# This script generates the deflate_offset_slot[] array, which is a condensed
+# map from offsets to offset slots.
+
+DEFLATE_OFFSET_SLOT_BASE = [
+	1    , 2    , 3    , 4     , 5     , 7     , 9     , 13    ,
+	17   , 25   , 33   , 49    , 65    , 97    , 129   , 193   ,
+	257  , 385  , 513  , 769   , 1025  , 1537  , 2049  , 3073  ,
+	4097 , 6145 , 8193 , 12289 , 16385 , 24577 ,
+]
+
+DEFLATE_EXTRA_OFFSET_BITS = [
+	0    , 0    , 0    , 0     , 1     , 1     , 2     , 2     ,
+	3    , 3    , 4    , 4     , 5     , 5     , 6     , 6     ,
+	7    , 7    , 8    , 8     , 9     , 9     , 10    , 10    ,
+	11   , 11   , 12   , 12    , 13    , 13    ,
+]
+
+offset_slot_map = [0] * 512
+
+for offset_slot, offset_base in enumerate(DEFLATE_OFFSET_SLOT_BASE):
+    num_extra_bits = DEFLATE_EXTRA_OFFSET_BITS[offset_slot]
+    offset_end = offset_base + (1 << num_extra_bits)
+    if offset_base <= 256:
+        for offset in range(offset_base, offset_end):
+            offset_slot_map[offset] = offset_slot
+    else:
+        for offset in range(offset_base, offset_end, 128):
+            offset_slot_map[256 + ((offset - 1) >> 7)] = offset_slot
+
+print('static const u8 deflate_offset_slot_map[512] = {')
+for i in range(0, len(offset_slot_map), 16):
+    print('\t', end='')
+    for j, v in enumerate(offset_slot_map[i:i+16]):
+        print(f'{v},', end='')
+        if j == 15:
+            print('')
+        else:
+            print(' ', end='')
+print('};')

scripts/gzip_tests.sh

@@ -7,13 +7,13 @@
 # whether the GNU versions or the libdeflate versions, or a combination, of
 # these programs are used.
 #
-# The environmental variable SMOKEDATA must also be set to a file containing
+# The environmental variable TESTDATA must also be set to a file containing
 # test data.
 #
 
 set -eu -o pipefail
 
-export -n GZIP GUNZIP SMOKEDATA
+export -n GZIP GUNZIP TESTDATA
 
 TMPDIR="$(mktemp -d)"
 CURRENT_TEST=
@@ -27,13 +27,14 @@ cleanup() {
 
 trap cleanup EXIT
 
-SMOKEDATA="$(realpath "$SMOKEDATA")"
+TESTDATA="$(readlink -f "$TESTDATA")"
 cd "$TMPDIR"
 
 begin_test() {
 	CURRENT_TEST="$1"
-	rm -rf -- "$TMPDIR"/*
-	cp "$SMOKEDATA" file
+	rm -rf -- "${TMPDIR:?}"/*
+	cp "$TESTDATA" file
+	chmod +w file
 }
 
 gzip() {
@@ -50,9 +51,9 @@ assert_status() {
 	shift 2
 	(
 		set +e
-		eval "$*" 2>&1 >/dev/null
+		{ eval "$*" > /dev/null; } 2>&1
 		local actual_status=$?
-		if [ $actual_status -ne $expected_status ]; then
+		if [ "$actual_status" != "$expected_status" ]; then
 			echo 1>&2 "Command '$*' exited with status" \
 				"$actual_status but expected status" \
 				"$expected_status"
@@ -60,7 +61,7 @@ assert_status() {
 		fi
 		exit 0
 	) > command_output
-	if ! egrep -q "$expected_msg" command_output; then
+	if ! grep -E -q "$expected_msg" command_output; then
 		echo 1>&2 "Expected output of command '$*' to match regex" \
 			"'$expected_msg'"
 		echo 1>&2 "Actual output was:"
@@ -83,13 +84,43 @@ assert_skipped() {
 	assert_warning '\<(ignored|skipping|unchanged)\>' "$@"
 }
 
+assert_equals() {
+	local expected="$1"
+	local actual="$2"
+
+	if [ "$expected" != "$actual" ]; then
+		echo 1>&2 "Expected '$expected', but got '$actual'"
+		return 1
+	fi
+}
+
+# Get the filesystem type.
+FSTYPE=$(df -T . | tail -1 | awk '{print $2}')
+
+# If gzip or gunzip is the GNU version, require that it supports the '-k'
+# option.  This option was added in v1.6, released in 2013.
+check_version_prereq() {
+	local prog=$1
+
+	if ! echo | { $prog -k || true; } |& grep -q 'invalid option'; then
+		return 0
+	fi
+	if ! $prog -V |& grep -q 'Free Software Foundation'; then
+		echo 1>&2 "Unexpected case: not GNU $prog, but -k option is invalid"
+		exit 1
+	fi
+	echo "GNU $prog is too old; skipping gzip/gunzip tests"
+	exit 0
+}
+check_version_prereq gzip
+check_version_prereq gunzip
 
 begin_test 'Basic compression and decompression works'
 cp file orig
 gzip file
-[ ! -e file -a -e file.gz ]
+[ ! -e file ] && [ -e file.gz ]
 gunzip file.gz
-[ -e file -a ! -e file.gz ]
+[ -e file ] && [ ! -e file.gz ]
 cmp file orig
 
 
@@ -121,15 +152,19 @@ cmp file.gz 2.gz
 cmp file orig
 
 
+# Note: in some of the commands below, we intentionally use 'cat file | gzip'
+# rather than 'gzip < file', in order to test the use of a pipe.  This produces
+# a shellcheck warning about 'cat' being unnecessary.  Suppress that warning by
+# using { cat file; true; }.
 begin_test 'Reading from stdin works'
 gzip < file > 1.gz
 gzip - < file > 2.gz
-cat file | gzip > 3.gz
-cat file | gzip - > 4.gz
+{ cat file; true; } | gzip > 3.gz
+{ cat file; true; } | gzip - > 4.gz
 cmp file <(gunzip < 1.gz)
 cmp file <(gunzip - < 2.gz)
-cmp file <(cat 3.gz | gunzip)
-cmp file <(cat 4.gz | gunzip -)
+cmp file <({ cat 3.gz; true; } | gunzip)
+cmp file <({ cat 4.gz; true; } | gunzip -)
 
 
 begin_test '-n option is accepted'
@@ -153,12 +188,11 @@ else
 	done
 	max_level=12
 fi
-for level in `seq 1 $max_level`; do
-	gzip -c -$level file > file$level
-	cmp file <(gunzip -c file$level)
+for level in $(seq 1 $max_level); do
+	gzip -c "-$level" file > "file$level"
+	cmp file <(gunzip -c "file$level")
 done
 rm file command_output
-cmp <(ls -S) <(ls -v) # file,file{1..max_level} have decreasing size
 
 
 begin_test 'Overwriting output file requires -f'
@@ -177,7 +211,7 @@ cmp file orig
 
 begin_test 'Nonexistent input file fails, even with -f'
 for prog in 'gzip' 'gzip -f' 'gunzip' 'gunzip -f'; do
-	assert_error 'No such file or directory' $prog NONEXISTENT
+	assert_error 'No such file or directory' "$prog" NONEXISTENT
 done
 
 
@@ -185,12 +219,22 @@ begin_test 'Compressing already-suffixed file requires -f or -c'
 gzip file
 gzip -c file.gz > c.gz
 gzip file.gz 2>&1 >/dev/null | grep -q 'already has .gz suffix'
-[ -e file.gz -a ! -e file.gz.gz ]
+[ -e file.gz ] && [ ! -e file.gz.gz ]
 gzip -f file.gz
-[ ! -e file.gz -a -e file.gz.gz ]
+[ ! -e file.gz ] && [ -e file.gz.gz ]
 cmp file.gz.gz c.gz
 
 
+begin_test 'gunzip -f -c passes through non-gzip data'
+echo hello > file
+cp file orig
+gunzip -f -c file > foo
+cmp file foo
+gzip file
+gunzip -f -c file.gz > foo
+cmp foo orig
+
+
 begin_test 'Decompressing unsuffixed file only works with -c'
 gzip file && mv file.gz file
 assert_skipped gunzip file
@@ -202,7 +246,7 @@ mv file file.gz && gunzip file.gz && cmp file orig
 begin_test '... unless there is a corresponding suffixed file'
 cp file orig
 gzip file
-[ ! -e file -a -e file.gz ]
+[ ! -e file ] && [ -e file.gz ]
 gunzip -c file > tmp
 cmp tmp orig
 rm tmp
@@ -211,7 +255,7 @@ gunzip -c file > tmp
 cmp tmp orig
 rm tmp file
 gunzip file
-[ -e file -a ! -e file.gz ]
+[ -e file ] && [ ! -e file.gz ]
 cmp file orig
 
 
@@ -233,9 +277,9 @@ begin_test '(gzip) symlink is rejected without -f or -c'
 ln -s file symlink1
 ln -s file symlink2
 assert_error 'Too many levels of symbolic links' gzip symlink1
-[ -e file -a -e symlink1 -a ! -e symlink1.gz ]
+[ -e file ] && [ -e symlink1 ] && [ ! -e symlink1.gz ]
 gzip -f symlink1
-[ -e file -a ! -e symlink1 -a -e symlink1.gz ]
+[ -e file ] && [ ! -e symlink1 ] && [ -e symlink1.gz ]
 gzip -c symlink2 > /dev/null
 
 
@@ -244,9 +288,9 @@ gzip file
 ln -s file.gz symlink1.gz
 ln -s file.gz symlink2.gz
 assert_error 'Too many levels of symbolic links' gunzip symlink1
-[ -e file.gz -a -e symlink1.gz -a ! -e symlink1 ]
+[ -e file.gz ] && [ -e symlink1.gz ] && [ ! -e symlink1 ]
 gunzip -f symlink1.gz
-[ -e file.gz -a ! -e symlink1.gz -a -e symlink1 ]
+[ -e file.gz ] && [ ! -e symlink1.gz ] && [ -e symlink1 ]
 gunzip -c symlink2.gz > /dev/null
 
 
@@ -262,35 +306,35 @@ assert_skipped gunzip -f foo.gz
 
 
 begin_test '(gzip) overwriting symlink does not follow symlink'
-echo 1 > 1
-echo 2 > 2
-gzip 1
-ln -s 1.gz 2.gz
-gzip -f 2
-gunzip 1.gz
-cmp <(echo 1) 1
+echo a > a
+echo b > b
+gzip a
+ln -s a.gz b.gz
+gzip -f b
+gunzip a.gz
+cmp <(echo a) a
 
 
 begin_test '(gunzip) overwriting symlink does not follow symlink'
-echo 1 > 1
-echo 2 > 2
-gzip 2
-ln -s 1 2
-gunzip -f 2.gz
-cmp <(echo 1) 1
-cmp <(echo 2) 2
+echo a > a
+echo b > b
+gzip b
+ln -s a b
+gunzip -f b.gz
+cmp <(echo a) a
+cmp <(echo b) b
 
 
 begin_test '(gzip) hard linked file skipped without -f or -c'
 cp file orig
 ln file link
-[ $(stat -c %h file) -eq 2 ]
+assert_equals 2 "$(stat -c %h file)"
 assert_skipped gzip file
 gzip -c file > /dev/null
-[ $(stat -c %h file) -eq 2 ]
+assert_equals 2 "$(stat -c %h file)"
 gzip -f file
-[ $(stat -c %h link) -eq 1 ]
-[ $(stat -c %h file.gz) -eq 1 ]
+assert_equals 1 "$(stat -c %h link)"
+assert_equals 1 "$(stat -c %h file.gz)"
 cmp link orig
 # XXX: GNU gzip skips hard linked files with -k, libdeflate's doesn't
 
@@ -299,22 +343,22 @@ begin_test '(gunzip) hard linked file skipped without -f or -c'
 gzip file
 ln file.gz link.gz
 cp file.gz orig.gz
-[ $(stat -c %h file.gz) -eq 2 ]
+assert_equals 2 "$(stat -c %h file.gz)"
 assert_skipped gunzip file.gz
 gunzip -c file.gz > /dev/null
-[ $(stat -c %h file.gz) -eq 2 ]
+assert_equals 2 "$(stat -c %h file.gz)"
 gunzip -f file
-[ $(stat -c %h link.gz) -eq 1 ]
-[ $(stat -c %h file) -eq 1 ]
+assert_equals 1 "$(stat -c %h link.gz)"
+assert_equals 1 "$(stat -c %h file)"
 cmp link.gz orig.gz
 
 
 begin_test 'Multiple files'
 cp file file2
 gzip file file2
-[ ! -e file -a ! -e file2 -a -e file.gz -a -e file2.gz ]
+[ ! -e file ] && [ ! -e file2 ] && [ -e file.gz ] && [ -e file2.gz ]
 gunzip file.gz file2.gz
-[ -e file -a -e file2 -a ! -e file.gz -a ! -e file2.gz ]
+[ -e file ] && [ -e file2 ] && [ ! -e file.gz ] && [ ! -e file2.gz ]
 
 
 begin_test 'Multiple files, continue on warning'
@@ -330,19 +374,21 @@ assert_skipped gunzip 1.gz 2.gz
 cmp 2 file
 
 
-begin_test 'Multiple files, continue on error'
-cp file 1
-cp file 2
-chmod -r 1
-assert_error 'Permission denied' gzip 1 2
-[ ! -e 1.gz ]
-cmp file <(gunzip -c 2.gz)
-rm -f 1
-cp 2.gz 1.gz
-chmod -r 1.gz
-assert_error 'Permission denied' gunzip 1.gz 2.gz
-[ ! -e 1 ]
-cmp 2 file
+if (( $(id -u) != 0 )); then
+	begin_test 'Multiple files, continue on error'
+	cp file 1
+	cp file 2
+	chmod a-r 1
+	assert_error 'Permission denied' gzip 1 2
+	[ ! -e 1.gz ]
+	cmp file <(gunzip -c 2.gz)
+	rm -f 1
+	cp 2.gz 1.gz
+	chmod a-r 1.gz
+	assert_error 'Permission denied' gunzip 1.gz 2.gz
+	[ ! -e 1 ]
+	cmp 2 file
+fi
 
 
 begin_test 'Compressing empty file'
@@ -360,20 +406,18 @@ echo 1 > foo.gz
 assert_error '\<not in gzip format\>' gunzip foo.gz
 echo abcdefgh > foo.gz
 assert_error '\<not in gzip format\>' gunzip foo.gz
-xxd -r > foo.gz <<-EOF
-00000000: 1f8b 0800 0000 0000 00ff 4b4c 4a4e 4924  ..........KLJNI$
-00000010: 1673 0100 6c5b a262 2e00 0000            .s..l[.b....
-EOF
+echo -ne '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4b\x4c\x4a\x4e\x49\x24\x16\x73\x01\x00\x6c\x5b\xa2\x62\x2e\x00\x00\x00' \
+	> foo.gz
 assert_error '\<(not in gzip format|crc error)\>' gunzip foo.gz
 
 
 for suf in .foo foo .blaaaaaaaaaaaaaaaargh; do
 	begin_test "Custom suffix: $suf"
 	gzip -S $suf file
-	[ ! -e file -a ! -e file.gz -a -e file$suf ]
+	[ ! -e file ] && [ ! -e file.gz ] && [ -e file$suf ]
 	assert_skipped gunzip file$suf
 	gunzip -S $suf file$suf
-	[ -e file -a ! -e file.gz -a ! -e file$suf ]
+	[ -e file ] && [ ! -e file.gz ] && [ ! -e file$suf ]
 done
 # DIFFERENCE: GNU gzip lower cases suffix, we don't
 
@@ -384,12 +428,20 @@ assert_error '\<invalid suffix\>' gunzip -S '""' file
 
 
 begin_test 'Timestamps and mode are preserved'
+if [ "$FSTYPE" = shiftfs ]; then
+	# In Travis CI, the filesystem (shiftfs) only supports seconds precision
+	# timestamps.  Nanosecond precision still sometimes seems to work,
+	# probably due to caching, but it is unreliable.
+	format='%a;%X;%Y'
+else
+	format='%a;%x;%y'
+fi
 chmod 777 file
-orig_stat="$(stat -c '%a;%x;%y' file)"
+orig_stat="$(stat -c "$format" file)"
 gzip file
 sleep 1
 gunzip file.gz
-[ "$(stat -c '%a;%x;%y' file)" = "$orig_stat" ]
+assert_equals "$orig_stat" "$(stat -c "$format" file)"
 
 
 begin_test 'Decompressing multi-member gzip file'
@@ -401,10 +453,10 @@ cmp file orig
 
 
 begin_test 'Decompressing multi-member gzip file (final member smaller)'
-echo 'hello world' > 2
-cat file 2 > orig
+echo 'hello world' > hello
+cat file hello > orig
 gzip -c file > file.gz
-gzip -c 2 >> file.gz
+gzip -c hello >> file.gz
 gunzip -f file.gz
 cmp file orig
 
@@ -422,6 +474,32 @@ for prog in gzip gunzip; do
 done
 
 
+begin_test '-t (test) option works'
+good_files=(
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIoAUrnJFZVKqTkp+txAQBqzFDrLQAAAA==')
+bad_files=(
+'H4sIAO1YYmAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+46ehsAAAA='
+'H4sIAO1YYmAAA3PMSVTITVTIzi85VABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsBAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIogUrnJFZVKqTkp+txAQBqzFDrLQAAAA=='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9L')
+for contents in "${good_files[@]}"; do
+	echo "$contents" | base64 -d | gzip -t
+done
+for contents in "${bad_files[@]}"; do
+	echo "$contents" | base64 -d > file
+	assert_error '\<invalid compressed data|file corrupt|unexpected end of file|Out of memory\>' \
+		gzip -t file
+done
+
+
+begin_test '-q (quiet) option works'
+mkdir dir
+gunzip -q dir &> output || true
+[ ! -s output ]
+
+
 begin_test 'Version information'
 gzip -V | grep -q Copyright
 gunzip -V | grep -q Copyright

scripts/make-windows-releases.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+for arch in 'i686' 'x86_64'; do
+	make clean
+	make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all \
+		benchmark.exe checksum.exe
+	dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin
+	rm -rf "$dir" "$dir.zip"
+	mkdir "$dir"
+	cp libdeflate.{dll,lib,def} libdeflatestatic.lib libdeflate.h ./*.exe \
+		"$dir"
+	${arch}-w64-mingw32-strip "$dir/libdeflate.dll" "$dir"/*.exe
+	for file in COPYING NEWS.md README.md; do
+		sed < $file > "$dir/${file}.txt" -e 's/$/\r/g'
+	done
+	(cd "$dir" && zip -r "../${dir}.zip" .)
+done

scripts/run_tests.sh

@@ -0,0 +1,329 @@
+#!/bin/bash
+#
+# Test script for libdeflate
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+MAKE="make -j$(getconf _NPROCESSORS_ONLN)"
+
+CC_VERSION=$($CC --version | head -1)
+
+ARCH=$(uname -m)
+
+for skip in SKIP_FREESTANDING SKIP_VALGRIND SKIP_UBSAN SKIP_ASAN SKIP_CFI \
+	    SKIP_SHARED_LIB; do
+	if [ "${!skip:-}" = "1" ]; then
+		eval $skip=true
+	else
+		eval $skip=false
+	fi
+done
+
+###############################################################################
+
+INDENT=0
+
+log() {
+	echo -n "[$(date)] "
+	head -c $(( INDENT * 4 )) /dev/zero | tr '\0' ' '
+	echo "$@"
+}
+
+begin() {
+	log "$@"
+	(( INDENT++ )) || true
+}
+
+end() {
+	(( INDENT-- )) || true
+}
+
+run_cmd() {
+	log "$@"
+	"$@" > /dev/null
+}
+
+fail() {
+	echo 1>&2 "$@"
+	exit 1
+}
+
+file_count() {
+	local dir=$1
+
+	find "$dir" -type f -o -type l | wc -l
+}
+
+cflags_supported() {
+	# -Werror is needed here in order for old versions of clang to reject
+	# invalid options.
+	echo 'int main(void){ return 0; }' \
+		| $CC $CFLAGS "$@" -Werror -x c - -o /dev/null 2>/dev/null
+}
+
+valgrind_version_at_least() {
+	local want_vers=$1
+	local vers
+
+	vers=$(valgrind --version | grep -E -o '[0-9\.]+' | head -1)
+
+	[ "$want_vers" = "$(echo -e "$vers\n$want_vers" | sort -V | head -1)" ]
+}
+
+build_and_run_tests() {
+	local quick=false
+	if [ "${1:-}" = "--quick" ]; then
+		quick=true
+		shift
+	fi
+
+	begin "CC=$CC CFLAGS=\"$CFLAGS\" WRAPPER=\"$WRAPPER\" $*"
+
+	# Build libdeflate, including the test programs.  Set the special test
+	# support flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	$MAKE "$@" TEST_SUPPORT__DO_NOT_USE=1 all test_programs > /dev/null
+
+	# When not using -march=native, run the tests multiple times with
+	# different combinations of CPU features disabled.  This is needed to
+	# test all variants of dynamically-dispatched code.
+	#
+	# For now, we aren't super exhausive in which combinations of features
+	# we test disabling.  We just disable the features roughly in order from
+	# newest to oldest for each architecture, cumulatively.  In practice,
+	# that's good enough to cover all the code.
+	local features=('')
+	if ! [[ "$CFLAGS" =~ "-march=native" ]] && ! $quick; then
+		case "$ARCH" in
+		i386|x86_64)
+			features+=(avx512bw avx2 avx bmi2 pclmul sse2)
+			;;
+		arm*|aarch*)
+			features+=(crc32 pmull neon)
+			;;
+		esac
+	fi
+	local disable_str=""
+	local feature
+	for feature in "${features[@]}"; do
+		if [ -n "$feature" ]; then
+			if [ -n "$disable_str" ]; then
+				disable_str+=","
+			fi
+			disable_str+="$feature"
+		fi
+		log "Using LIBDEFLATE_DISABLE_CPU_FEATURES=$disable_str"
+		LIBDEFLATE_DISABLE_CPU_FEATURES="$disable_str" \
+		    sh ./scripts/exec_tests.sh > /dev/null
+	done
+	end
+}
+
+verify_freestanding_build() {
+	# It is expected that sanitizer builds link to external functions.
+	if [[ "$CFLAGS" =~ "-fsanitize" ]]; then
+		return 0
+	fi
+	log "Verifying that freestanding build is really freestanding"
+	if nm libdeflate.so | grep -q ' U '; then
+		echo 1>&2 "Freestanding lib links to external functions!:"
+		nm libdeflate.so | grep ' U '
+		return 1
+	fi
+	if ldd libdeflate.so | grep -q -v '\<statically linked\>'; then
+		echo 1>&2 "Freestanding lib links to external libraries!:"
+		ldd libdeflate.so
+		return 1
+	fi
+}
+
+gzip_tests() {
+	local gzips=("$PWD/gzip")
+	local gunzips=("$PWD/gunzip")
+	if [ "${1:-}" != "--quick" ]; then
+		gzips+=(/bin/gzip)
+		gunzips+=(/bin/gunzip)
+	fi
+	local gzip gunzip
+
+	begin "Running gzip program tests with CC=\"$CC\" CFLAGS=\"$CFLAGS\""
+	$MAKE gzip gunzip > /dev/null
+	for gzip in "${gzips[@]}"; do
+		for gunzip in "${gunzips[@]}"; do
+			log "GZIP=$gzip, GUNZIP=$gunzip"
+			GZIP="$gzip" GUNZIP="$gunzip" TESTDATA="$TESTDATA" \
+				./scripts/gzip_tests.sh
+		done
+	done
+	end
+}
+
+do_run_tests() {
+	build_and_run_tests "$@"
+	if [ "${1:-}" != "--quick" ]; then
+		if $SKIP_FREESTANDING; then
+			log "Skipping freestanding build tests due to SKIP_FREESTANDING=1"
+		else
+			build_and_run_tests FREESTANDING=1
+			verify_freestanding_build
+		fi
+	fi
+	gzip_tests "$@"
+}
+
+check_symbol_prefixes() {
+	log "Checking that all global symbols are prefixed with \"libdeflate_\""
+	$MAKE libdeflate.a > /dev/null
+	if nm libdeflate.a | grep ' T ' | grep -E -v " _?libdeflate_"; then
+		fail "Some global symbols aren't prefixed with \"libdeflate_\""
+	fi
+	log "Checking that all exported symbols are prefixed with \"libdeflate\""
+	$MAKE libdeflate.so > /dev/null
+	if nm libdeflate.so | grep ' T ' \
+			| grep -E -v " (libdeflate_|_init\>|_fini\>)"; then
+		fail "Some exported symbols aren't prefixed with \"libdeflate_\""
+	fi
+}
+
+test_use_shared_lib() {
+	if $SKIP_SHARED_LIB; then
+		log "Skipping USE_SHARED_LIB=1 tests due to SKIP_SHARED_LIB=1"
+		return
+	fi
+	log "Testing USE_SHARED_LIB=1"
+	$MAKE gzip > /dev/null
+	if ldd gzip | grep -q 'libdeflate.so'; then
+		fail "Binary should be statically linked by default"
+	fi
+	$MAKE USE_SHARED_LIB=1 all check > /dev/null
+	ldd gzip > "$TMPDIR/ldd.out"
+	if ! grep -q 'libdeflate.so' "$TMPDIR/ldd.out"; then
+		cat 1>&2 "$TMPDIR/ldd.out"
+		fail "Binary isn't dynamically linked"
+	fi
+	rm "$TMPDIR/ldd.out"
+}
+
+install_uninstall_tests() {
+	local shell
+
+	begin "Testing 'make install' and 'make uninstall'"
+	for shell in '/bin/bash' '/bin/dash'; do
+		log "Trying SHELL=$shell"
+		$MAKE SHELL=$shell clean > /dev/null
+		$MAKE SHELL=$shell DESTDIR="$TMPDIR/inst" install > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" == 0 )); then
+			fail "'make install' didn't install any files"
+		fi
+		make SHELL=$shell DESTDIR="$TMPDIR/inst" uninstall > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" != 0 )); then
+			fail "'make uninstall' didn't uninstall all files"
+		fi
+		rm -r "$TMPDIR/inst"
+	done
+	end
+}
+
+run_tests() {
+	export WRAPPER="" # no wrapper by default; overridden by valgrind tests
+	local cflags
+
+	begin "Running tests"
+	do_run_tests
+	end
+
+	cflags=("-O3")
+	if cflags_supported "${cflags[@]}" "-march=native"; then
+		cflags+=("-march=native")
+	fi
+	begin "Running tests with ${cflags[*]}"
+	CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests
+	end
+
+	# Need valgrind 3.9.0 for '--errors-for-leak-kinds=all'
+	# Need valgrind 3.12.0 for armv8 crypto and crc instructions
+	if $SKIP_VALGRIND; then
+		log "Skipping valgrind tests due to SKIP_VALGRIND=1"
+	elif valgrind_version_at_least 3.12.0; then
+		begin "Running tests with Valgrind"
+		WRAPPER="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all" \
+			do_run_tests --quick
+		end
+	fi
+
+	cflags=("-fsanitize=undefined" "-fno-sanitize-recover=undefined")
+	if $SKIP_UBSAN; then
+		log "Skipping UBSAN tests due to SKIP_UBSAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with UBSAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping UBSAN tests because compiler ($CC_VERSION) doesn't support UBSAN"
+	fi
+
+	cflags=("-fsanitize=address" "-fno-sanitize-recover=address")
+	if $SKIP_ASAN; then
+		log "Skipping ASAN tests due to SKIP_ASAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with ASAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping ASAN tests because compiler ($CC_VERSION) doesn't support ASAN"
+	fi
+
+	cflags=("-fsanitize=cfi" "-fno-sanitize-recover=cfi" "-flto"
+		"-fvisibility=hidden")
+	if $SKIP_CFI; then
+		log "Skipping CFI tests due to SKIP_CFI=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with CFI"
+		CFLAGS="$CFLAGS ${cflags[*]}" AR=llvm-ar do_run_tests --quick
+		end
+	else
+		log "Skipping CFI tests because compiler ($CC_VERSION) doesn't support CFI"
+	fi
+
+	install_uninstall_tests
+	check_symbol_prefixes
+	test_use_shared_lib
+}
+
+###############################################################################
+
+log "Starting libdeflate tests"
+run_tests
+log "All tests passed!"

tools/afl-fuzz/prepare_for_fuzz.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-set -e
-
-make -C ../../ clean
-make clean
-AFL_HARDEN=1 make CC=afl-gcc -C ../../
-AFL_HARDEN=1 make CC=afl-gcc
-
-for dir in $(find . -mindepth 1 -maxdepth 1 -type d); do
-	rm -rf /tmp/$dir
-	cp -va $dir /tmp/$dir
-	mkdir -p /tmp/$dir/outputs
-done

tools/android_build.sh

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-ARCH="arm32"
-COMPILER="gcc"
-NDKDIR="/opt/android-ndk"
-ENABLE_NEON=false
-ENABLE_CRYPTO=false
-
-usage() {
-	cat << EOF
-Usage: $0 [OPTION]... -- [BENCHMARK_PROGRAM_ARG]...
-Build the libdeflate test programs for Android
-
-  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
-  --compiler=COMPILER  Compiler: gcc|clang (default: $COMPILER)
-  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
-  --enable-neon        Enable NEON instructions
-  --enable-crypto      Enable crypto extensions (implies NEON too)
-EOF
-}
-
-if ! options=$(getopt -o '' \
-	-l 'arch:,compiler:,ndkdir:,enable-neon,enable-crypto,help' -- "$@"); then
-	usage
-	exit 1
-fi
-
-eval set -- "$options"
-
-while [ $# -gt 0 ]; do
-	case "$1" in
-	--arch)
-		ARCH="$2"
-		shift
-		;;
-	--compiler)
-		COMPILER="$2"
-		shift
-		;;
-	--ndkdir)
-		NDKDIR="$2"
-		shift
-		;;
-	--enable-neon)
-		ENABLE_NEON=true
-		;;
-	--enable-crypto)
-		ENABLE_CRYPTO=true
-		;;
-	--help)
-		usage
-		exit 0
-		;;
-	--)
-		shift
-		break
-		;;
-	*)
-		echo 1>&2 "Unknown option \"$1\""
-		usage
-		exit 1
-	esac
-	shift
-done
-
-CFLAGS="-fPIC"
-
-case "$ARCH" in
-arm|arm32|aarch32)
-	GCC_TOOLCHAIN="arm-linux-androideabi-4.9"
-	CLANG_TARGET="armv7-none-linux-androideabi"
-	if $ENABLE_CRYPTO; then
-		CFLAGS+=" -march=armv7-a -mfloat-abi=softfp -mfpu=crypto-neon-fp-armv8"
-	elif $ENABLE_NEON; then
-		CFLAGS+=" -march=armv7-a -mfloat-abi=softfp -mfpu=neon"
-	else
-		CFLAGS+=" -march=armv6"
-	fi
-	CFLAGS+=" --sysroot=\"$NDKDIR/platforms/android-12/arch-arm\""
-	;;
-arm64|aarch64)
-	GCC_TOOLCHAIN="aarch64-linux-android-4.9"
-	CLANG_TARGET="aarch64-none-linux-android"
-	if $ENABLE_CRYPTO; then
-		CFLAGS+=" -march=armv8-a+crypto"
-	else
-		CFLAGS+=" -march=armv8-a"
-	fi
-	CFLAGS+=" --sysroot=\"$NDKDIR/platforms/android-21/arch-arm64\""
-	;;
-*)
-	echo 1>&2 "Unknown architecture: \"$ARCH\""
-	usage
-	exit 1
-esac
-
-case "$COMPILER" in
-gcc)
-	CC="\"$NDKDIR/toolchains/$GCC_TOOLCHAIN/prebuilt/linux-x86_64/bin/${GCC_TOOLCHAIN%-*}-gcc\""
-	;;
-clang)
-	CC="\"$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\""
-	CFLAGS+=" -target \"$CLANG_TARGET\""
-	CFLAGS+=" -gcc-toolchain \"$NDKDIR/toolchains/$GCC_TOOLCHAIN/prebuilt/linux-x86_64\""
-	;;
-*)
-	echo 1>&2 "Unknown compiler: \"$COMPILER\""
-	usage
-	exit 1
-esac
-
-make -j$(grep -c processor /proc/cpuinfo) test_programs \
-	CC="$CC" CFLAGS="$CFLAGS" LDFLAGS="-pie"

tools/checksum_benchmarks.sh

@@ -1,153 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-if [ $# -eq 0 ]; then
-	FILE="$HOME/data/silesia"
-	[ ! -e "$FILE" ] && FILE="$HOME/silesia"
-	[ ! -e "$FILE" ] && FILE="$HOME/data/testdata"
-	[ ! -e "$FILE" ] && FILE="$HOME/testdata"
-	echo "Using default FILE: $FILE"
-	echo
-elif [ $# -eq 1 ]; then
-	FILE="$1"
-else
-	echo "Usage: $0 [FILE]" 1>&2
-	exit 1
-fi
-
-ARCH="$(uname -m)"
-
-if [ -n "$(git status -s | egrep 'adler32_impl.h|crc32_impl.h')" ]; then
-	echo "This script will overwrite adler32_impl.h and crc32_impl.h," \
-		"which have uncommitted changes.  Refusing to run." 1>&2
-	exit 1
-fi
-
-have_cpu_feature() {
-	local feature="$1"
-	local tag
-	case $ARCH in
-	arm*|aarch*)
-		tag="Features"
-		;;
-	*)
-		tag="flags"
-		;;
-	esac
-	grep -q "^$tag"$'[ \t]'"*:.*\<$feature\>" /proc/cpuinfo
-}
-
-make_and_test() {
-	make "$@" checksum test_checksums > /dev/null
-	./test_checksums > /dev/null
-}
-
-__do_benchmark() {
-	local impl="$1" speed
-	shift
-	local flags="$CKSUM_FLAGS $*"
-
-	speed=$(./checksum $flags -t "$FILE" | \
-		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
-	printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
-}
-
-do_benchmark() {
-	local impl="$1"
-
-	if [ "$impl" = zlib ]; then
-		__do_benchmark "$impl" "-Z"
-	else
-		make_and_test CFLAGS="$EXTRA_CFLAGS"
-		__do_benchmark "libdeflate, $impl"
-		if [ $ARCH = x86_64 ]; then
-			make_and_test CFLAGS="-m32 $EXTRA_CFLAGS"
-			__do_benchmark "libdeflate, $impl, 32-bit"
-		fi
-	fi
-}
-
-sort_by_speed() {
-	awk '{print $NF, $0}' | sort -nr | cut -f2- -d' '
-}
-
-disable_impl() {
-	local name="$1"
-	local extra_cflags="$2"
-
-	sed -i '/^\#ifdef DISPATCH_'"$name"'$/aif (0)' lib/*/{adler,crc}32_impl.h
-	EXTRA_CFLAGS+=" $extra_cflags"
-}
-
-restore_impls() {
-	git checkout -f lib/*/{adler,crc}32_impl.h
-}
-
-trap restore_impls EXIT
-
-cat << EOF
-Method                                       Speed (MB/s)
-------                                       ------------
-EOF
-
-# CRC-32
-CKSUM_NAME="CRC-32"
-CKSUM_FLAGS=""
-EXTRA_CFLAGS=""
-{
-case $ARCH in
-i386|x86_64)
-	if have_cpu_feature pclmulqdq && have_cpu_feature avx; then
-		do_benchmark "PCLMUL/AVX"
-		disable_impl "PCLMUL_AVX" "-mno-avx"
-	fi
-	if have_cpu_feature pclmulqdq; then
-		do_benchmark "PCLMUL"
-		disable_impl "PCLMUL" "-mno-pclmul"
-	fi
-	;;
-arm*|aarch*)
-	if have_cpu_feature pmull; then
-		do_benchmark "PMULL"
-		disable_impl "PMULL" ""
-	fi
-	;;
-esac
-do_benchmark "generic"
-do_benchmark "zlib"
-} | sort_by_speed
-
-# Adler-32
-CKSUM_NAME="Adler-32"
-CKSUM_FLAGS="-A"
-EXTRA_CFLAGS=""
-echo
-{
-case $ARCH in
-i386|x86_64)
-	if have_cpu_feature avx2; then
-		do_benchmark "AVX2"
-		disable_impl "AVX2" "-mno-avx2"
-	fi
-	if have_cpu_feature sse2; then
-		do_benchmark "SSE2"
-		disable_impl "SSE2" "-mno-sse2"
-	fi
-	;;
-arm*)
-	if have_cpu_feature neon; then
-		do_benchmark "NEON"
-		disable_impl "NEON" "-mfpu=vfpv3"
-	fi
-	;;
-aarch*)
-	if have_cpu_feature asimd; then
-		do_benchmark "NEON"
-		disable_impl "NEON" "-march=armv8-a+nosimd"
-	fi
-	;;
-esac
-do_benchmark "generic"
-do_benchmark "zlib"
-} | sort_by_speed

tools/exec_tests.sh

@@ -1,30 +0,0 @@
-#
-# Helper script used by run_tests.sh, not intended to be run directly
-#
-
-set -eu
-
-run_cmd() {
-	echo "$WRAPPER $@"
-	$WRAPPER "$@" > /dev/null
-}
-
-run_cmd ./test_checksums
-
-for format in '' '-g' '-z'; do
-	for ref_impl in '' '-Y' '-Z'; do
-		run_cmd ./benchmark $format $ref_impl $SMOKEDATA
-	done
-done
-for level in 1 3 7 9; do
-	for ref_impl in '' '-Y'; do
-		run_cmd ./benchmark -$level $ref_impl $SMOKEDATA
-	done
-done
-for level in 1 3 7 9 12; do
-	for ref_impl in '' '-Z'; do
-		run_cmd ./benchmark -$level $ref_impl $SMOKEDATA
-	done
-done
-
-echo "exec_tests finished successfully" # Needed for 'adb shell'

tools/make-windows-releases

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-for arch in 'i686' 'x86_64'; do
-	make clean
-	make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all \
-		benchmark.exe checksum.exe
-	dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin
-	rm -rf $dir ${dir}.zip
-	mkdir $dir
-	cp libdeflate.dll libdeflate.lib libdeflate.h *.exe $dir
-	${arch}-w64-mingw32-strip ${dir}/libdeflate.dll ${dir}/*.exe
-	for file in COPYING NEWS; do
-		sed < $file > ${dir}/${file}.txt -e 's/$/\r/g'
-	done
-	for file in README.md; do
-		sed < $file > ${dir}/${file} -e 's/$/\r/g'
-	done
-	(cd ${dir} && zip -r ../${dir}.zip .)
-done

tools/mips_build.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-TOOLCHAIN_DIR=$HOME/src/ddwrt-toolchains/toolchain-mips_34kc_gcc-5.1.0_musl-1.1.9
-
-make -j$(grep -c processor /proc/cpuinfo) all test_programs \
-	CC="$TOOLCHAIN_DIR/bin/mips-openwrt-linux-musl-gcc" \
-	CFLAGS="-DNEED_PRINTF -Werror"

tools/produce_gzip_benchmark_table.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-set -e
-
-do_benchmark() {
-	usize=$(stat -c %s "$file")
-	"$HOME/proj/libdeflate/benchmark" -g -s $usize "$@" "$file" \
-			| grep Compressed | cut -f 4 -d ' '
-}
-
-echo "File | zlib -6 | zlib -9 | libdeflate -6 | libdeflate -9 | libdeflate -12"
-echo "-----|---------|---------|---------------|---------------|---------------"
-
-for file in "$@"; do
-	echo -n "$(basename "$file")"
-	results=()
-	results+=($(do_benchmark -Y -6))
-	results+=($(do_benchmark -Y -9))
-	results+=($(do_benchmark -6))
-	results+=($(do_benchmark -9))
-	results+=($(do_benchmark -12))
-	best=2000000000
-	for result in "${results[@]}"; do
-		if (( result < best)); then
-			best=$result
-		fi
-	done
-	for result in "${results[@]}"; do
-		if (( result == best )); then
-			em="**"
-		else
-			em=""
-		fi
-		echo -n " | ${em}${result}${em}"
-	done
-	echo
-done

tools/run_tests.sh

@@ -1,366 +0,0 @@
-#!/bin/bash
-#
-# Test script for libdeflate
-#
-#	Usage: ./tools/run_tests.sh [TESTGROUP]... [-TESTGROUP]...
-#
-# By default all tests are run, but it is possible to explicitly include or
-# exclude specific test groups.
-#
-
-set -eu -o pipefail
-cd "$(dirname "$0")/.."
-
-TESTGROUPS=(all)
-
-set_test_groups() {
-	TESTGROUPS=("$@")
-	local have_exclusion=0
-	local have_all=0
-	for group in "${TESTGROUPS[@]}"; do
-		if [[ $group == -* ]]; then
-			have_exclusion=1
-		elif [[ $group == all ]]; then
-			have_all=1
-		fi
-	done
-	if (( have_exclusion && !have_all )); then
-		TESTGROUPS=(all "${TESTGROUPS[@]}")
-	fi
-}
-
-if [ $# -gt 0 ]; then
-	set_test_groups "$@"
-fi
-
-SMOKEDATA="${SMOKEDATA:=$HOME/data/smokedata}"
-if [ ! -e "$SMOKEDATA" ]; then
-	echo "SMOKEDATA (value: $SMOKEDATA) does not exist.  Set the" \
-	      "environmental variable SMOKEDATA to a file to use in" \
-	      "compression/decompression tests." 1>&2
-	exit 1
-fi
-
-NDKDIR="${NDKDIR:=/opt/android-ndk}"
-
-FILES=("$SMOKEDATA" ./tools/exec_tests.sh benchmark test_checksums)
-EXEC_TESTS_CMD="WRAPPER= SMOKEDATA=\"$(basename $SMOKEDATA)\" sh exec_tests.sh"
-NPROC=$(grep -c processor /proc/cpuinfo)
-VALGRIND="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all"
-SANITIZE_CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined,integer"
-
-TMPFILE="$(mktemp)"
-trap "rm -f \"$TMPFILE\"" EXIT
-
-###############################################################################
-
-rm -f run_tests.log
-exec >  >(tee -ia run_tests.log)
-exec 2> >(tee -ia run_tests.log >&2)
-
-TESTS_SKIPPED=0
-log_skip() {
-	log "[WARNING, TEST SKIPPED]: $@"
-	TESTS_SKIPPED=1
-}
-
-log() {
-	echo "[$(date)] $@"
-}
-
-run_cmd() {
-	log "$@"
-	"$@" > /dev/null
-}
-
-test_group_included() {
-	local included=0 group
-	for group in "${TESTGROUPS[@]}"; do
-		if [ "$group" = "$1" ]; then
-			included=1 # explicitly included
-			break
-		fi
-		if [ "$group" = "-$1" ]; then
-			included=0 # explicitly excluded
-			break
-		fi
-		if [ "$group" = "all" ]; then # implicitly included
-			included=1
-		fi
-	done
-	if (( included )); then
-		log "Starting test group: $1"
-	fi
-	(( included ))
-}
-
-have_valgrind() {
-	if ! type -P valgrind > /dev/null; then
-		log_skip "valgrind not found; can't run tests with valgrind"
-		return 1
-	fi
-}
-
-have_ubsan() {
-	if ! type -P clang > /dev/null; then
-		log_skip "clang not found; can't run tests with UBSAN"
-		return 1
-	fi
-}
-
-have_python() {
-	if ! type -P python3 > /dev/null; then
-		log_skip "Python not found"
-		return 1
-	fi
-}
-
-###############################################################################
-
-native_build_and_test() {
-	make "$@" -j$NPROC all test_programs > /dev/null
-	WRAPPER="$WRAPPER" SMOKEDATA="$SMOKEDATA" sh ./tools/exec_tests.sh \
-			> /dev/null
-}
-
-native_tests() {
-	test_group_included native || return 0
-	local compiler compilers_to_try=(gcc)
-	local cflags cflags_to_try=("")
-	shopt -s nullglob
-	compilers_to_try+=(/usr/bin/gcc-[0-9]*)
-	compilers_to_try+=(/usr/bin/clang-[0-9]*)
-	compilers_to_try+=(/opt/gcc*/bin/gcc)
-	compilers_to_try+=(/opt/clang*/bin/clang)
-	shopt -u nullglob
-
-	if [ "$(uname -m)" = "x86_64" ]; then
-		cflags_to_try+=("-march=native")
-		cflags_to_try+=("-m32")
-	fi
-	for compiler in ${compilers_to_try[@]}; do
-		for cflags in "${cflags_to_try[@]}"; do
-			if [ "$cflags" = "-m32" ] && \
-			   $compiler -v |& grep -q -- '--disable-multilib'
-			then
-				continue
-			fi
-			log "Running tests with CC=$compiler," \
-				"CFLAGS=$cflags"
-			WRAPPER= native_build_and_test \
-				CC=$compiler CFLAGS="$cflags -Werror"
-		done
-	done
-
-	if have_valgrind; then
-		log "Running tests with Valgrind"
-		WRAPPER="$VALGRIND" native_build_and_test
-	fi
-
-	if have_ubsan; then
-		log "Running tests with undefined behavior sanitizer"
-		WRAPPER= native_build_and_test CC=clang CFLAGS="$SANITIZE_CFLAGS"
-	fi
-}
-
-###############################################################################
-
-checksum_benchmarks() {
-	test_group_included checksum_benchmarks || return 0
-	./tools/checksum_benchmarks.sh
-}
-
-###############################################################################
-
-android_build_and_test() {
-	run_cmd ./tools/android_build.sh --ndkdir="$NDKDIR" "$@"
-	run_cmd adb push "${FILES[@]}" /data/local/tmp/
-
-	# Note: adb shell always returns 0, even if the shell command fails...
-	log "adb shell \"cd /data/local/tmp && $EXEC_TESTS_CMD\""
-	adb shell "cd /data/local/tmp && $EXEC_TESTS_CMD" > "$TMPFILE"
-	if ! grep -q "exec_tests finished successfully" "$TMPFILE"; then
-		log "Android test failure!  adb shell output:"
-		cat "$TMPFILE"
-		return 1
-	fi
-}
-
-android_tests() {
-	local compiler
-
-	test_group_included android || return 0
-	if [ ! -e $NDKDIR ]; then
-		log_skip "Android NDK was not found in NDKDIR=$NDKDIR!" \
-		         "If you want to run the Android tests, set the" \
-			 "environmental variable NDKDIR to the location of" \
-			 "your Android NDK installation"
-		return 0
-	fi
-
-	if ! type -P adb > /dev/null; then
-		log_skip "adb (android-tools) is not installed"
-		return 0
-	fi
-
-	if ! adb devices | grep -q 'device$'; then
-		log_skip "No Android device is currently attached"
-		return 0
-	fi
-
-	for compiler in gcc clang; do
-		for flags in "" "--enable-neon" "--enable-crypto"; do
-			for arch in arm32 arm64; do
-				android_build_and_test --arch=$arch \
-					--compiler=$compiler $flags
-			done
-		done
-	done
-}
-
-###############################################################################
-
-mips_tests() {
-	test_group_included mips || return 0
-	if [ "$(hostname)" != "zzz" ] || [ "$(uname -m)" != "x86_64" ]; then
-		log_skip "MIPS tests are not supported on this host"
-		return 0
-	fi
-	if ! ping -c 1 dd-wrt > /dev/null; then
-		log_skip "Can't run MIPS tests: dd-wrt system not available"
-		return 0
-	fi
-	run_cmd ./tools/mips_build.sh
-	run_cmd scp "${FILES[@]}" root@dd-wrt:
-	run_cmd ssh root@dd-wrt "$EXEC_TESTS_CMD"
-
-	log "Checking that compression on big endian CPU produces same output"
-	run_cmd scp gzip root@dd-wrt:
-	run_cmd ssh root@dd-wrt \
-		"rm -f big*.gz;
-		 ./gzip -c -6 $(basename $SMOKEDATA) > big6.gz;
-		 ./gzip -c -10 $(basename $SMOKEDATA) > big10.gz"
-	run_cmd scp root@dd-wrt:big*.gz .
-	make -j$NPROC gzip > /dev/null
-	./gzip -c -6 "$SMOKEDATA" > little6.gz
-	./gzip -c -10 "$SMOKEDATA" > little10.gz
-	if ! cmp big6.gz little6.gz || ! cmp big10.gz little10.gz; then
-		echo 1>&2 "Compressed data differed on big endian vs. little endian!"
-		return 1
-	fi
-	rm big*.gz little*.gz
-}
-
-###############################################################################
-
-windows_tests() {
-	local arch
-
-	test_group_included windows || return 0
-
-	# Windows: currently compiled but not run
-	for arch in i686 x86_64; do
-		local compiler=${arch}-w64-mingw32-gcc
-		if ! type -P $compiler > /dev/null; then
-			log_skip "$compiler not found"
-			continue
-		fi
-		run_cmd make CC=$compiler CFLAGS=-Werror -j$NPROC \
-			all test_programs
-	done
-}
-
-###############################################################################
-
-static_analysis_tests() {
-	test_group_included static_analysis || return 0
-	if ! type -P scan-build > /dev/null; then
-		log_skip "clang static analyzer (scan-build) not found"
-		return 0
-	fi
-	run_cmd scan-build --status-bugs make -j$NPROC all test_programs
-}
-
-###############################################################################
-
-gzip_tests() {
-	test_group_included gzip || return 0
-
-	local gzip gunzip
-	run_cmd make -j$NPROC gzip gunzip
-	for gzip in "$PWD/gzip" /bin/gzip; do
-		for gunzip in "$PWD/gunzip" /bin/gunzip; do
-			log "Running gzip program tests with GZIP=$gzip," \
-				"GUNZIP=$gunzip"
-			GZIP="$gzip" GUNZIP="$gunzip" SMOKEDATA="$SMOKEDATA" \
-				./tools/gzip_tests.sh
-		done
-	done
-
-	if have_valgrind; then
-		log "Running gzip program tests with Valgrind"
-		GZIP="$VALGRIND $PWD/gzip" GUNZIP="$VALGRIND $PWD/gunzip" \
-			SMOKEDATA="$SMOKEDATA" ./tools/gzip_tests.sh
-	fi
-
-	if have_ubsan; then
-		log "Running gzip program tests with undefined behavior sanitizer"
-		run_cmd make -j$NPROC CC=clang CFLAGS="$SANITIZE_CFLAGS" gzip gunzip
-		GZIP="$PWD/gzip" GUNZIP="$PWD/gunzip" \
-			SMOKEDATA="$SMOKEDATA" ./tools/gzip_tests.sh
-	fi
-}
-
-###############################################################################
-
-edge_case_tests() {
-	test_group_included edge_case || return 0
-
-	# Regression test for "deflate_compress: fix corruption with long
-	# literal run".  Try to compress a file longer than 65535 bytes where no
-	# 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes
-	# apart, and the distribution of bytes remains constant throughout, and
-	# yet not all bytes are used so the data is still slightly compressible.
-	# There will be no matches in this data, but the compressor should still
-	# output a compressed block, and this block should contain more than
-	# 65535 consecutive literals, which triggered the bug.
-	#
-	# Note: on random data, this situation is extremely unlikely if the
-	# compressor uses all matches it finds, since random data will on
-	# average have a 3-byte match every (256**3)/32768 = 512 bytes.
-	if have_python; then
-		python3 > "$TMPFILE" << EOF
-import sys
-for i in range(2):
-    for stride in range(1,251):
-        b = bytes(stride*multiple % 251 for multiple in range(251))
-        sys.stdout.buffer.write(b)
-EOF
-		run_cmd make -j$NPROC benchmark
-		run_cmd ./benchmark -3 "$TMPFILE"
-		run_cmd ./benchmark -6 "$TMPFILE"
-		run_cmd ./benchmark -12 "$TMPFILE"
-	fi
-}
-
-###############################################################################
-
-log "Starting libdeflate tests"
-log "	TESTGROUPS=(${TESTGROUPS[@]})"
-log "	SMOKEDATA=$SMOKEDATA"
-log "	NDKDIR=$NDKDIR"
-
-native_tests
-checksum_benchmarks
-android_tests
-mips_tests
-windows_tests
-static_analysis_tests
-gzip_tests
-edge_case_tests
-
-if (( TESTS_SKIPPED )); then
-	log "No tests failed, but some tests were skipped.  See above."
-else
-	log "All tests passed!"
-fi

tools/windows_build.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-make -j CC=i686-w64-mingw32-gcc all test_programs
-cp -vf *.exe /j/exe/
-make -j CC=x86_64-w64-mingw32-gcc all test_programs
-cp -vf *.exe /j/exe64/
-
-sudo $HOME/bin/sudo/restart-smbd