deflate_compress: optimize gen_codewords()

This can be slightly faster.

.cirrus.yml

@@ -0,0 +1,8 @@
+task:
+  freebsd_instance:
+    matrix:
+      - image_family: freebsd-12-3
+      - image_family: freebsd-13-0
+  install_script: pkg install -y gmake
+  script:
+    - gmake check

.github/workflows/android.yml

@@ -0,0 +1,104 @@
+name: cross compile with android-ndk
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  cross-compile-for-android:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11]
+        target-abi: [armeabi-v7a, arm64-v8a, x86, x86_64]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - run:  brew install tree file
+
+      - run:  tree --version
+      - run:  file --version
+      - run:  make --version
+
+      - uses: actions/checkout@v2
+
+      - name: build for ${{ matrix.target-abi }}
+        run: |
+          COLOR_GREEN='\033[0;32m'        # Green
+          COLOR_PURPLE='\033[0;35m'       # Purple
+          COLOR_OFF='\033[0m'             # Reset
+
+          run() {
+            printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF $COLOR_GREEN$*$COLOR_OFF"
+            eval "$*"
+          }
+
+          BUILD_MACHINE_OS_TYPE=$(uname | tr A-Z a-z)
+          BUILD_MACHINE_OS_ARCH=$(uname -m)
+
+          export ANDROID_NDK_HOME="$ANDROID_NDK_LATEST_HOME"
+
+          env | sed -n '/^ANDROID_/p'
+
+          ANDROID_NDK_VERS=$(grep "Pkg.Revision" "$ANDROID_NDK_HOME/source.properties" | cut -d " " -f3)
+          ANDROID_NDK_BASE=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$BUILD_MACHINE_OS_TYPE-$BUILD_MACHINE_OS_ARCH
+          ANDROID_NDK_BIND=$ANDROID_NDK_BASE/bin
+          SYSROOT=$ANDROID_NDK_BASE/sysroot
+
+          printf '%s = %s\n' ANDROID_NDK_HOME "$ANDROID_NDK_HOME"
+          printf '%s = %s\n' ANDROID_NDK_BIND "$ANDROID_NDK_BIND"
+          printf '%s = %s\n' ANDROID_NDK_VERS "$ANDROID_NDK_VERS"
+          printf '%s\n'
+
+          export CPPFLAGS="--sysroot $SYSROOT"
+          export   CFLAGS="--sysroot $SYSROOT -fpic"
+          export  LDFLAGS="--sysroot $SYSROOT"
+
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) CC=armv7a-linux-androideabi21-clang ;;
+            arm64-v8a)   CC=aarch64-linux-android21-clang    ;;
+            x86)         CC=i686-linux-android21-clang       ;;
+            x86_64)      CC=x86_64-linux-android21-clang     ;;
+          esac
+
+          AR=llvm-ar
+
+          export PATH="$ANDROID_NDK_BIND:$PATH"
+
+          printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF ${COLOR_GREEN}show PATH$COLOR_OFF"
+          printf '%s\n' "$PATH" | tr ':' '\n'
+          printf '%s\n'
+
+          run make clean install PREFIX=$PWD/install.d CC="$CC" CPPFLAGS="'$CPPFLAGS'" CFLAGS="'$CFLAGS'" AR="$AR" V=1
+
+      - run:  tree install.d
+
+      - run:  |
+          file install.d/lib/libdeflate.so
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, ARM aarch64,' ;;
+            x86)         file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, Intel 80386,' ;;
+            x86_64)      file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, x86-64,'      ;;
+          esac
+
+      - run:  |
+          file install.d/bin/libdeflate-gzip
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
+            x86)         file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
+            x86_64)      file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
+          esac
+
+      - run:  |
+          file install.d/bin/libdeflate-gunzip
+          case ${{ matrix.target-abi }} in
+            armeabi-v7a) file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
+            arm64-v8a)   file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
+            x86)         file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
+            x86_64)      file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
+          esac

.github/workflows/ci.yml

@@ -0,0 +1,118 @@
+name: CI
+on: [pull_request]
+env:
+  CFLAGS: -Werror
+
+jobs:
+  x86_64-build-and-test:
+    name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, ubuntu-18.04]
+        compiler: [gcc, clang]
+    runs-on: ${{ matrix.os }}
+    env:
+      CC: ${{ matrix.compiler }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang llvm libz-dev valgrind
+    - run: scripts/run_tests.sh
+
+  other-arch-build-and-test:
+    name: Build and test (${{ matrix.arch }}, Debian Buster, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        arch: [armv6, armv7, aarch64, s390x, ppc64le]
+        compiler: [gcc, clang]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: uraimo/run-on-arch-action@v2.0.5
+        with:
+          arch: ${{ matrix.arch }}
+          distro: buster
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update
+            apt-get install -y build-essential clang llvm libz-dev valgrind
+          run: |
+            # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64
+            # here.  (It's probably something related to the QEMU user-mode
+            # emulation that run-on-arch-action uses.)
+            export SKIP_VALGRIND=1
+            export SKIP_ASAN=1
+
+            case ${{ matrix.arch }} in
+            s390x)
+              # On s390x, in freestanding builds the shared library links to an
+              # external symbol __clzdi2, even when -static-libgcc is used.
+              export SKIP_FREESTANDING=1
+              ;;
+            aarch64)
+              # "ldd: exited with unknown exit code (139)"
+              if [ ${{ matrix.compiler }} = clang ]; then
+                export SKIP_SHARED_LIB=1
+              fi
+              ;;
+            esac
+
+            export CC=${{ matrix.compiler }}
+            scripts/run_tests.sh
+
+  macos-build-and-test:
+    name: Build and test (macOS)
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: make all check
+
+  windows-build-and-test:
+    name: Build and test (Windows)
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - shell: bash
+      run: |
+        PATH="C:\\msys64\\mingw64\\bin:C:\\msys64\\usr\\bin:$PATH" \
+          make CC=gcc all check
+
+  run-clang-static-analyzer:
+    name: Run clang static analyzer
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang-tools
+    - name: Run clang static analyzer
+      run: make scan-build
+
+  run-shellcheck:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y shellcheck
+    - name: Run shellcheck
+      run: make shellcheck
+
+  cross-compile-for-windows:
+    name: Cross compile for Windows
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev
+    - name: 32-bit build
+      run: make CC=i686-w64-mingw32-gcc all test_programs
+    - name: 64-bit build
+      run: make CC=x86_64-w64-mingw32-gcc all test_programs

.gitignore

@@ -1,16 +1,21 @@
 *.a
+*.def
 *.dll
+*.dllobj
+*.dylib
 *.exe
 *.exp
 *.lib
 *.o
 *.obj
 *.so
-/.lib-cflags
-/.prog-cflags
+*.so.*
+/.build-config
 /programs/config.h
 /benchmark
+/checksum
 /gzip
 /gunzip
+/test_*
 tags
 cscope*

COPYING.LIB

@@ -1,121 +0,0 @@
-Creative Commons Legal Code
-
-CC0 1.0 Universal
-
-    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
-    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
-    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
-    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
-    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
-    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
-    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
-    HEREUNDER.
-
-Statement of Purpose
-
-The laws of most jurisdictions throughout the world automatically confer
-exclusive Copyright and Related Rights (defined below) upon the creator
-and subsequent owner(s) (each and all, an "owner") of an original work of
-authorship and/or a database (each, a "Work").
-
-Certain owners wish to permanently relinquish those rights to a Work for
-the purpose of contributing to a commons of creative, cultural and
-scientific works ("Commons") that the public can reliably and without fear
-of later claims of infringement build upon, modify, incorporate in other
-works, reuse and redistribute as freely as possible in any form whatsoever
-and for any purposes, including without limitation commercial purposes.
-These owners may contribute to the Commons to promote the ideal of a free
-culture and the further production of creative, cultural and scientific
-works, or to gain reputation or greater distribution for their Work in
-part through the use and efforts of others.
-
-For these and/or other purposes and motivations, and without any
-expectation of additional consideration or compensation, the person
-associating CC0 with a Work (the "Affirmer"), to the extent that he or she
-is an owner of Copyright and Related Rights in the Work, voluntarily
-elects to apply CC0 to the Work and publicly distribute the Work under its
-terms, with knowledge of his or her Copyright and Related Rights in the
-Work and the meaning and intended legal effect of CC0 on those rights.
-
-1. Copyright and Related Rights. A Work made available under CC0 may be
-protected by copyright and related or neighboring rights ("Copyright and
-Related Rights"). Copyright and Related Rights include, but are not
-limited to, the following:
-
-  i. the right to reproduce, adapt, distribute, perform, display,
-     communicate, and translate a Work;
- ii. moral rights retained by the original author(s) and/or performer(s);
-iii. publicity and privacy rights pertaining to a person's image or
-     likeness depicted in a Work;
- iv. rights protecting against unfair competition in regards to a Work,
-     subject to the limitations in paragraph 4(a), below;
-  v. rights protecting the extraction, dissemination, use and reuse of data
-     in a Work;
- vi. database rights (such as those arising under Directive 96/9/EC of the
-     European Parliament and of the Council of 11 March 1996 on the legal
-     protection of databases, and under any national implementation
-     thereof, including any amended or successor version of such
-     directive); and
-vii. other similar, equivalent or corresponding rights throughout the
-     world based on applicable law or treaty, and any national
-     implementations thereof.
-
-2. Waiver. To the greatest extent permitted by, but not in contravention
-of, applicable law, Affirmer hereby overtly, fully, permanently,
-irrevocably and unconditionally waives, abandons, and surrenders all of
-Affirmer's Copyright and Related Rights and associated claims and causes
-of action, whether now known or unknown (including existing as well as
-future claims and causes of action), in the Work (i) in all territories
-worldwide, (ii) for the maximum duration provided by applicable law or
-treaty (including future time extensions), (iii) in any current or future
-medium and for any number of copies, and (iv) for any purpose whatsoever,
-including without limitation commercial, advertising or promotional
-purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
-member of the public at large and to the detriment of Affirmer's heirs and
-successors, fully intending that such Waiver shall not be subject to
-revocation, rescission, cancellation, termination, or any other legal or
-equitable action to disrupt the quiet enjoyment of the Work by the public
-as contemplated by Affirmer's express Statement of Purpose.
-
-3. Public License Fallback. Should any part of the Waiver for any reason
-be judged legally invalid or ineffective under applicable law, then the
-Waiver shall be preserved to the maximum extent permitted taking into
-account Affirmer's express Statement of Purpose. In addition, to the
-extent the Waiver is so judged Affirmer hereby grants to each affected
-person a royalty-free, non transferable, non sublicensable, non exclusive,
-irrevocable and unconditional license to exercise Affirmer's Copyright and
-Related Rights in the Work (i) in all territories worldwide, (ii) for the
-maximum duration provided by applicable law or treaty (including future
-time extensions), (iii) in any current or future medium and for any number
-of copies, and (iv) for any purpose whatsoever, including without
-limitation commercial, advertising or promotional purposes (the
-"License"). The License shall be deemed effective as of the date CC0 was
-applied by Affirmer to the Work. Should any part of the License for any
-reason be judged legally invalid or ineffective under applicable law, such
-partial invalidity or ineffectiveness shall not invalidate the remainder
-of the License, and in such case Affirmer hereby affirms that he or she
-will not (i) exercise any of his or her remaining Copyright and Related
-Rights in the Work or (ii) assert any associated claims and causes of
-action with respect to the Work, in either case contrary to Affirmer's
-express Statement of Purpose.
-
-4. Limitations and Disclaimers.
-
- a. No trademark or patent rights held by Affirmer are waived, abandoned,
-    surrendered, licensed or otherwise affected by this document.
- b. Affirmer offers the Work as-is and makes no representations or
-    warranties of any kind concerning the Work, express, implied,
-    statutory or otherwise, including without limitation warranties of
-    title, merchantability, fitness for a particular purpose, non
-    infringement, or the absence of latent or other defects, accuracy, or
-    the present or absence of errors, whether or not discoverable, all to
-    the greatest extent permissible under applicable law.
- c. Affirmer disclaims responsibility for clearing rights of other persons
-    that may apply to the Work or any use thereof, including without
-    limitation any person's Copyright and Related Rights in the Work.
-    Further, Affirmer disclaims responsibility for obtaining any necessary
-    consents, permissions or other rights required for any use of the
-    Work.
- d. Affirmer understands and acknowledges that Creative Commons is not a
-    party to this document and has no duty or obligation with respect to
-    this CC0 or use of the Work.

Makefile

@@ -3,6 +3,12 @@
 #
 # Define V=1 to enable "verbose" mode, showing all executed commands.
 #
+# Define USE_SHARED_LIB to link the binaries to the shared library version of
+# libdeflate rather than to the static library version.
+#
+# Define DISABLE_SHARED to disable building and installing of shared library.
+# Defining this variable undefines USE_SHARED_LIB.
+#
 # Define DECOMPRESSION_ONLY to omit all compression code, building a
 # decompression-only library.  If doing this, you must also build a specific
 # library target such as 'libdeflate.a', as the programs will no longer compile.
@@ -11,38 +17,137 @@
 #
 # Define DISABLE_ZLIB to disable support for the zlib wrapper format.
 #
+# Define PREFIX to override the installation prefix, like './configure --prefix'
+# in autotools-based projects (default: /usr/local)
+#
+# Define BINDIR to override where to install binaries, like './configure
+# --bindir' in autotools-based projects (default: PREFIX/bin)
+#
+# Define INCDIR to override where to install headers, like './configure
+# --includedir' in autotools-based projects (default: PREFIX/include)
+#
+# Define LIBDIR to override where to install libraries, like './configure
+# --libdir' in autotools-based projects (default: PREFIX/lib)
+#
+# Define DESTDIR to override the installation destination directory
+# (default: empty string)
+#
+# Define FREESTANDING to build a freestanding library, i.e. a library that
+# doesn't link to any libc functions like malloc(), free(), and memcpy().
+# All users will need to call libdeflate_set_memory_allocator().
+#
+# You can also specify a custom CC, CFLAGS, CPPFLAGS, and/or LDFLAGS.
+#
 ##############################################################################
 
-CC ?= gcc
-AR ?= ar
+VERSION = $(shell sed -n 's/\#define LIBDEFLATE_VERSION_STRING.*"\(.*\)"/\1/p' \
+	    libdeflate.h)
 
-STATIC_LIB_SUFFIX := .a
-SHARED_LIB_SUFFIX := .so
-PROG_SUFFIX       :=
-PROG_CFLAGS       :=
-PIC_REQUIRED      := 1
-HARD_LINKS        := 1
+#### Common compiler flags.  You can add additional flags by defining CFLAGS
+#### in the environment or on the 'make' command line.
+####
+#### The default optimization flags can be overridden, e.g. via CFLAGS="-O3" or
+#### CFLAGS="-O0 -fno-omit-frame-pointer".  But this usually isn't recommended;
+#### you're unlikely to get significantly better performance even with -O3.
 
-# Compiling for Windows with MinGW?
-ifneq ($(findstring -mingw,$(CC)),)
-    ifeq ($(AR),ar)
-        AR := $(patsubst %-gcc,%-ar,$(CC))
-    endif
-    STATIC_LIB_SUFFIX := .lib
-    SHARED_LIB_SUFFIX := .dll
-    PROG_SUFFIX       := .exe
-    PROG_CFLAGS       := -static -municode
-    PIC_REQUIRED      :=
-    HARD_LINKS        :=
+cc-option = $(shell if $(CC) $(1) -c -x c /dev/null -o /dev/null \
+	      1>&2 2>/dev/null; then echo $(1); fi)
+
+override CFLAGS :=							\
+	-O2 -fomit-frame-pointer -std=c99 -I. -Wall -Wundef		\
+	$(call cc-option,-Wpedantic)					\
+	$(call cc-option,-Wdeclaration-after-statement)			\
+	$(call cc-option,-Wmissing-prototypes)				\
+	$(call cc-option,-Wstrict-prototypes)				\
+	$(call cc-option,-Wvla)						\
+	$(call cc-option,-Wimplicit-fallthrough)			\
+	$(CFLAGS)
+
+FREESTANDING :=
+ifdef FREESTANDING
+override CPPFLAGS += -DFREESTANDING
+LIB_CFLAGS += -ffreestanding -nostdlib
+endif
+
+# Don't use this option except for testing; it isn't a stable interface.
+TEST_SUPPORT__DO_NOT_USE :=
+ifdef TEST_SUPPORT__DO_NOT_USE
+override CPPFLAGS += -DTEST_SUPPORT__DO_NOT_USE
 endif
 
 ##############################################################################
 
-#### Common compiler flags; not intended to be overridden
+PREFIX ?= /usr/local
+BINDIR ?= $(PREFIX)/bin
+INCDIR ?= $(PREFIX)/include
+LIBDIR ?= $(PREFIX)/lib
 
-override CFLAGS += -O2 -fomit-frame-pointer -std=gnu89 -I. -Icommon	\
-		-Wall -Wundef -Wdeclaration-after-statement		\
-		-Wmissing-prototypes -Wstrict-prototypes
+SOVERSION          := 0
+
+STATIC_LIB_SUFFIX  := .a
+PROG_SUFFIX        :=
+PROG_CFLAGS        :=
+HARD_LINKS         := 1
+
+TARGET_MACHINE     := $(shell $(CC) -dumpmachine 2>/dev/null)
+
+# Compiling for Windows with MinGW or LLVM?
+ifneq ($(findstring -mingw,$(TARGET_MACHINE))$(findstring -windows-gnu,$(TARGET_MACHINE)),)
+    STATIC_LIB_SUFFIX  := static.lib
+    SHARED_LIB         := libdeflate.dll
+    SHARED_LIB_SYMLINK :=
+    SHARED_LIB_CFLAGS  :=
+    SHARED_LIB_LDFLAGS := -Wl,--out-implib,libdeflate.lib \
+                          -Wl,--output-def,libdeflate.def
+    # Only if not LLVM
+    ifeq ($(findstring -windows-gnu,$(TARGET_MACHINE)),)
+        SHARED_LIB_LDFLAGS += -Wl,--add-stdcall-alias
+    endif
+
+    PROG_SUFFIX        := .exe
+    PROG_CFLAGS        := -static -municode
+    HARD_LINKS         :=
+    override CFLAGS    := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
+
+    # If AR was not already overridden, then derive it from $(CC).
+    # Note that CC may take different forms, e.g. "cc", "gcc", "clang",
+    # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1", or
+    # "x86_64-w64-mingw32-clang".
+    # On Windows it may also have a .exe extension.
+    ifeq ($(AR),ar)
+        AR := $(shell echo $(CC) | \
+                sed -E 's/(g?cc|clang)(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\4/')
+    endif
+
+# Compiling for macOS?
+else ifneq ($(findstring -darwin,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.$(SOVERSION).dylib
+   SHARED_LIB_SYMLINK := libdeflate.dylib
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -install_name $(LIBDIR)/$(SHARED_LIB)
+
+# Compiling for Android?
+else ifneq ($(findstring -android,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.so
+   SHARED_LIB_SYMLINK :=
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
+
+# Compiling for Haiku?
+else ifneq ($(findstring -haiku,$(TARGET_MACHINE)),)
+   SHARED_LIB         := libdeflate.so.$(SOVERSION)
+   SHARED_LIB_SYMLINK := libdeflate.so
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
+   HARD_LINKS         :=
+
+# Compiling for Linux, FreeBSD, etc.
+else
+   SHARED_LIB         := libdeflate.so.$(SOVERSION)
+   SHARED_LIB_SYMLINK := libdeflate.so
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
+endif
 
 ##############################################################################
 
@@ -61,157 +166,259 @@ endif
 
 ##############################################################################
 
-COMMON_HEADERS := $(wildcard common/*.h)
-ALL_TARGETS :=
+# Rebuild if a user-specified setting that affects the build changed.
+.build-config: FORCE
+	@flags=$$(							\
+		echo 'USE_SHARED_LIB=$(USE_SHARED_LIB)';		\
+		echo 'DISABLE_SHARED=$(DISABLE_SHARED)';		\
+		echo 'DECOMPRESSION_ONLY=$(DECOMPRESSION_ONLY)';	\
+		echo 'DISABLE_GZIP=$(DISABLE_GZIP)';			\
+		echo 'DISABLE_ZLIB=$(DISABLE_ZLIB)';			\
+		echo 'FREESTANDING=$(FREESTANDING)';			\
+		echo 'CC=$(CC)';					\
+		echo 'CFLAGS=$(CFLAGS)';				\
+		echo 'CPPFLAGS=$(CPPFLAGS)';				\
+		echo 'LDFLAGS=$(LDFLAGS)';				\
+	);								\
+	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then		\
+		[ -e $@ ] && echo "Rebuilding due to new settings";	\
+		echo "$$flags" > $@;					\
+	fi
+
+##############################################################################
+
+COMMON_HEADERS := $(wildcard common/*.h) libdeflate.h
+DEFAULT_TARGETS :=
 
 #### Library
 
 STATIC_LIB := libdeflate$(STATIC_LIB_SUFFIX)
-SHARED_LIB := libdeflate$(SHARED_LIB_SUFFIX)
 
 LIB_CFLAGS += $(CFLAGS) -fvisibility=hidden -D_ANSI_SOURCE
 
+LIB_HEADERS := $(wildcard lib/*.h) $(wildcard lib/*/*.h)
 
-LIB_SRC := lib/aligned_malloc.c lib/deflate_decompress.c lib/x86_cpu_features.c
+LIB_SRC := lib/deflate_decompress.c lib/utils.c \
+	   $(wildcard lib/*/cpu_features.c)
 
 DECOMPRESSION_ONLY :=
-ifeq ($(DECOMPRESSION_ONLY),)
+ifndef DECOMPRESSION_ONLY
     LIB_SRC += lib/deflate_compress.c
 endif
 
 DISABLE_ZLIB :=
-ifeq ($(DISABLE_ZLIB),)
+ifndef DISABLE_ZLIB
     LIB_SRC += lib/adler32.c lib/zlib_decompress.c
-    ifeq ($(DECOMPRESSION_ONLY),)
+    ifndef DECOMPRESSION_ONLY
         LIB_SRC += lib/zlib_compress.c
     endif
 endif
 
 DISABLE_GZIP :=
-ifeq ($(DISABLE_GZIP),)
+ifndef DISABLE_GZIP
     LIB_SRC += lib/crc32.c lib/gzip_decompress.c
-    ifeq ($(DECOMPRESSION_ONLY),)
+    ifndef DECOMPRESSION_ONLY
         LIB_SRC += lib/gzip_compress.c
     endif
 endif
 
-LIB_OBJ := $(LIB_SRC:.c=.o)
-LIB_PIC_OBJ := $(LIB_SRC:.c=.pic.o)
-ifeq ($(PIC_REQUIRED),)
-    SHLIB_OBJ := $(LIB_OBJ)
-else
-    SHLIB_OBJ := $(LIB_PIC_OBJ)
-endif
+STATIC_LIB_OBJ := $(LIB_SRC:.c=.o)
+SHARED_LIB_OBJ := $(LIB_SRC:.c=.shlib.o)
 
-# Compile position dependent library object files
-$(LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) $<
+# Compile static library object files
+$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) $<
 
-# Compile position independent library object files
-$(LIB_PIC_OBJ): %.pic.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) -fPIC $<
-
-# Link shared library
-$(SHARED_LIB):$(SHLIB_OBJ)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) -shared $+
-
-ALL_TARGETS += $(SHARED_LIB)
+# Compile shared library object files
+$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_CFLAGS) -DLIBDEFLATE_DLL $<
 
 # Create static library
-$(STATIC_LIB):$(LIB_OBJ)
+$(STATIC_LIB):$(STATIC_LIB_OBJ)
 	$(QUIET_AR) $(AR) cr $@ $+
 
-ALL_TARGETS += $(STATIC_LIB)
+DEFAULT_TARGETS += $(STATIC_LIB)
 
-# Rebuild if CC or LIB_CFLAGS changed
-.lib-cflags: FORCE
-	@flags='$(CC):$(LIB_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding library due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
+# Create shared library
+$(SHARED_LIB):$(SHARED_LIB_OBJ)
+	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_LDFLAGS) -shared $+
+
+ifdef DISABLE_SHARED
+undefine SHARED_LIB_SYMLINK
+else
+DEFAULT_TARGETS += $(SHARED_LIB)
+endif
+
+ifdef SHARED_LIB_SYMLINK
+# Create the symlink libdeflate.so => libdeflate.so.$SOVERSION
+$(SHARED_LIB_SYMLINK):$(SHARED_LIB)
+	$(QUIET_LN) ln -sf $+ $@
+DEFAULT_TARGETS += $(SHARED_LIB_SYMLINK)
+endif
 
 ##############################################################################
 
 #### Programs
 
-PROG_CFLAGS += $(CFLAGS)		\
-	       -D_DEFAULT_SOURCE	\
-	       -D_FILE_OFFSET_BITS=64	\
+PROG_CFLAGS += $(CFLAGS)		 \
+	       -D_POSIX_C_SOURCE=200809L \
+	       -D_FILE_OFFSET_BITS=64	 \
 	       -DHAVE_CONFIG_H
 
-PROG_COMMON_HEADERS := programs/prog_util.h programs/config.h
-PROG_COMMON_SRC := programs/prog_util.c programs/tgetopt.c
-PROG_SPECIFIC_SRC := programs/gzip.c programs/benchmark.c
+ALL_PROG_COMMON_HEADERS := programs/config.h \
+			   programs/prog_util.h \
+			   programs/test_util.h
+PROG_COMMON_SRC      := programs/prog_util.c \
+			programs/tgetopt.c
+NONTEST_PROG_SRC     := programs/gzip.c
+TEST_PROG_COMMON_SRC := programs/test_util.c
+TEST_PROG_SRC        := programs/benchmark.c \
+			programs/checksum.c \
+			$(filter-out $(TEST_PROG_COMMON_SRC),$(wildcard programs/test_*.c))
 
-PROG_COMMON_OBJ := $(PROG_COMMON_SRC:.c=.o)
-PROG_SPECIFIC_OBJ := $(PROG_SPECIFIC_SRC:.c=.o)
-PROG_OBJ := $(PROG_COMMON_OBJ) $(PROG_SPECIFIC_OBJ)
+NONTEST_PROGRAMS := $(NONTEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
+DEFAULT_TARGETS  += $(NONTEST_PROGRAMS)
+TEST_PROGRAMS    := $(TEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
+
+PROG_COMMON_OBJ      := $(PROG_COMMON_SRC:%.c=%.o)
+NONTEST_PROG_OBJ     := $(NONTEST_PROG_SRC:%.c=%.o)
+TEST_PROG_COMMON_OBJ := $(TEST_PROG_COMMON_SRC:%.c=%.o)
+TEST_PROG_OBJ        := $(TEST_PROG_SRC:%.c=%.o)
+
+ALL_PROG_OBJ	     := $(PROG_COMMON_OBJ) $(NONTEST_PROG_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(TEST_PROG_OBJ)
 
 # Generate autodetected configuration header
-programs/config.h:programs/detect.sh .prog-cflags
-	$(QUIET_GEN) CC=$(CC) $< > $@
+programs/config.h:scripts/detect.sh .build-config
+	$(QUIET_GEN) CC="$(CC)" CFLAGS="$(PROG_CFLAGS)" $< > $@
 
 # Compile program object files
-$(PROG_OBJ): %.o: %.c $(PROG_COMMON_HEADERS) $(COMMON_HEADERS) .prog-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(PROG_CFLAGS) $<
+$(ALL_PROG_OBJ): %.o: %.c $(ALL_PROG_COMMON_HEADERS) $(COMMON_HEADERS) \
+			.build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(PROG_CFLAGS) $<
 
-# Link benchmark program
-benchmark$(PROG_SUFFIX):programs/benchmark.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
+# Link the programs.
+#
+# Note: the test programs are not compiled by default.  One reason is that the
+# test programs must be linked with zlib for doing comparisons.
 
-# 'benchmark' is not compiled by default because of the zlib link requirement.
-#ALL_TARGETS += benchmark$(PROG_SUFFIX)
+ifdef USE_SHARED_LIB
+LIB := $(SHARED_LIB)
+else
+LIB := $(STATIC_LIB)
+endif
 
-# Link gzip program
-gzip$(PROG_SUFFIX):programs/gzip.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
+$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(LIB)
 	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+
 
-ALL_TARGETS += gzip$(PROG_SUFFIX)
+$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(LIB)
+	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
 
-ifeq ($(HARD_LINKS),)
-# No hard links; copy gzip to gunzip
-gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
-	$(QUIET_CP) cp -f $< $@
-else
+ifdef HARD_LINKS
 # Hard link gunzip to gzip
 gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
 	$(QUIET_LN) ln -f $< $@
+else
+# No hard links; copy gzip to gunzip
+gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
+	$(QUIET_CP) cp -f $< $@
 endif
 
-ALL_TARGETS += gunzip$(PROG_SUFFIX)
-
-# Rebuild if CC or PROG_CFLAGS changed
-.prog-cflags: FORCE
-	@flags='$(CC):$(PROG_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding programs due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
+DEFAULT_TARGETS += gunzip$(PROG_SUFFIX)
 
 ##############################################################################
 
-all:$(ALL_TARGETS)
+all:$(DEFAULT_TARGETS)
+
+# We don't generate the pkg-config file until install time, as needed
+# definitions might not exist until then.
+PKGCONFBASE := libdeflate.pc.in
+
+# Install the files.  Note: not all versions of the 'install' program have the
+# '-D' and '-t' options, so don't use them; use portable commands only.
+install:all $(PKGCONFBASE)
+	install -d $(DESTDIR)$(LIBDIR)/pkgconfig $(DESTDIR)$(INCDIR)	\
+		$(DESTDIR)$(BINDIR)
+	install -m644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)
+	if [ -z "$(DISABLE_SHARED)" ]; then				\
+		install -m755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR);	\
+	fi
+	sed -e "s|@PREFIX@|$(PREFIX)|"					\
+	    -e "s|@LIBDIR@|$(LIBDIR)|"					\
+	    -e "s|@INCDIR@|$(INCDIR)|"					\
+	    -e "s|@VERSION@|$(VERSION)|"				\
+	    $(PKGCONFBASE) > $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
+	chmod 644 $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
+	install -m644 libdeflate.h $(DESTDIR)$(INCDIR)
+	install -m755 gzip$(PROG_SUFFIX) \
+		$(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)
+	if [ -n "$(HARD_LINKS)" ]; then					\
+		ln -f $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)	\
+		      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
+	else								\
+		ln -sf libdeflate-gzip$(PROG_SUFFIX)			\
+		       $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
+	fi
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		ln -sf $(SHARED_LIB)					\
+		       $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
+uninstall:
+	rm -f $(DESTDIR)$(LIBDIR)/$(STATIC_LIB)				\
+	      $(DESTDIR)$(LIBDIR)/$(SHARED_LIB)				\
+	      $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc		\
+	      $(DESTDIR)$(INCDIR)/libdeflate.h				\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)		\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX)
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
+test_programs:$(TEST_PROGRAMS)
+
+# A minimal 'make check' target.  This only runs some quick tests;
+# use scripts/run_tests.sh if you want to run the full tests.
+check:test_programs
+	LD_LIBRARY_PATH=. ./benchmark$(PROG_SUFFIX) < ./benchmark$(PROG_SUFFIX)
+	for prog in test_*; do		\
+		LD_LIBRARY_PATH=. ./$$prog || exit 1;	\
+	done
+
+# Run the clang static analyzer.
+scan-build:
+	scan-build --status-bugs make all test_programs
+
+# Run shellcheck on all shell scripts.
+shellcheck:
+	shellcheck scripts/*.sh
 
 help:
 	@echo "Available targets:"
 	@echo "------------------"
-	@for target in $(ALL_TARGETS) benchmark$(PROG_SUFFIX); do \
+	@for target in $(DEFAULT_TARGETS) $(TEST_PROGRAMS); do \
 		echo -e "$$target";		\
 	done
 
 clean:
-	rm -f *.a *.dll *.exe *.exp *.so \
-		lib/*.o lib/*.obj programs/*.o programs/*.obj \
-		benchmark gzip gunzip programs/config.h \
-		libdeflate.lib libdeflatestatic.lib \
-		.lib-cflags .prog-cflags
+	rm -f *.a *.dll *.exe *.exp *.dylib *.so \
+		lib/*.o lib/*/*.o \
+		lib/*.obj lib/*/*.obj \
+		lib/*.dllobj lib/*/*.dllobj \
+		programs/*.o programs/*.obj \
+		$(DEFAULT_TARGETS) $(TEST_PROGRAMS) programs/config.h \
+		libdeflate.lib libdeflate.def libdeflatestatic.lib \
+		.build-config
 
 realclean: clean
 	rm -f tags cscope*
 
 FORCE:
 
-.PHONY: all help clean realclean
+.PHONY: all install uninstall test_programs check scan-build shellcheck help \
+	clean realclean
 
 .DEFAULT_GOAL = all

Makefile.msc

@@ -5,45 +5,53 @@
 #	nmake /f Makefile.msc
 #
 
+.SUFFIXES: .c .obj .dllobj
+
 CC = cl
 LD = link
 AR = lib
-CFLAGS = /MD /O2 -I. -Icommon
+CFLAGS = /MD /O2 -I.
 LDFLAGS =
 
-STATICLIB = libdeflatestatic.lib
-SHAREDLIB = libdeflate.dll
-IMPLIB    = libdeflate.lib
+STATIC_LIB = libdeflatestatic.lib
+SHARED_LIB = libdeflate.dll
+IMPORT_LIB = libdeflate.lib
 
-LIB_OBJ = lib/aligned_malloc.obj	\
+STATIC_LIB_OBJ =			\
 	  lib/adler32.obj		\
 	  lib/crc32.obj			\
 	  lib/deflate_compress.obj	\
 	  lib/deflate_decompress.obj	\
 	  lib/gzip_compress.obj		\
 	  lib/gzip_decompress.obj	\
-	  lib/x86_cpu_features.obj	\
+	  lib/utils.obj			\
+	  lib/x86/cpu_features.obj	\
 	  lib/zlib_compress.obj		\
 	  lib/zlib_decompress.obj
 
+SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj)
+
 PROG_COMMON_OBJ = programs/prog_util.obj \
 		  programs/tgetopt.obj \
-		  $(STATICLIB)
+		  $(STATIC_LIB)
 
 PROG_CFLAGS = $(CFLAGS) -Iprograms
 
-all: $(STATICLIB) $(SHAREDLIB) $(IMPLIB) gzip.exe gunzip.exe
+all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe
 
 .c.obj:
 	$(CC) -c /Fo$@ $(CFLAGS) $**
 
-$(STATICLIB): $(LIB_OBJ)
-	$(AR) $(ARFLAGS) -out:$@ $(LIB_OBJ)
+.c.dllobj:
+	$(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_DLL $**
 
-$(SHAREDLIB): $(LIB_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPLIB) $(LIB_OBJ)
+$(STATIC_LIB): $(STATIC_LIB_OBJ)
+	$(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ)
 
-$(IMPLIB): $(SHAREDLIB)
+$(SHARED_LIB): $(SHARED_LIB_OBJ)
+	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ)
+
+$(IMPORT_LIB): $(SHARED_LIB)
 
 gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ)
 	$(LD) $(LDFLAGS) -out:$@ $**
@@ -53,4 +61,5 @@ gunzip.exe:gzip.exe
 
 clean:
 	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
-		lib\*.obj programs\*.obj 2>nul
+		lib\*.obj lib\x86\*.obj lib\*.dllobj lib\x86\*.dllobj \
+		programs\*.obj 2>nul

NEWS

@@ -1,16 +0,0 @@
-Version 0.2:
-	Implemented a new block splitting algorithm which typically improves the
-	compression ratio slightly at all compression levels.
-
-	The compressor now outputs each block using the cheapest type (dynamic
-	Huffman, static Huffman, or uncompressed).
-
-	The gzip program has received an overhaul and now behaves more like the
-	standard version.
-
-	Build system updates, including: some build options were changed and
-	some build options were removed, and the default 'make' target now
-	includes the gzip program as well as the library.
-
-Version 0.1:
-	Initial official release.

NEWS.md

@@ -0,0 +1,279 @@
+# libdeflate release notes
+
+## Version 1.10
+
+* Added an additional check to the decompressor to make it quickly detect
+  certain bad inputs and not try to generate an unbounded amount of output.
+
+  Note: this was only a problem when decompressing with an unknown output size,
+  which isn't the recommended use case of libdeflate.  However,
+  `libdeflate-gunzip` has to do this, and it would run out of memory as it would
+  keep trying to allocate a larger output buffer.
+
+* Fixed a build error on Solaris.
+
+* Cleaned up a few things in the compression code.
+
+## Version 1.9
+
+* Made many improvements to the compression algorithms, and rebalanced the
+  compression levels:
+
+  * Heuristics were implemented which significantly improve the compression
+    ratio on data where short matches aren't useful, such as DNA sequencing
+    data.  This applies to all compression levels, but primarily to levels 1-9.
+
+  * Level 1 was made much faster, though it often compresses slightly worse than
+    before (but still better than zlib).
+
+  * Levels 8-9 were also made faster, though they often compress slightly worse
+    than before (but still better than zlib).  On some data, levels 8-9 are much
+    faster and compress much better than before; this change addressed an issue
+    where levels 8-9 did poorly on certain files.  The algorithm used by levels
+    8-9 is now more similar to that of levels 6-7 than to that of levels 10-12.
+
+  * Levels 2-3, 7, and 10-12 were strengthened slightly.
+
+  * Levels 4-6 were also strengthened slightly, but some of this improvement was
+    traded off to speed them up slightly as well.
+
+  * Levels 1-9 had their per-compressor memory usage greatly reduced.
+
+  As always, compression ratios will vary depending on the input data, and
+  compression speeds will vary depending on the input data and target platform.
+
+* `make install` will now install a pkg-config file for libdeflate.
+
+* The Makefile now supports the `DISABLE_SHARED` parameter to disable building
+  the shared library.
+
+* Improved the Android build support in the Makefile.
+
+## Version 1.8
+
+* Added `-t` (test) option to `libdeflate-gunzip`.
+
+* Unaligned access optimizations are now enabled on WebAssembly builds.
+
+* Fixed a build error when building with the Intel C Compiler (ICC).
+
+* Fixed a build error when building with uClibc.
+
+* libdeflate's CI system has switched from Travis CI to GitHub Actions.
+
+* Made some improvements to test scripts.
+
+## Version 1.7
+
+* Added support for compression level 0, "no compression".
+
+* Added an ARM CRC32 instruction accelerated implementation of CRC32.
+
+* Added support for linking the programs to the shared library version of
+  libdeflate rather than to the static library version.
+
+* Made the compression level affect the minimum input size at which compression
+  is attempted.
+
+* Fixed undefined behavior in x86 Adler32 implementation.  (No miscompilations
+  were observed in practice.)
+
+* Fixed undefined behavior in x86 CPU feature code.  (No miscompilations were
+  observed in practice.)
+
+* Fixed installing shared lib symlink on macOS.
+
+* Documented third-party bindings.
+
+* Made a lot of improvements to the testing scripts and the CI configuration
+  file.
+
+* Lots of other small improvements and cleanups.
+
+## Version 1.6
+
+* Prevented gcc 10 from miscompiling libdeflate (workaround for
+  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+
+* Removed workaround for gcc 5 and earlier producing slow code on ARM32.  If
+  this affects you, please upgrade your compiler.
+
+* New API function: `libdeflate_zlib_decompress_ex()`.  It provides the actual
+  size of the stream that was decompressed, like the gzip and DEFLATE
+  equivalents.
+
+* `libdeflate_zlib_decompress()` now accepts trailing bytes after the end of the
+  stream, like the gzip and DEFLATE equivalents.
+
+* Added support for custom memory allocators.  (New API function:
+  `libdeflate_set_memory_allocator()`)
+
+* Added support for building the library in freestanding mode.
+
+* Building libdeflate no longer requires `CPPFLAGS=-Icommon`.
+
+## Version 1.5
+
+* Fixed up stdcall support on 32-bit Windows: the functions are now exported
+  using both suffixed and non-suffixed names, and fixed `libdeflate.h` to be
+  MSVC-compatible again.
+
+## Version 1.4
+
+* The 32-bit Windows build of libdeflate now uses the "stdcall" calling
+  convention instead of "cdecl".  If you're calling `libdeflate.dll` directly
+  from C or C++, you'll need to recompile your code.  If you're calling it from
+  another language, or calling it indirectly using `LoadLibrary()`, you'll need
+  to update your code to use the stdcall calling convention.
+
+* The Makefile now supports building libdeflate as a shared
+  library (`.dylib`) on macOS.
+
+* Fixed a bug where support for certain optimizations and optional features
+  (file access hints and more precise timestamps) was incorrectly omitted when
+  libdeflate was compiled with `-Werror`.
+
+* Added `make check` target to the Makefile.
+
+* Added CI configuration files.
+
+## Version 1.3
+
+* `make install` now supports customizing the directories into which binaries,
+  headers, and libraries are installed.
+
+* `make install` now installs into `/usr/local` by default.  To change it, use
+  e.g. `make install PREFIX=/usr`.
+
+* `make install` now works on more platforms.
+
+* The Makefile now supports overriding the optimization flags.
+
+* The compression functions now correctly handle an output data buffer >= 4 GiB
+  in size, and `gzip` and `gunzip` now correctly handle multi-gigabyte files (if
+  enough memory is available).
+
+## Version 1.2
+
+* Slight improvements to decompression speed.
+
+* Added an AVX-512BW implementation of Adler-32.
+
+* The Makefile now supports a user-specified installation `PREFIX`.
+
+* Fixed build error with some Visual Studio versions.
+
+## Version 1.1
+
+* Fixed crash in CRC-32 code when the prebuilt libdeflate for 32-bit Windows was
+  called by a program built with Visual Studio.
+
+* Improved the worst-case decompression speed of malicious data.
+
+* Fixed build error when compiling for an ARM processor without hardware
+  floating point support.
+
+* Improved performance on the PowerPC64 architecture.
+
+* Added soname to `libdeflate.so`, to make packaging easier.
+
+* Added `make install` target to the Makefile.
+
+* The Makefile now supports user-specified `CPPFLAGS`.
+
+* The Windows binary releases now include the import library for
+  `libdeflate.dll`.  `libdeflate.lib` is now the import library, and
+  `libdeflatestatic.lib` is the static library.
+
+## Version 1.0
+
+* Added support for multi-member gzip files.
+
+* Moved architecture-specific code into subdirectories.  If you aren't using the
+  provided Makefile to build libdeflate, you now need to compile `lib/*.c` and
+  `lib/*/*.c` instead of just `lib/*.c`.
+
+* Added an ARM PMULL implementation of CRC-32, which speeds up gzip compression
+  and decompression on 32-bit and 64-bit ARM processors that have the
+  Cryptography Extensions.
+
+* Improved detection of CPU features, resulting in accelerated functions being
+  used in more cases.  This includes:
+
+  * Detect CPU features on 32-bit x86, not just 64-bit as was done previously.
+
+  * Detect CPU features on ARM, both 32 and 64-bit.  (Limited to Linux only
+    currently.)
+
+## Version 0.8
+
+* Build fixes for certain platforms and compilers.
+
+* libdeflate now produces the same output on all CPU architectures.
+
+* Improved documentation for building libdeflate on Windows.
+
+## Version 0.7
+
+* Fixed a very rare bug that caused data to be compressed incorrectly.  The bug
+  affected compression levels 7 and below since libdeflate v0.2.  Although there
+  have been no user reports of the bug, and I believe it would have been highly
+  unlikely to encounter on realistic data, it could occur on data specially
+  crafted to reproduce it.
+
+* Fixed a compilation error when building with clang 3.7.
+
+## Version 0.6
+
+* Various improvements to the gzip program's behavior.
+
+* Faster CRC-32 on AVX-capable processors.
+
+* Other minor changes.
+
+## Version 0.5
+
+* The CRC-32 checksum algorithm has been optimized with carryless multiplication
+  instructions for `x86_64` (PCLMUL).  This speeds up gzip compression and
+  decompression.
+
+* Build fixes for certain platforms and compilers.
+
+* Added more test programs and scripts.
+
+* libdeflate is now entirely MIT-licensed.
+
+## Version 0.4
+
+* The Adler-32 checksum algorithm has been optimized with vector instructions
+  for `x86_64` (SSE2 and AVX2) and ARM (NEON).  This speeds up zlib compression
+  and decompression.
+
+* To avoid naming collisions, functions and definitions in libdeflate's API have
+  been renamed to be prefixed with `libdeflate_` or `LIBDEFLATE_`.  Programs
+  using the old API will need to be updated.
+
+* Various bug fixes and other improvements.
+
+## Version 0.3
+
+* Some bug fixes and other minor changes.
+
+## Version 0.2
+
+* Implemented a new block splitting algorithm which typically improves the
+  compression ratio slightly at all compression levels.
+
+* The compressor now outputs each block using the cheapest type (dynamic
+  Huffman, static Huffman, or uncompressed).
+
+* The gzip program has received an overhaul and now behaves more like the
+  standard version.
+
+* Build system updates, including: some build options were changed and some
+  build options were removed, and the default 'make' target now includes the
+  gzip program as well as the library.
+
+## Version 0.1
+
+* Initial official release.

README.md

@@ -18,38 +18,149 @@ libdeflate itself is a library, but the following command-line programs which
 use this library are also provided:
 
 * gzip (or gunzip), a program which mostly behaves like the standard equivalent,
-  except that it does not yet support reading from standard input and does not
+  except that it does not yet have good streaming support and therefore does not
   yet support very large files
 * benchmark, a program for benchmarking in-memory compression and decompression
 
+For the release notes, see the [NEWS file](NEWS.md).
+
+## Table of Contents
+
+- [Building](#building)
+  - [For UNIX](#for-unix)
+  - [For macOS](#for-macos)
+  - [For Windows](#for-windows)
+    - [Using Cygwin](#using-cygwin)
+    - [Using MSYS2](#using-msys2)
+- [API](#api)
+- [Bindings for other programming languages](#bindings-for-other-programming-languages)
+- [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip)
+- [Compression levels](#compression-levels)
+- [Motivation](#motivation)
+- [License](#license)
+
+
 # Building
 
 ## For UNIX
 
-Just run `make`.  You need GNU Make and either GCC or Clang.  There is no `make
-install` yet; just copy the file(s) to where you want.
+Just run `make`, then (if desired) `make install`.  You need GNU Make and either
+GCC or Clang.  GCC is recommended because it builds slightly faster binaries.
 
-By default, all targets are built, including the library and programs, with the
-exception of the `benchmark` program.  `make help` shows the available targets.
-There are also several options which can be set on the `make` command line.  See
-the Makefile for details.
+By default, the following targets are built: the static library `libdeflate.a`,
+the shared library `libdeflate.so`, the `gzip` program, and the `gunzip` program
+(which is actually just a hard link to `gzip`).  Benchmarking and test programs
+such as `benchmark` are not built by default.  You can run `make help` to
+display the available build targets.
+
+There are also many options which can be set on the `make` command line, e.g. to
+omit library features or to customize the directories into which `make install`
+installs files.  See the Makefile for details.
+
+## For macOS
+
+Prebuilt macOS binaries can be installed with [Homebrew](https://brew.sh):
+
+    brew install libdeflate
+
+But if you need to build the binaries yourself, see the section for UNIX above.
 
 ## For Windows
 
-MinGW (GCC) is the recommended compiler to use when building binaries for
-Windows.  MinGW can be used on either Windows or Linux.  Use a command like:
+Prebuilt Windows binaries can be downloaded from
+https://github.com/ebiggers/libdeflate/releases.  But if you need to build the
+binaries yourself, MinGW (gcc) is the recommended compiler to use.  If you're
+performing the build *on* Windows (as opposed to cross-compiling for Windows on
+Linux, for example), you'll need to follow the directions in **one** of the two
+sections below to set up a minimal UNIX-compatible environment using either
+Cygwin or MSYS2, then do the build.  (Other MinGW distributions may not work, as
+they often omit basic UNIX tools such as `sh`.)
 
-    $ make CC=x86_64-w64-mingw32-gcc
+Alternatively, libdeflate may be built using the Visual Studio toolchain by
+running `nmake /f Makefile.msc`.  However, while this is supported in the sense
+that it will produce working binaries, it is not recommended because the
+binaries built with MinGW will be significantly faster.
 
-Windows binaries prebuilt with MinGW may also be downloaded from
-https://github.com/ebiggers/libdeflate/releases.
-
-Alternatively, a separate Makefile, `Makefile.msc`, is provided for the tools
-that come with Visual Studio, for those who strongly prefer that toolchain.
-
-As usual, 64-bit binaries are faster than 32-bit binaries and should be
+Also note that 64-bit binaries are faster than 32-bit binaries and should be
 preferred whenever possible.
 
+### Using Cygwin
+
+Run the Cygwin installer, available from https://cygwin.com/setup-x86_64.exe.
+When you get to the package selection screen, choose the following additional
+packages from category "Devel":
+
+- git
+- make
+- mingw64-i686-binutils
+- mingw64-i686-gcc-g++
+- mingw64-x86_64-binutils
+- mingw64-x86_64-gcc-g++
+
+(You may skip the mingw64-i686 packages if you don't need to build 32-bit
+binaries.)
+
+After the installation finishes, open a Cygwin terminal.  Then download
+libdeflate's source code (if you haven't already) and `cd` into its directory:
+
+    git clone https://github.com/ebiggers/libdeflate
+    cd libdeflate
+
+(Note that it's not required to use `git`; an alternative is to extract a .zip
+or .tar.gz archive of the source code downloaded from the releases page.
+Also, in case you need to find it in the file browser, note that your home
+directory in Cygwin is usually located at `C:\cygwin64\home\<your username>`.)
+
+Then, to build 64-bit binaries:
+
+    make CC=x86_64-w64-mingw32-gcc
+
+or to build 32-bit binaries:
+
+    make CC=i686-w64-mingw32-gcc
+
+### Using MSYS2
+
+Run the MSYS2 installer, available from http://www.msys2.org/.  After
+installing, open an MSYS2 shell and run:
+
+    pacman -Syu
+
+Say `y`, then when it's finished, close the shell window and open a new one.
+Then run the same command again:
+
+    pacman -Syu
+
+Then, install the packages needed to build libdeflate:
+
+    pacman -S git \
+              make \
+              mingw-w64-i686-binutils \
+              mingw-w64-i686-gcc \
+              mingw-w64-x86_64-binutils \
+              mingw-w64-x86_64-gcc
+
+(You may skip the mingw-w64-i686 packages if you don't need to build 32-bit
+binaries.)
+
+Then download libdeflate's source code (if you haven't already):
+
+    git clone https://github.com/ebiggers/libdeflate
+
+(Note that it's not required to use `git`; an alternative is to extract a .zip
+or .tar.gz archive of the source code downloaded from the releases page.
+Also, in case you need to find it in the file browser, note that your home
+directory in MSYS2 is usually located at `C:\msys64\home\<your username>`.)
+
+Then, to build 64-bit binaries, open "MSYS2 MinGW 64-bit" from the Start menu
+and run the following commands:
+
+    cd libdeflate
+    make clean
+    make
+
+Or to build 32-bit binaries, do the same but use "MSYS2 MinGW 32-bit" instead.
+
 # API
 
 libdeflate has a simple API that is not zlib-compatible.  You can create
@@ -71,6 +182,30 @@ This enables you to allocate an output buffer of the correct size without
 guessing.  However, libdeflate's decompression routines do optionally provide
 the actual number of output bytes in case you need it.
 
+Windows developers: note that the calling convention of libdeflate.dll is
+"stdcall" -- the same as the Win32 API.  If you call into libdeflate.dll using a
+non-C/C++ language, or dynamically using LoadLibrary(), make sure to use the
+stdcall convention.  Using the wrong convention may crash your application.
+(Note: older versions of libdeflate used the "cdecl" convention instead.)
+
+# Bindings for other programming languages
+
+The libdeflate project itself only provides a C library.  If you need to use
+libdeflate from a programming language other than C or C++, consider using the
+following bindings:
+
+* C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET)
+* Go: [go-libdeflate](https://github.com/4kills/go-libdeflate)
+* Java: [libdeflate-java](https://github.com/astei/libdeflate-java)
+* Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl)
+* Python: [deflate](https://github.com/dcwatson/deflate)
+* Ruby: [libdeflate-ruby](https://github.com/kaorimatz/libdeflate-ruby)
+* Rust: [libdeflater](https://github.com/adamkewley/libdeflater)
+
+Note: these are third-party projects which haven't necessarily been vetted by
+the authors of libdeflate.  Please direct all questions, bugs, and improvements
+for these bindings to their authors.
+
 # DEFLATE vs. zlib vs. gzip
 
 The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib
@@ -143,16 +278,8 @@ processors.  libdeflate addresses this by providing an optimized DEFLATE
 implementation which can be used for benchmarking purposes.  And, of course,
 real applications can use it as well.
 
-That being said, I have also started [a separate
-project](https://github.com/ebiggers/xpack) for an experimental, more modern
-compression format.
-
 # License
 
-The library portion of libdeflate has been released into the public domain.
-There is NO WARRANTY, to the extent permitted by law.  See the CC0 Public Domain
-Dedication in the COPYING.LIB file for details.
+libdeflate is [MIT-licensed](COPYING).
 
-The non-library portions of the project, such as the gzip program, may be
-modified and/or redistributed under the terms of the MIT license.  There is NO
-WARRANTY, to the extent permitted by law.  See the COPYING file for details.
+I am not aware of any patents or patent applications relevant to libdeflate.

common/common_defs.h

@@ -47,9 +47,7 @@
 #endif
 
 /* Fixed-width integer types */
-#ifndef PRIu32
-#  include <inttypes.h>
-#endif
+#include <stdint.h>
 typedef uint8_t u8;
 typedef uint16_t u16;
 typedef uint32_t u32;
@@ -76,7 +74,7 @@ typedef size_t machine_word_t;
 /*                         Optional compiler features                         */
 /* ========================================================================== */
 
-/* LIBEXPORT - annotate a function that is part of the library API */
+/* LIBEXPORT - export a function from a shared library */
 #ifndef LIBEXPORT
 #  define LIBEXPORT
 #endif
@@ -121,10 +119,42 @@ typedef size_t machine_word_t;
 #  define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
 #endif
 
-/* Does the compiler support __attribute__((target("bmi2")))? */
+/* Which targets are supported with the 'target' function attribute? */
 #ifndef COMPILER_SUPPORTS_BMI2_TARGET
 #  define COMPILER_SUPPORTS_BMI2_TARGET 0
 #endif
+#ifndef COMPILER_SUPPORTS_AVX_TARGET
+#  define COMPILER_SUPPORTS_AVX_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET 0
+#endif
+
+/*
+ * Which targets are supported with the 'target' function attribute and have
+ * intrinsics that work within 'target'-ed functions?
+ */
+#ifndef COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0
+#endif
 
 /* _aligned_attribute(n) - declare that the annotated variable, or variables of
  * the annotated type, are to be aligned on n-byte boundaries */
@@ -139,12 +169,8 @@ typedef size_t machine_word_t;
 #define MIN(a, b)		((a) <= (b) ? (a) : (b))
 #define MAX(a, b)		((a) >= (b) ? (a) : (b))
 #define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
-
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
-#  define STATIC_ASSERT(expr)	_Static_assert((expr), "")
-#else
-#  define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
-#endif
+#define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
 
 /* ========================================================================== */
 /*                           Endianness handling                              */
@@ -221,39 +247,6 @@ static forceinline u64 bswap64(u64 n)
 #  define UNALIGNED_ACCESS_IS_FAST 0
 #endif
 
-/*
- * DEFINE_UNALIGNED_TYPE(type) - a macro that, given an integer type 'type',
- * defines load_type_unaligned(addr) and store_type_unaligned(v, addr) functions
- * which load and store variables of type 'type' from/to unaligned memory
- * addresses.  If not defined, a fallback is used.
- */
-#ifndef DEFINE_UNALIGNED_TYPE
-
-/*
- * Although memcpy() may seem inefficient, it *usually* gets optimized
- * appropriately by modern compilers.  It's portable and may be the best we can
- * do for a fallback...
- */
-#include <string.h>
-
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	type v;							\
-	memcpy(&v, p, sizeof(v));				\
-	return v;						\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	memcpy(p, &v, sizeof(v));				\
-}
-
-#endif /* !DEFINE_UNALIGNED_TYPE */
-
 /* ========================================================================== */
 /*                             Bit scan functions                             */
 /* ========================================================================== */

common/compiler_gcc.h

@@ -1,12 +1,33 @@
 /*
- * compiler_gcc.h - definitions for the GNU C Compiler.  Currently this also
- * handles clang and the Intel C Compiler.
+ * compiler_gcc.h - definitions for the GNU C Compiler.  This also handles clang
+ * and the Intel C Compiler (icc).
+ *
+ * TODO: icc is not well tested, so some things are currently disabled even
+ * though they maybe can be enabled on some icc versions.
  */
 
-#define GCC_PREREQ(major, minor)					\
-	(!defined(__clang__) && !defined(__INTEL_COMPILER) &&		\
-	 (__GNUC__ > (major) ||						\
-	  (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
+#if !defined(__clang__) && !defined(__INTEL_COMPILER)
+#  define GCC_PREREQ(major, minor)		\
+	(__GNUC__ > (major) ||			\
+	 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#  define GCC_PREREQ(major, minor)	0
+#endif
+
+/* Note: only check the clang version when absolutely necessary!
+ * "Vendors" such as Apple can use different version numbers. */
+#ifdef __clang__
+#  ifdef __apple_build_version__
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__apple_build_version__ >= (apple_version))
+#  else
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__clang_major__ > (major) ||			\
+	 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+#  endif
+#else
+#  define CLANG_PREREQ(major, minor, apple_version)	0
+#endif
 
 #ifndef __has_attribute
 #  define __has_attribute(attribute)	0
@@ -33,12 +54,131 @@
 #define prefetchw(addr)		__builtin_prefetch((addr), 1)
 #define _aligned_attribute(n)	__attribute__((aligned(n)))
 
-#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE		\
+#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE	\
 	(GCC_PREREQ(4, 4) || __has_attribute(target))
 
-#define COMPILER_SUPPORTS_BMI2_TARGET				\
-	(COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE &&		\
-	 (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di)))
+#if COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+
+#  if defined(__i386__) || defined(__x86_64__)
+
+#    define COMPILER_SUPPORTS_PCLMUL_TARGET	\
+	(GCC_PREREQ(4, 4) || __has_builtin(__builtin_ia32_pclmulqdq128))
+
+#    define COMPILER_SUPPORTS_AVX_TARGET	\
+	(GCC_PREREQ(4, 6) || __has_builtin(__builtin_ia32_maxps256))
+
+#    define COMPILER_SUPPORTS_BMI2_TARGET	\
+	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di))
+
+#    define COMPILER_SUPPORTS_AVX2_TARGET	\
+	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256))
+
+#    define COMPILER_SUPPORTS_AVX512BW_TARGET	\
+	(GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512))
+
+	/*
+	 * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics
+	 * not available in the main target could not be used in 'target'
+	 * attribute functions.  Unfortunately clang has no feature test macro
+	 * for this so we have to check its version.
+	 */
+#    if GCC_PREREQ(4, 9) || CLANG_PREREQ(3, 8, 7030000)
+#      define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS	1
+#      define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_PCLMUL_TARGET
+#      define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_AVX2_TARGET
+#      define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_AVX512BW_TARGET
+#    endif
+
+#  elif defined(__arm__) || defined(__aarch64__)
+
+    /*
+     * Determine whether NEON and crypto intrinsics are supported.
+     *
+     * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither
+     * was available unless enabled in the main target.
+     *
+     * But even after that, to include <arm_neon.h> (which contains both the
+     * basic NEON intrinsics and the crypto intrinsics) the main target still
+     * needs to have:
+     *   - gcc: hardware floating point support
+     *   - clang: NEON support (but not necessarily crypto support)
+     */
+#    if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \
+        (defined(__clang__) && defined(__ARM_NEON))
+#      define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1
+       /*
+        * The crypto intrinsics are broken on arm32 with clang, even when using
+        * -mfpu=crypto-neon-fp-armv8, because clang's <arm_neon.h> puts them
+        * behind __aarch64__.  Undefine __ARM_FEATURE_CRYPTO in that case...
+        */
+#      if defined(__clang__) && defined(__arm__)
+#        undef __ARM_FEATURE_CRYPTO
+#      elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__)
+#        define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1
+#      endif
+#    endif
+
+     /*
+      * Determine whether ARM CRC32 intrinsics are supported.
+      *
+      * This support has been affected by several gcc bugs, which we must avoid
+      * by only allowing gcc versions that have the corresponding fixes.  First,
+      * gcc commit 943766d37ae4 ("[arm] Fix use of CRC32 intrinsics with Armv8-a
+      * and hard-float"), i.e. gcc 8.4+, 9.3+, 10.1+, or 11+, is needed.
+      * Second, gcc commit c1cdabe3aab8 ("arm: reorder assembler architecture
+      * directives [PR101723]"), i.e. gcc 9.5+, 10.4+, 11.3+, or 12+, is needed
+      * when binutils is 2.34 or later, due to
+      * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104439.  We use the second
+      * set of prerequisites, as they are stricter and we have no way to detect
+      * the binutils version in C source without requiring a configure script.
+      *
+      * Yet another gcc bug makes arm_acle.h sometimes not define the crc
+      * functions even when the corresponding builtins are available.  However,
+      * we work around this later when including arm_acle.h.
+      *
+      * Things are a bit easier with clang -- we can just check whether the
+      * crc builtins are available.  However, clang's arm_acle.h is broken in
+      * the same way as gcc's, which we work around later in the same way.
+      */
+#    if GCC_PREREQ(11, 3) || \
+        (GCC_PREREQ(10, 4) && !GCC_PREREQ(11, 0)) || \
+        (GCC_PREREQ(9, 5) && !GCC_PREREQ(10, 0)) || \
+        (defined(__clang__) && __has_builtin(__builtin_arm_crc32b))
+#      define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1
+#    endif
+
+#  endif /* __arm__ || __aarch64__ */
+
+#endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */
+
+/*
+ * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed
+ * integers (e.g. __v4si), not vectors of unsigned integers (e.g.  __v4su).  But
+ * we need the unsigned ones in order to avoid signed integer overflow, which is
+ * undefined behavior.  Add the missing definitions for the unsigned ones if
+ * needed.
+ */
+#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \
+    (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) || \
+    defined(__INTEL_COMPILER)
+typedef unsigned long long  __v2du __attribute__((__vector_size__(16)));
+typedef unsigned int        __v4su __attribute__((__vector_size__(16)));
+typedef unsigned short      __v8hu __attribute__((__vector_size__(16)));
+typedef unsigned char      __v16qu __attribute__((__vector_size__(16)));
+typedef unsigned long long  __v4du __attribute__((__vector_size__(32)));
+typedef unsigned int        __v8su __attribute__((__vector_size__(32)));
+typedef unsigned short     __v16hu __attribute__((__vector_size__(32)));
+typedef unsigned char      __v32qu __attribute__((__vector_size__(32)));
+#endif
+
+#ifdef __INTEL_COMPILER
+typedef int   __v16si __attribute__((__vector_size__(64)));
+typedef short __v32hi __attribute__((__vector_size__(64)));
+typedef char  __v64qi __attribute__((__vector_size__(64)));
+#endif
 
 /* Newer gcc supports __BYTE_ORDER__.  Older gcc doesn't. */
 #ifdef __BYTE_ORDER__
@@ -57,30 +197,39 @@
 #  define bswap64	__builtin_bswap64
 #endif
 
-#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED)
+#if defined(__x86_64__) || defined(__i386__) || \
+    defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
+    /*
+     * For all compilation purposes, WebAssembly behaves like any other CPU
+     * instruction set. Even though WebAssembly engine might be running on top
+     * of different actual CPU architectures, the WebAssembly spec itself
+     * permits unaligned access and it will be fast on most of those platforms,
+     * and simulated at the engine level on others, so it's worth treating it
+     * as a CPU architecture with fast unaligned access.
+    */ defined(__wasm__)
 #  define UNALIGNED_ACCESS_IS_FAST 1
 #endif
 
-/* With gcc, we can access unaligned memory through 'packed' structures. */
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-struct type##unaligned {					\
-	type v;							\
-} __attribute__((packed));					\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	return ((const struct type##unaligned *)p)->v;		\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	((struct type##unaligned *)p)->v = v;			\
-}
-
 #define bsr32(n)	(31 - __builtin_clz(n))
 #define bsr64(n)	(63 - __builtin_clzll(n))
 #define bsf32(n)	__builtin_ctz(n)
 #define bsf64(n)	__builtin_ctzll(n)
+
+#if defined(__arm__) && \
+	(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
+static forceinline unsigned int
+rbit32(unsigned int v)
+{
+	__asm__("rbit %0, %1\n" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#elif defined(__aarch64__)
+static forceinline unsigned int
+rbit32(unsigned int v)
+{
+	__asm__("rbit %w0, %w1\n" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#endif /* __aarch64__ */

common/compiler_msc.h

@@ -2,7 +2,8 @@
  * compiler_msc.h - definitions for the Microsoft C Compiler
  */
 
-#define BUILDING_LIBDEFLATE
+#include <stdint.h>
+#include <stdlib.h> /* for _byteswap_*() */
 
 #define LIBEXPORT	__declspec(dllexport)
 
@@ -24,24 +25,6 @@ typedef long long ssize_t;
 typedef int ssize_t;
 #endif
 
-/*
- * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
- * inttypes.h.  Work around this by defining the PRI* macros ourselves.
- */
-#include <stdint.h>
-#define PRIu8  "hhu"
-#define PRIu16 "hu"
-#define PRIu32 "u"
-#define PRIu64 "llu"
-#define PRIi8  "hhi"
-#define PRIi16 "hi"
-#define PRIi32 "i"
-#define PRIi64 "lli"
-#define PRIx8  "hhx"
-#define PRIx16 "hx"
-#define PRIx32 "x"
-#define PRIx64 "llx"
-
 /* Assume a little endian architecture with fast unaligned access */
 #define CPU_IS_LITTLE_ENDIAN()		1
 #define UNALIGNED_ACCESS_IS_FAST	1

lib/adler32.c

@@ -1,26 +1,38 @@
 /*
  * adler32.c - Adler-32 checksum algorithm
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
+#include "lib_common.h"
+#include "libdeflate.h"
 
-/*
- * The Adler-32 divisor, or "base", value.
- */
+/* The Adler-32 divisor, or "base", value. */
 #define DIVISOR 65521
 
 /*
- * MAX_BYTES_PER_CHUNK is the most bytes that can be processed without the
+ * MAX_CHUNK_SIZE is the most bytes that can be processed without the
  * possibility of s2 overflowing when it is represented as an unsigned 32-bit
  * integer.  This value was computed using the following Python script:
  *
@@ -40,58 +52,43 @@
  * has value 0xFF and that s1 and s2 started with the highest possible values
  * modulo the divisor.
  */
-#define MAX_BYTES_PER_CHUNK	5552
+#define MAX_CHUNK_SIZE	5552
 
-/* Number of bytes to process per loop iteration  */
-#define UNROLL_FACTOR	4
+typedef u32 (*adler32_func_t)(u32, const u8 *, size_t);
 
-u32
-adler32(const void *buffer, size_t size)
+/* Include architecture-specific implementations if available */
+#undef DEFAULT_IMPL
+#undef DISPATCH
+#if defined(__arm__) || defined(__aarch64__)
+#  include "arm/adler32_impl.h"
+#elif defined(__i386__) || defined(__x86_64__)
+#  include "x86/adler32_impl.h"
+#endif
+
+/* Define a generic implementation if needed */
+#ifndef DEFAULT_IMPL
+#define DEFAULT_IMPL adler32_generic
+static u32 adler32_generic(u32 adler, const u8 *p, size_t size)
 {
-	u32 s1 = 1;
-	u32 s2 = 0;
-	const u8 *p = buffer;
-	const u8 *end = p + size;
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+	const u8 * const end = p + size;
+
 	while (p != end) {
-		size_t chunk_size = MIN(end - p, MAX_BYTES_PER_CHUNK);
+		size_t chunk_size = MIN(end - p, MAX_CHUNK_SIZE);
 		const u8 *chunk_end = p + chunk_size;
+		size_t num_unrolled_iterations = chunk_size / 4;
 
-	#if UNROLL_FACTOR > 1
-		size_t num_unrolled_iterations = chunk_size / UNROLL_FACTOR;
 		while (num_unrolled_iterations--) {
 			s1 += *p++;
 			s2 += s1;
-		#if UNROLL_FACTOR >= 2
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 3
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 4
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 5
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 6
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 7
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 8
-			s1 += *p++;
-			s2 += s1;
-		#endif
-			STATIC_ASSERT(UNROLL_FACTOR <= 8);
 		}
-	#endif /* UNROLL_FACTOR > 1 */
 		while (p != chunk_end) {
 			s1 += *p++;
 			s2 += s1;
@@ -99,5 +96,35 @@ adler32(const void *buffer, size_t size)
 		s1 %= DIVISOR;
 		s2 %= DIVISOR;
 	}
+
 	return (s2 << 16) | s1;
 }
+#endif /* !DEFAULT_IMPL */
+
+#ifdef DISPATCH
+static u32 dispatch(u32, const u8 *, size_t);
+
+static volatile adler32_func_t adler32_impl = dispatch;
+
+/* Choose the fastest implementation at runtime */
+static u32 dispatch(u32 adler, const u8 *buffer, size_t size)
+{
+	adler32_func_t f = arch_select_adler32_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
+	adler32_impl = f;
+	return adler32_impl(adler, buffer, size);
+}
+#else
+#  define adler32_impl DEFAULT_IMPL /* only one implementation, use it */
+#endif
+
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
+libdeflate_adler32(u32 adler, const void *buffer, size_t size)
+{
+	if (buffer == NULL) /* return initial value */
+		return 1;
+	return adler32_impl(adler, buffer, size);
+}

lib/adler32.h

@@ -1,12 +0,0 @@
-/*
- * adler32.h - Adler-32 checksum algorithm
- */
-
-#ifndef _LIB_ADLER32_H
-#define _LIB_ADLER32_H
-
-#include "common_defs.h"
-
-extern u32 adler32(const void *buffer, size_t size);
-
-#endif /* _LIB_ADLER32_H */

lib/adler32_vec_template.h

@@ -0,0 +1,124 @@
+/*
+ * adler32_vec_template.h - template for vectorized Adler-32 implementations
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file contains a template for vectorized Adler-32 implementations.
+ *
+ * The inner loop between reductions modulo 65521 of an unvectorized Adler-32
+ * implementation looks something like this:
+ *
+ *	do {
+ *		s1 += *p;
+ *		s2 += s1;
+ *	} while (++p != chunk_end);
+ *
+ * For vectorized calculation of s1, we only need to sum the input bytes.  They
+ * can be accumulated into multiple counters which are eventually summed
+ * together.
+ *
+ * For vectorized calculation of s2, the basic idea is that for each iteration
+ * that processes N bytes, we can perform the following vectorizable
+ * calculation:
+ *
+ *	s2 += N*byte_1 + (N-1)*byte_2 + (N-2)*byte_3 + ... + 1*byte_N
+ *
+ * Or, equivalently, we can sum the byte_1...byte_N for each iteration into N
+ * separate counters, then do the multiplications by N...1 just once at the end
+ * rather than once per iteration.
+ *
+ * Also, we must account for how previous bytes will affect s2 by doing the
+ * following at beginning of each iteration:
+ *
+ *	s2 += s1 * N
+ *
+ * Furthermore, like s1, "s2" can actually be multiple counters which are
+ * eventually summed together.
+ */
+
+static u32 ATTRIBUTES
+FUNCNAME(u32 adler, const u8 *p, size_t size)
+{
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+	const u8 * const end = p + size;
+	const u8 *vend;
+	const size_t max_chunk_size =
+		MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) -
+		(MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) %
+		 IMPL_SEGMENT_SIZE);
+
+	/* Process a byte at a time until the needed alignment is reached */
+	if (p != end && (uintptr_t)p % IMPL_ALIGNMENT) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+		} while (p != end && (uintptr_t)p % IMPL_ALIGNMENT);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	/*
+	 * Process "chunks" of bytes using vector instructions.  Chunk sizes are
+	 * limited to MAX_CHUNK_SIZE, which guarantees that s1 and s2 never
+	 * overflow before being reduced modulo DIVISOR.  For vector processing,
+	 * chunk sizes are also made evenly divisible by IMPL_SEGMENT_SIZE and
+	 * may be further limited to IMPL_MAX_CHUNK_SIZE.
+	 */
+	STATIC_ASSERT(IMPL_SEGMENT_SIZE % IMPL_ALIGNMENT == 0);
+	vend = end - ((size_t)(end - p) % IMPL_SEGMENT_SIZE);
+	while (p != vend) {
+		size_t chunk_size = MIN((size_t)(vend - p), max_chunk_size);
+
+		s2 += s1 * chunk_size;
+
+		FUNCNAME_CHUNK((const void *)p, (const void *)(p + chunk_size),
+			       &s1, &s2);
+
+		p += chunk_size;
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	/* Process any remaining bytes */
+	if (p != end) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+		} while (p != end);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	return (s2 << 16) | s1;
+}
+
+#undef FUNCNAME
+#undef FUNCNAME_CHUNK
+#undef ATTRIBUTES
+#undef IMPL_ALIGNMENT
+#undef IMPL_SEGMENT_SIZE
+#undef IMPL_MAX_CHUNK_SIZE

lib/aligned_malloc.c

@@ -1,46 +0,0 @@
-/*
- * aligned_malloc.c - aligned memory allocation
- *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
- *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
- *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
- */
-
-/*
- * This file provides portable aligned memory allocation functions that only
- * use malloc() and free().  This avoids portability problems with
- * posix_memalign(), aligned_alloc(), etc.
- */
-
-#include <stdlib.h>
-
-#include "aligned_malloc.h"
-
-void *
-aligned_malloc(size_t alignment, size_t size)
-{
-	const uintptr_t mask = alignment - 1;
-	char *ptr = NULL;
-	char *raw_ptr;
-
-	raw_ptr = malloc(mask + sizeof(size_t) + size);
-	if (raw_ptr) {
-		ptr = (char *)raw_ptr + sizeof(size_t);
-		ptr = (void *)(((uintptr_t)ptr + mask) & ~mask);
-		*((size_t *)ptr - 1) = ptr - raw_ptr;
-	}
-	return ptr;
-}
-
-void
-aligned_free(void *ptr)
-{
-	if (ptr)
-		free((char *)ptr - *((size_t *)ptr - 1));
-}

lib/aligned_malloc.h

@@ -1,13 +0,0 @@
-/*
- * aligned_malloc.c - aligned memory allocation
- */
-
-#ifndef _LIB_ALIGNED_MALLOC_H
-#define _LIB_ALIGNED_MALLOC_H
-
-#include "common_defs.h"
-
-extern void *aligned_malloc(size_t alignment, size_t size);
-extern void aligned_free(void *ptr);
-
-#endif /* _LIB_ALIGNED_MALLOC_H */

lib/arm/adler32_impl.h

@@ -0,0 +1,125 @@
+/*
+ * arm/adler32_impl.h - ARM implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_ADLER32_IMPL_H
+#define LIB_ARM_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/* NEON implementation */
+#undef DISPATCH_NEON
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__ARM_NEON) || (ARM_CPU_FEATURES_ENABLED &&	\
+				 COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_neon
+#  define FUNCNAME_CHUNK	adler32_neon_chunk
+#  define IMPL_ALIGNMENT	16
+#  define IMPL_SEGMENT_SIZE	32
+/* Prevent unsigned overflow of the 16-bit precision byte counters */
+#  define IMPL_MAX_CHUNK_SIZE	(32 * (0xFFFF / 0xFF))
+#  ifdef __ARM_NEON
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_neon
+#  else
+#    ifdef __arm__
+#      define ATTRIBUTES	__attribute__((target("fpu=neon")))
+#    else
+#      define ATTRIBUTES	__attribute__((target("+simd")))
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_NEON	1
+#  endif
+#  include <arm_neon.h>
+static forceinline ATTRIBUTES void
+adler32_neon_chunk(const uint8x16_t *p, const uint8x16_t * const end,
+		   u32 *s1, u32 *s2)
+{
+	uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 };
+	uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_a = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_b = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_c = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_d = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+	do {
+		const uint8x16_t bytes1 = *p++;
+		const uint8x16_t bytes2 = *p++;
+		uint16x8_t tmp;
+
+		v_s2 += v_s1;
+
+		/* Vector Pairwise Add Long (u8 => u16) */
+		tmp = vpaddlq_u8(bytes1);
+
+		/* Vector Pairwise Add and Accumulate Long (u8 => u16) */
+		tmp = vpadalq_u8(tmp, bytes2);
+
+		/* Vector Pairwise Add and Accumulate Long (u16 => u32) */
+		v_s1 = vpadalq_u16(v_s1, tmp);
+
+		/* Vector Add Wide (u8 => u16) */
+		v_byte_sums_a = vaddw_u8(v_byte_sums_a, vget_low_u8(bytes1));
+		v_byte_sums_b = vaddw_u8(v_byte_sums_b, vget_high_u8(bytes1));
+		v_byte_sums_c = vaddw_u8(v_byte_sums_c, vget_low_u8(bytes2));
+		v_byte_sums_d = vaddw_u8(v_byte_sums_d, vget_high_u8(bytes2));
+
+	} while (p != end);
+
+	/* Vector Shift Left (u32) */
+	v_s2 = vqshlq_n_u32(v_s2, 5);
+
+	/* Vector Multiply Accumulate Long (u16 => u32) */
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),  (uint16x4_t) { 32, 31, 30, 29 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_a), (uint16x4_t) { 28, 27, 26, 25 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),  (uint16x4_t) { 24, 23, 22, 21 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_b), (uint16x4_t) { 20, 19, 18, 17 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),  (uint16x4_t) { 16, 15, 14, 13 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_c), (uint16x4_t) { 12, 11, 10,  9 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_byte_sums_d), (uint16x4_t) {  8,  7,  6,  5 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_d), (uint16x4_t) {  4,  3,  2,  1 });
+
+	*s1 += v_s1[0] + v_s1[1] + v_s1[2] + v_s1[3];
+	*s2 += v_s2[0] + v_s2[1] + v_s2[2] + v_s2[3];
+}
+#  include "../adler32_vec_template.h"
+#endif /* NEON implementation */
+
+#ifdef DISPATCH
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_NEON
+	if (features & ARM_CPU_FEATURE_NEON)
+		return adler32_neon;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_ARM_ADLER32_IMPL_H */

lib/arm/cpu_features.c

@@ -0,0 +1,180 @@
+/*
+ * arm/cpu_features.c - feature detection for ARM processors
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * ARM CPUs don't have a standard way for unprivileged programs to detect CPU
+ * features.  But an OS-specific way can be used when available.
+ */
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#if ARM_CPU_FEATURES_ENABLED
+
+#ifdef __linux__
+/*
+ * On Linux, arm32 and arm64 CPU features can be detected by reading the
+ * AT_HWCAP and AT_HWCAP2 values from /proc/self/auxv.
+ *
+ * Ideally we'd use the C library function getauxval(), but it's not guaranteed
+ * to be available: it was only added to glibc in 2.16, and in Android it was
+ * added to API level 18 for arm32 and level 21 for arm64.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+
+#define AT_HWCAP	16
+#define AT_HWCAP2	26
+
+static void scan_auxv(unsigned long *hwcap, unsigned long *hwcap2)
+{
+	int fd;
+	unsigned long auxbuf[32];
+	int filled = 0;
+	int i;
+
+	fd = open("/proc/self/auxv", O_RDONLY);
+	if (fd < 0)
+		return;
+
+	for (;;) {
+		do {
+			int ret = read(fd, &((char *)auxbuf)[filled],
+				       sizeof(auxbuf) - filled);
+			if (ret <= 0) {
+				if (ret < 0 && errno == EINTR)
+					continue;
+				goto out;
+			}
+			filled += ret;
+		} while (filled < 2 * sizeof(long));
+
+		i = 0;
+		do {
+			unsigned long type = auxbuf[i];
+			unsigned long value = auxbuf[i + 1];
+
+			if (type == AT_HWCAP)
+				*hwcap = value;
+			else if (type == AT_HWCAP2)
+				*hwcap2 = value;
+			i += 2;
+			filled -= 2 * sizeof(long);
+		} while (filled >= 2 * sizeof(long));
+
+		memmove(auxbuf, &auxbuf[i], filled);
+	}
+out:
+	close(fd);
+}
+
+static u32 get_arm_cpu_features(void)
+{
+	u32 features = 0;
+	unsigned long hwcap = 0;
+	unsigned long hwcap2 = 0;
+
+	scan_auxv(&hwcap, &hwcap2);
+
+#ifdef __arm__
+	STATIC_ASSERT(sizeof(long) == 4);
+	if (hwcap & (1 << 12))	/* HWCAP_NEON */
+		features |= ARM_CPU_FEATURE_NEON;
+	if (hwcap2 & (1 << 1))	/* HWCAP2_PMULL */
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap2 & (1 << 4))	/* HWCAP2_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
+#else
+	STATIC_ASSERT(sizeof(long) == 8);
+	if (hwcap & (1 << 1))	/* HWCAP_ASIMD */
+		features |= ARM_CPU_FEATURE_NEON;
+	if (hwcap & (1 << 4))	/* HWCAP_PMULL */
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap & (1 << 7))	/* HWCAP_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
+#endif
+	return features;
+}
+
+#elif defined(__APPLE__)
+/* On Apple platforms, arm64 CPU features can be detected via sysctlbyname(). */
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+
+static const struct {
+	const char *name;
+	u32 feature;
+} feature_sysctls[] = {
+	{ "hw.optional.neon",		ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.AdvSIMD",	ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.arm.FEAT_PMULL",	ARM_CPU_FEATURE_PMULL },
+	{ "hw.optional.armv8_crc32",	ARM_CPU_FEATURE_CRC32 },
+};
+
+static u32 get_arm_cpu_features(void)
+{
+	u32 features = 0;
+	size_t i;
+
+	for (i = 0; i < ARRAY_LEN(feature_sysctls); i++) {
+		const char *name = feature_sysctls[i].name;
+		u32 val = 0;
+		size_t valsize = sizeof(val);
+
+		if (sysctlbyname(name, &val, &valsize, NULL, 0) == 0 &&
+		    valsize == sizeof(val) && val == 1)
+			features |= feature_sysctls[i].feature;
+	}
+	return features;
+}
+#else
+#error "unhandled case"
+#endif
+
+static const struct cpu_feature arm_cpu_feature_table[] = {
+	{ARM_CPU_FEATURE_NEON,		"neon"},
+	{ARM_CPU_FEATURE_PMULL,		"pmull"},
+	{ARM_CPU_FEATURE_CRC32,		"crc32"},
+};
+
+volatile u32 _cpu_features = 0;
+
+void setup_cpu_features(void)
+{
+	u32 features = get_arm_cpu_features();
+
+	disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
+					 ARRAY_LEN(arm_cpu_feature_table));
+
+	_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
+}
+
+#endif /* ARM_CPU_FEATURES_ENABLED */

lib/arm/cpu_features.h

@@ -0,0 +1,41 @@
+/*
+ * arm/cpu_features.h - feature detection for ARM processors
+ */
+
+#ifndef LIB_ARM_CPU_FEATURES_H
+#define LIB_ARM_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if (defined(__arm__) || defined(__aarch64__)) && \
+	(defined(__linux__) || \
+	 (defined(__aarch64__) && defined(__APPLE__))) && \
+	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE && \
+	!defined(FREESTANDING)
+#  define ARM_CPU_FEATURES_ENABLED 1
+#else
+#  define ARM_CPU_FEATURES_ENABLED 0
+#endif
+
+#if ARM_CPU_FEATURES_ENABLED
+
+#define ARM_CPU_FEATURE_NEON		0x00000001
+#define ARM_CPU_FEATURE_PMULL		0x00000002
+#define ARM_CPU_FEATURE_CRC32		0x00000004
+
+#define ARM_CPU_FEATURES_KNOWN		0x80000000
+
+extern volatile u32 _cpu_features;
+
+void setup_cpu_features(void);
+
+static inline u32 get_cpu_features(void)
+{
+	if (_cpu_features == 0)
+		setup_cpu_features();
+	return _cpu_features;
+}
+
+#endif /* ARM_CPU_FEATURES_ENABLED */
+
+#endif /* LIB_ARM_CPU_FEATURES_H */

lib/arm/crc32_impl.h

@@ -0,0 +1,251 @@
+/*
+ * arm/crc32_impl.h
+ *
+ * Copyright 2017 Jun He <jun.he@linaro.org>
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_CRC32_IMPL_H
+#define LIB_ARM_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/* Implementation using ARM CRC32 instructions */
+#undef DISPATCH_ARM
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRC32) || \
+     (ARM_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS))
+#  ifdef __ARM_FEATURE_CRC32
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_arm
+#  else
+#    ifdef __arm__
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("armv8-a,crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("arch=armv8-a+crc")))
+#      endif
+#    else
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("+crc")))
+#      endif
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_ARM	1
+#  endif
+
+/*
+ * gcc's (as of 10.1) version of arm_acle.h for arm32, and clang's (as of
+ * 10.0.1) version of arm_acle.h for both arm32 and arm64, have a bug where they
+ * only define the CRC32 functions like __crc32b() when __ARM_FEATURE_CRC32 is
+ * defined.  That prevents them from being used via __attribute__((target)) when
+ * the main target doesn't have CRC32 support enabled.  The actual built-ins
+ * like __builtin_arm_crc32b() are available and work, however; it's just the
+ * wrappers in arm_acle.h like __crc32b() that erroneously don't get defined.
+ * Work around this by manually defining __ARM_FEATURE_CRC32.
+ */
+#ifndef __ARM_FEATURE_CRC32
+#  define __ARM_FEATURE_CRC32	1
+#endif
+#include <arm_acle.h>
+
+static u32 ATTRIBUTES
+crc32_arm(u32 remainder, const u8 *p, size_t size)
+{
+	while (size != 0 && (uintptr_t)p & 7) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	while (size >= 64) {
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 0)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 1)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 2)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 3)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 4)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 5)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 6)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 7)));
+		p += 64;
+		size -= 64;
+	}
+
+	while (size >= 8) {
+		remainder = __crc32d(remainder, le64_bswap(*(u64 *)p));
+		p += 8;
+		size -= 8;
+	}
+
+	while (size != 0) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	return remainder;
+}
+#undef ATTRIBUTES
+#endif /* Implementation using ARM CRC32 instructions */
+
+/*
+ * CRC-32 folding with ARM Crypto extension-PMULL
+ *
+ * This works the same way as the x86 PCLMUL version.
+ * See x86/crc32_pclmul_template.h for an explanation.
+ */
+#undef DISPATCH_PMULL
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRYPTO) ||	\
+     (ARM_CPU_FEATURES_ENABLED &&	\
+      COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS)) && \
+      /* not yet tested on big endian, probably needs changes to work there */ \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#  define FUNCNAME		crc32_pmull
+#  define FUNCNAME_ALIGNED	crc32_pmull_aligned
+#  ifdef __ARM_FEATURE_CRYPTO
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_pmull
+#  else
+#    ifdef __arm__
+#      define ATTRIBUTES	__attribute__((target("fpu=crypto-neon-fp-armv8")))
+#    else
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("crypto")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("+crypto")))
+#      endif
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_PMULL	1
+#  endif
+
+#include <arm_neon.h>
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_00(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
+				     (poly64_t)vget_low_u8(b));
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_10(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
+				     (poly64_t)vget_high_u8(b));
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_11(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_high_p64((poly64x2_t)a, (poly64x2_t)b);
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+fold_128b(uint8x16_t dst, uint8x16_t src, uint8x16_t multipliers)
+{
+	return dst ^ clmul_00(src, multipliers) ^ clmul_11(src, multipliers);
+}
+
+static forceinline ATTRIBUTES u32
+crc32_pmull_aligned(u32 remainder, const uint8x16_t *p, size_t nr_segs)
+{
+	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
+	const uint8x16_t multipliers_4 =
+		(uint8x16_t)(uint64x2_t){ 0x8F352D95, 0x1D9513D7 };
+	const uint8x16_t multipliers_1 =
+		(uint8x16_t)(uint64x2_t){ 0xAE689191, 0xCCAA009E };
+	const uint8x16_t final_multiplier =
+		(uint8x16_t)(uint64x2_t){ 0xB8BC6765 };
+	const uint8x16_t mask32 = (uint8x16_t)(uint32x4_t){ 0xFFFFFFFF };
+	const uint8x16_t barrett_reduction_constants =
+			(uint8x16_t)(uint64x2_t){ 0x00000001F7011641,
+						  0x00000001DB710641 };
+	const uint8x16_t zeroes = (uint8x16_t){ 0 };
+
+	const uint8x16_t * const end = p + nr_segs;
+	const uint8x16_t * const end512 = p + (nr_segs & ~3);
+	uint8x16_t x0, x1, x2, x3;
+
+	x0 = *p++ ^ (uint8x16_t)(uint32x4_t){ remainder };
+	if (nr_segs >= 4) {
+		x1 = *p++;
+		x2 = *p++;
+		x3 = *p++;
+
+		/* Fold 512 bits at a time */
+		while (p != end512) {
+			x0 = fold_128b(*p++, x0, multipliers_4);
+			x1 = fold_128b(*p++, x1, multipliers_4);
+			x2 = fold_128b(*p++, x2, multipliers_4);
+			x3 = fold_128b(*p++, x3, multipliers_4);
+		}
+
+		/* Fold 512 bits => 128 bits */
+		x1 = fold_128b(x1, x0, multipliers_1);
+		x2 = fold_128b(x2, x1, multipliers_1);
+		x0 = fold_128b(x3, x2, multipliers_1);
+	}
+
+	/* Fold 128 bits at a time */
+	while (p != end)
+		x0 = fold_128b(*p++, x0, multipliers_1);
+
+	/* Fold 128 => 96 bits, implicitly appending 32 zeroes */
+	x0 = vextq_u8(x0, zeroes, 8) ^ clmul_10(x0, multipliers_1);
+
+	/* Fold 96 => 64 bits */
+	x0 = vextq_u8(x0, zeroes, 4) ^ clmul_00(x0 & mask32, final_multiplier);
+
+	/* Reduce 64 => 32 bits using Barrett reduction */
+	x1 = x0;
+	x0 = clmul_00(x0 & mask32, barrett_reduction_constants);
+	x0 = clmul_10(x0 & mask32, barrett_reduction_constants);
+	return vgetq_lane_u32((uint32x4_t)(x0 ^ x1), 1);
+}
+#define IMPL_ALIGNMENT		16
+#define IMPL_SEGMENT_SIZE	16
+#include "../crc32_vec_template.h"
+#endif /* PMULL implementation */
+
+#ifdef DISPATCH
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_ARM
+	if (features & ARM_CPU_FEATURE_CRC32)
+		return crc32_arm;
+#endif
+#ifdef DISPATCH_PMULL
+	if (features & ARM_CPU_FEATURE_PMULL)
+		return crc32_pmull;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_ARM_CRC32_IMPL_H */

lib/arm/matchfinder_impl.h

@@ -0,0 +1,86 @@
+/*
+ * arm/matchfinder_impl.h - ARM implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_MATCHFINDER_IMPL_H
+#define LIB_ARM_MATCHFINDER_IMPL_H
+
+#ifdef __ARM_NEON
+#  include <arm_neon.h>
+static forceinline void
+matchfinder_init_neon(mf_pos_t *data, size_t size)
+{
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+	};
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_neon
+
+static forceinline void
+matchfinder_rebase_neon(mf_pos_t *data, size_t size)
+{
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+	};
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = vqaddq_s16(p[0], v);
+		p[1] = vqaddq_s16(p[1], v);
+		p[2] = vqaddq_s16(p[2], v);
+		p[3] = vqaddq_s16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_neon
+
+#endif /* __ARM_NEON */
+
+#endif /* LIB_ARM_MATCHFINDER_IMPL_H */

lib/bt_matchfinder.h

@@ -1,15 +1,28 @@
 /*
  * bt_matchfinder.h - Lempel-Ziv matchfinding with a hash table of binary trees
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
  * ----------------------------------------------------------------------------
  *
@@ -49,6 +62,8 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_BT_MATCHFINDER_H
+#define LIB_BT_MATCHFINDER_H
 
 #include "matchfinder_common.h"
 
@@ -56,18 +71,18 @@
 #define BT_MATCHFINDER_HASH3_WAYS  2
 #define BT_MATCHFINDER_HASH4_ORDER 16
 
-#define BT_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
-	 (1UL << BT_MATCHFINDER_HASH4_ORDER))
+#define BT_MATCHFINDER_TOTAL_HASH_SIZE		\
+	(((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
+	  (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 /* Representation of a match found by the bt_matchfinder  */
 struct lz_match {
 
 	/* The number of bytes matched.  */
-	u32 length;
+	u16 length;
 
 	/* The offset back from the current position that was matched.  */
-	u32 offset;
+	u16 offset;
 };
 
 struct bt_matchfinder {
@@ -84,24 +99,24 @@ struct bt_matchfinder {
 	 * 'child_tab[pos * 2]' and 'child_tab[pos * 2 + 1]', respectively.  */
 	mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE];
 
-}
-#ifdef _aligned_attribute
-_aligned_attribute(MATCHFINDER_ALIGNMENT)
-#endif
-;
+} MATCHFINDER_ALIGNED;
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void
 bt_matchfinder_init(struct bt_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 bt_matchfinder_slide_window(struct bt_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct bt_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 static forceinline mf_pos_t *
@@ -117,7 +132,7 @@ bt_right_child(struct bt_matchfinder *mf, s32 node)
 }
 
 /* The minimum permissible value of 'max_len' for bt_matchfinder_get_matches()
- * and bt_matchfinder_skip_position().  There must be sufficiently many bytes
+ * and bt_matchfinder_skip_byte().  There must be sufficiently many bytes
  * remaining to load a 32-bit integer from the *next* position.  */
 #define BT_MATCHFINDER_REQUIRED_NBYTES	5
 
@@ -131,15 +146,13 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 				const u32 nice_len,
 				const u32 max_search_depth,
 				u32 * const restrict next_hashes,
-				u32 * const restrict best_len_ret,
 				struct lz_match * restrict lz_matchptr,
 				const bool record_matches)
 {
 	const u8 *in_next = in_base + cur_pos;
 	u32 depth_remaining = max_search_depth;
 	const s32 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
-	u32 next_seq4;
-	u32 next_seq3;
+	u32 next_hashseq;
 	u32 hash3;
 	u32 hash4;
 	s32 cur_node;
@@ -155,14 +168,13 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 	STATIC_ASSERT(BT_MATCHFINDER_HASH3_WAYS >= 1 &&
 		      BT_MATCHFINDER_HASH3_WAYS <= 2);
 
-	next_seq4 = load_u32_unaligned(in_next + 1);
-	next_seq3 = loaded_u32_to_u24(next_seq4);
+	next_hashseq = get_unaligned_le32(in_next + 1);
 
 	hash3 = next_hashes[0];
 	hash4 = next_hashes[1];
 
-	next_hashes[0] = lz_hash(next_seq3, BT_MATCHFINDER_HASH3_ORDER);
-	next_hashes[1] = lz_hash(next_seq4, BT_MATCHFINDER_HASH4_ORDER);
+	next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, BT_MATCHFINDER_HASH3_ORDER);
+	next_hashes[1] = lz_hash(next_hashseq, BT_MATCHFINDER_HASH4_ORDER);
 	prefetchw(&mf->hash3_tab[next_hashes[0]]);
 	prefetchw(&mf->hash4_tab[next_hashes[1]]);
 
@@ -199,7 +211,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 	if (cur_node <= cutoff) {
 		*pending_lt_ptr = MATCHFINDER_INITVAL;
 		*pending_gt_ptr = MATCHFINDER_INITVAL;
-		*best_len_ret = best_len;
 		return lz_matchptr;
 	}
 
@@ -211,8 +222,7 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 		matchptr = &in_base[cur_node];
 
 		if (matchptr[len] == in_next[len]) {
-			len = lz_extend(in_next, matchptr, len + 1,
-					(record_matches ? max_len : nice_len));
+			len = lz_extend(in_next, matchptr, len + 1, max_len);
 			if (!record_matches || len > best_len) {
 				if (record_matches) {
 					best_len = len;
@@ -223,7 +233,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 				if (len >= nice_len) {
 					*pending_lt_ptr = *bt_left_child(mf, cur_node);
 					*pending_gt_ptr = *bt_right_child(mf, cur_node);
-					*best_len_ret = best_len;
 					return lz_matchptr;
 				}
 			}
@@ -248,7 +257,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 		if (cur_node <= cutoff || !--depth_remaining) {
 			*pending_lt_ptr = MATCHFINDER_INITVAL;
 			*pending_gt_ptr = MATCHFINDER_INITVAL;
-			*best_len_ret = best_len;
 			return lz_matchptr;
 		}
 	}
@@ -277,12 +285,6 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
  *	The precomputed hash codes for the sequence beginning at @in_next.
  *	These will be used and then updated with the precomputed hashcodes for
  *	the sequence beginning at @in_next + 1.
- * @best_len_ret
- *	If a match of length >= 4 was found, then the length of the longest such
- *	match is written here; otherwise 3 is written here.  (Note: this is
- *	redundant with the 'struct lz_match' array, but this is easier for the
- *	compiler to optimize when inlined and the caller immediately does a
- *	check against 'best_len'.)
  * @lz_matchptr
  *	An array in which this function will record the matches.  The recorded
  *	matches will be sorted by strictly increasing length and (non-strictly)
@@ -300,7 +302,6 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
 			   u32 nice_len,
 			   u32 max_search_depth,
 			   u32 next_hashes[2],
-			   u32 *best_len_ret,
 			   struct lz_match *lz_matchptr)
 {
 	return bt_matchfinder_advance_one_byte(mf,
@@ -310,7 +311,6 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
 					       nice_len,
 					       max_search_depth,
 					       next_hashes,
-					       best_len_ret,
 					       lz_matchptr,
 					       true);
 }
@@ -322,23 +322,22 @@ bt_matchfinder_get_matches(struct bt_matchfinder *mf,
  * must do hashing and tree re-rooting.
  */
 static forceinline void
-bt_matchfinder_skip_position(struct bt_matchfinder *mf,
-			     const u8 *in_base,
-			     ptrdiff_t cur_pos,
-			     u32 max_len,
-			     u32 nice_len,
-			     u32 max_search_depth,
-			     u32 next_hashes[2])
+bt_matchfinder_skip_byte(struct bt_matchfinder *mf,
+			 const u8 *in_base,
+			 ptrdiff_t cur_pos,
+			 u32 nice_len,
+			 u32 max_search_depth,
+			 u32 next_hashes[2])
 {
-	u32 best_len;
 	bt_matchfinder_advance_one_byte(mf,
 					in_base,
 					cur_pos,
-					max_len,
+					nice_len,
 					nice_len,
 					max_search_depth,
 					next_hashes,
-					&best_len,
 					NULL,
 					false);
 }
+
+#endif /* LIB_BT_MATCHFINDER_H */

lib/cpu_features_common.h

@@ -0,0 +1,88 @@
+/*
+ * cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
+ *
+ * Copyright 2020 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_CPU_FEATURES_COMMON_H
+#define LIB_CPU_FEATURES_COMMON_H
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+#  define _GNU_SOURCE 1 /* for strdup() and strtok_r() */
+#  include <stdio.h>
+#  include <stdlib.h>
+#  include <string.h>
+#endif
+
+#include "lib_common.h"
+
+struct cpu_feature {
+	u32 bit;
+	const char *name;
+};
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+	char *env_value, *strbuf, *p, *saveptr = NULL;
+	size_t i;
+
+	env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
+	if (!env_value)
+		return;
+	strbuf = strdup(env_value);
+	if (!strbuf)
+		abort();
+	p = strtok_r(strbuf, ",", &saveptr);
+	while (p) {
+		for (i = 0; i < feature_table_length; i++) {
+			if (strcmp(p, feature_table[i].name) == 0) {
+				*features &= ~feature_table[i].bit;
+				break;
+			}
+		}
+		if (i == feature_table_length) {
+			fprintf(stderr,
+				"unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
+				p);
+			abort();
+		}
+		p = strtok_r(NULL, ",", &saveptr);
+	}
+	free(strbuf);
+}
+#else /* TEST_SUPPORT__DO_NOT_USE */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+}
+#endif /* !TEST_SUPPORT__DO_NOT_USE */
+
+#endif /* LIB_CPU_FEATURES_COMMON_H */

lib/crc32.c

@@ -1,22 +1,35 @@
 /*
  * crc32.c - CRC-32 checksum algorithm for the gzip format
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /*
  * High-level description of CRC
  * =============================
  *
- * Consider a bit sequence 'bits[1...len]'.  Interpet 'bits' as the "message"
+ * Consider a bit sequence 'bits[1...len]'.  Interpret 'bits' as the "message"
  * polynomial M(x) with coefficients in GF(2) (the field of integers modulo 2),
  * where the coefficient of 'x^i' is 'bits[len - i]'.  Then, compute:
  *
@@ -24,7 +37,7 @@
  *
  * where G(x) is a selected "generator" polynomial of degree 'n'.  The remainder
  * R(x) is a polynomial of max degree 'n - 1'.  The CRC of 'bits' is R(x)
- * interpeted as a bitstring of length 'n'.
+ * interpreted as a bitstring of length 'n'.
  *
  * CRC used in gzip
  * ================
@@ -57,16 +70,16 @@
  * subtraction can be implemented as bitwise exclusive OR (since we are working
  * in GF(2)).  Here is an unoptimized implementation:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t nbytes)
+ *	static u32 crc32_gzip(const u8 *buffer, size_t size)
  *	{
  *		u32 remainder = 0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < nbytes * 8 + 32; i++) {
+ *		for (size_t i = 0; i < size * 8 + 32; i++) {
  *			int bit;
  *			u32 multiple;
  *
- *			if (i < nbytes * 8)
+ *			if (i < size * 8)
  *				bit = (buffer[i / 8] >> (i % 8)) & 1;
  *			else
  *				bit = 0; // one of the 32 appended 0 bits
@@ -101,12 +114,12 @@
  * of 'multiple' until 32 bits later, we need not actually add each message bit
  * until that point:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t nbytes)
+ *	static u32 crc32_gzip(const u8 *buffer, size_t size)
  *	{
  *		u32 remainder = ~0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < nbytes * 8; i++) {
+ *		for (size_t i = 0; i < size * 8; i++) {
  *			int bit;
  *			u32 multiple;
  *
@@ -151,39 +164,67 @@
  *
  * In crc32_slice8(), this method is extended to 8 bytes at a time.  The
  * intermediate remainder (which we never actually store explicitly) is 96 bits.
+ *
+ * On CPUs that support fast carryless multiplication, CRCs can be computed even
+ * more quickly via "folding".  See e.g. the x86 PCLMUL implementation.
  */
 
-#define CRC32_SLICE8
+#include "lib_common.h"
+#include "libdeflate.h"
 
-#include "crc32.h"
+typedef u32 (*crc32_func_t)(u32, const u8 *, size_t);
+
+/* Include architecture-specific implementations if available */
+#undef CRC32_SLICE1
+#undef CRC32_SLICE4
+#undef CRC32_SLICE8
+#undef DEFAULT_IMPL
+#undef DISPATCH
+#if defined(__arm__) || defined(__aarch64__)
+#  include "arm/crc32_impl.h"
+#elif defined(__i386__) || defined(__x86_64__)
+#  include "x86/crc32_impl.h"
+#endif
+
+/*
+ * Define a generic implementation (crc32_slice8()) if needed.  crc32_slice1()
+ * may also be needed as a fallback for architecture-specific implementations.
+ */
+
+#ifndef DEFAULT_IMPL
+#  define CRC32_SLICE8	1
+#  define DEFAULT_IMPL	crc32_slice8
+#endif
+
+#if defined(CRC32_SLICE1) || defined(CRC32_SLICE4) || defined(CRC32_SLICE8)
 #include "crc32_table.h"
-
 static forceinline u32
 crc32_update_byte(u32 remainder, u8 next_byte)
 {
 	return (remainder >> 8) ^ crc32_table[(u8)remainder ^ next_byte];
 }
+#endif
 
 #ifdef CRC32_SLICE1
 static u32
-crc32_slice1(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice1(u32 remainder, const u8 *buffer, size_t size)
 {
 	size_t i;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x100);
 
-	for (i = 0; i < nbytes; i++)
+	for (i = 0; i < size; i++)
 		remainder = crc32_update_byte(remainder, buffer[i]);
 	return remainder;
 }
-#endif
+#endif /* CRC32_SLICE1 */
 
 #ifdef CRC32_SLICE4
 static u32
-crc32_slice4(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice4(u32 remainder, const u8 *buffer, size_t size)
 {
 	const u8 *p = buffer;
-	const u8 *end = buffer + nbytes;
+	const u8 *end = buffer + size;
 	const u8 *end32;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x400);
@@ -206,14 +247,14 @@ crc32_slice4(u32 remainder, const u8 *buffer, size_t nbytes)
 
 	return remainder;
 }
-#endif
+#endif /* CRC32_SLICE4 */
 
 #ifdef CRC32_SLICE8
 static u32
-crc32_slice8(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice8(u32 remainder, const u8 *buffer, size_t size)
 {
 	const u8 *p = buffer;
-	const u8 *end = buffer + nbytes;
+	const u8 *end = buffer + size;
 	const u8 *end64;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x800);
@@ -241,20 +282,32 @@ crc32_slice8(u32 remainder, const u8 *buffer, size_t nbytes)
 
 	return remainder;
 }
+#endif /* CRC32_SLICE8 */
+
+#ifdef DISPATCH
+static u32 dispatch(u32, const u8 *, size_t);
+
+static volatile crc32_func_t crc32_impl = dispatch;
+
+/* Choose the fastest implementation at runtime */
+static u32 dispatch(u32 remainder, const u8 *buffer, size_t size)
+{
+	crc32_func_t f = arch_select_crc32_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
+	crc32_impl = f;
+	return crc32_impl(remainder, buffer, size);
+}
+#else
+#  define crc32_impl DEFAULT_IMPL /* only one implementation, use it */
 #endif
 
-u32
-crc32_gzip(const void *buffer, size_t nbytes)
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
+libdeflate_crc32(u32 remainder, const void *buffer, size_t size)
 {
-	u32 remainder = ~0;
-#if defined(CRC32_SLICE1)
-	remainder = crc32_slice1(remainder, buffer, nbytes);
-#elif defined(CRC32_SLICE4)
-	remainder = crc32_slice4(remainder, buffer, nbytes);
-#elif defined(CRC32_SLICE8)
-	remainder = crc32_slice8(remainder, buffer, nbytes);
-#else
-#  error "don't know which CRC-32 implementation to use!"
-#endif
-	return ~remainder;
+	if (buffer == NULL) /* return initial value */
+		return 0;
+	return ~crc32_impl(~remainder, buffer, size);
 }

lib/crc32.h

@@ -1,12 +0,0 @@
-/*
- * crc32.h - CRC-32 checksum algorithm for the gzip format
- */
-
-#ifndef _LIB_CRC32_H
-#define _LIB_CRC32_H
-
-#include "common_defs.h"
-
-extern u32 crc32_gzip(const void *buffer, size_t size);
-
-#endif /* _LIB_CRC32_H */

lib/crc32_vec_template.h

@@ -0,0 +1,61 @@
+/*
+ * crc32_vec_template.h - template for vectorized CRC-32 implementations
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#define CRC32_SLICE1	1
+static u32 crc32_slice1(u32, const u8 *, size_t);
+
+/*
+ * Template for vectorized CRC-32 implementations.
+ *
+ * Note: on unaligned ends of the buffer, we fall back to crc32_slice1() instead
+ * of crc32_slice8() because only a few bytes need to be processed, so a smaller
+ * table is preferable.
+ */
+static u32 ATTRIBUTES
+FUNCNAME(u32 remainder, const u8 *p, size_t size)
+{
+	if ((uintptr_t)p % IMPL_ALIGNMENT) {
+		size_t n = MIN(size, -(uintptr_t)p % IMPL_ALIGNMENT);
+
+		remainder = crc32_slice1(remainder, p, n);
+		p += n;
+		size -= n;
+	}
+	if (size >= IMPL_SEGMENT_SIZE) {
+		remainder = FUNCNAME_ALIGNED(remainder, (const void *)p,
+					     size / IMPL_SEGMENT_SIZE);
+		p += size - (size % IMPL_SEGMENT_SIZE);
+		size %= IMPL_SEGMENT_SIZE;
+	}
+	return crc32_slice1(remainder, p, size);
+}
+
+#undef FUNCNAME
+#undef FUNCNAME_ALIGNED
+#undef ATTRIBUTES
+#undef IMPL_ALIGNMENT
+#undef IMPL_SEGMENT_SIZE

lib/decompress_template.h

@@ -1,15 +1,28 @@
 /*
- * decompress_impl.h
+ * decompress_template.h
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /*
@@ -18,11 +31,11 @@
  * target instruction sets.
  */
 
-static enum decompress_result ATTRIBUTES
-FUNCNAME(struct deflate_decompressor * restrict d,
+static enum libdeflate_result ATTRIBUTES
+FUNCNAME(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
 	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_out_nbytes_ret)
+	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
 {
 	u8 *out_next = out;
 	u8 * const out_end = out_next + out_nbytes_avail;
@@ -30,7 +43,7 @@ FUNCNAME(struct deflate_decompressor * restrict d,
 	const u8 * const in_end = in_next + in_nbytes;
 	bitbuf_t bitbuf = 0;
 	unsigned bitsleft = 0;
-	size_t overrun_count = 0;
+	size_t overread_count = 0;
 	unsigned i;
 	unsigned is_final_block;
 	unsigned block_type;
@@ -76,23 +89,17 @@ next_block:
 		STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == ((1 << 4) - 1) + 4);
 		num_explicit_precode_lens = POP_BITS(4) + 4;
 
+		d->static_codes_loaded = false;
+
 		/* Read the precode codeword lengths.  */
 		STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
-		if (CAN_ENSURE(DEFLATE_NUM_PRECODE_SYMS * 3)) {
-
-			ENSURE_BITS(DEFLATE_NUM_PRECODE_SYMS * 3);
-
-			for (i = 0; i < num_explicit_precode_lens; i++)
-				d->precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-		} else {
-			for (i = 0; i < num_explicit_precode_lens; i++) {
-				ENSURE_BITS(3);
-				d->precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-			}
+		for (i = 0; i < num_explicit_precode_lens; i++) {
+			ENSURE_BITS(3);
+			d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
 		}
 
 		for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
-			d->precode_lens[deflate_precode_lens_permutation[i]] = 0;
+			d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
 
 		/* Build the decode table for the precode.  */
 		SAFETY_CHECK(build_precode_decode_table(d));
@@ -111,13 +118,13 @@ next_block:
 			STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
 
 			/* Read the next precode symbol.  */
-			entry = d->precode_decode_table[BITS(DEFLATE_MAX_PRE_CODEWORD_LEN)];
+			entry = d->u.l.precode_decode_table[BITS(DEFLATE_MAX_PRE_CODEWORD_LEN)];
 			REMOVE_BITS(entry & HUFFDEC_LENGTH_MASK);
 			presym = entry >> HUFFDEC_RESULT_SHIFT;
 
 			if (presym < 16) {
 				/* Explicit codeword length  */
-				d->lens[i++] = presym;
+				d->u.l.lens[i++] = presym;
 				continue;
 			}
 
@@ -144,36 +151,37 @@ next_block:
 			if (presym == 16) {
 				/* Repeat the previous length 3 - 6 times  */
 				SAFETY_CHECK(i != 0);
-				rep_val = d->lens[i - 1];
+				rep_val = d->u.l.lens[i - 1];
 				STATIC_ASSERT(3 + ((1 << 2) - 1) == 6);
 				rep_count = 3 + POP_BITS(2);
-				d->lens[i + 0] = rep_val;
-				d->lens[i + 1] = rep_val;
-				d->lens[i + 2] = rep_val;
-				d->lens[i + 3] = rep_val;
-				d->lens[i + 4] = rep_val;
-				d->lens[i + 5] = rep_val;
+				d->u.l.lens[i + 0] = rep_val;
+				d->u.l.lens[i + 1] = rep_val;
+				d->u.l.lens[i + 2] = rep_val;
+				d->u.l.lens[i + 3] = rep_val;
+				d->u.l.lens[i + 4] = rep_val;
+				d->u.l.lens[i + 5] = rep_val;
 				i += rep_count;
 			} else if (presym == 17) {
 				/* Repeat zero 3 - 10 times  */
 				STATIC_ASSERT(3 + ((1 << 3) - 1) == 10);
 				rep_count = 3 + POP_BITS(3);
-				d->lens[i + 0] = 0;
-				d->lens[i + 1] = 0;
-				d->lens[i + 2] = 0;
-				d->lens[i + 3] = 0;
-				d->lens[i + 4] = 0;
-				d->lens[i + 5] = 0;
-				d->lens[i + 6] = 0;
-				d->lens[i + 7] = 0;
-				d->lens[i + 8] = 0;
-				d->lens[i + 9] = 0;
+				d->u.l.lens[i + 0] = 0;
+				d->u.l.lens[i + 1] = 0;
+				d->u.l.lens[i + 2] = 0;
+				d->u.l.lens[i + 3] = 0;
+				d->u.l.lens[i + 4] = 0;
+				d->u.l.lens[i + 5] = 0;
+				d->u.l.lens[i + 6] = 0;
+				d->u.l.lens[i + 7] = 0;
+				d->u.l.lens[i + 8] = 0;
+				d->u.l.lens[i + 9] = 0;
 				i += rep_count;
 			} else {
 				/* Repeat zero 11 - 138 times  */
 				STATIC_ASSERT(11 + ((1 << 7) - 1) == 138);
 				rep_count = 11 + POP_BITS(7);
-				memset(&d->lens[i], 0, rep_count * sizeof(d->lens[i]));
+				memset(&d->u.l.lens[i], 0,
+				       rep_count * sizeof(d->u.l.lens[i]));
 				i += rep_count;
 			}
 		}
@@ -191,7 +199,7 @@ next_block:
 
 		SAFETY_CHECK(len == (u16)~nlen);
 		if (unlikely(len > out_end - out_next))
-			return DECOMPRESS_INSUFFICIENT_SPACE;
+			return LIBDEFLATE_INSUFFICIENT_SPACE;
 		SAFETY_CHECK(len <= in_end - in_next);
 
 		memcpy(out_next, in_next, len);
@@ -203,48 +211,61 @@ next_block:
 	} else {
 		SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
 
-		/* Static Huffman block: set the static Huffman codeword
-		 * lengths.  Then the remainder is the same as decompressing a
-		 * dynamic Huffman block.  */
+		/*
+		 * Static Huffman block: build the decode tables for the static
+		 * codes.  Skip doing so if the tables are already set up from
+		 * an earlier static block; this speeds up decompression of
+		 * degenerate input of many empty or very short static blocks.
+		 *
+		 * Afterwards, the remainder is the same as decompressing a
+		 * dynamic Huffman block.
+		 */
+
+		if (d->static_codes_loaded)
+			goto have_decode_tables;
+
+		d->static_codes_loaded = true;
 
 		STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
 		STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
 
 		for (i = 0; i < 144; i++)
-			d->lens[i] = 8;
+			d->u.l.lens[i] = 8;
 		for (; i < 256; i++)
-			d->lens[i] = 9;
+			d->u.l.lens[i] = 9;
 		for (; i < 280; i++)
-			d->lens[i] = 7;
+			d->u.l.lens[i] = 7;
 		for (; i < 288; i++)
-			d->lens[i] = 8;
+			d->u.l.lens[i] = 8;
 
 		for (; i < 288 + 32; i++)
-			d->lens[i] = 5;
+			d->u.l.lens[i] = 5;
 
 		num_litlen_syms = 288;
 		num_offset_syms = 32;
-
 	}
 
 	/* Decompressing a Huffman block (either dynamic or static)  */
 
 	SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
 	SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
+have_decode_tables:
 
 	/* The main DEFLATE decode loop  */
 	for (;;) {
 		u32 entry;
 		u32 length;
 		u32 offset;
+		const u8 *src;
+		u8 *dst;
 
 		/* Decode a litlen symbol.  */
 		ENSURE_BITS(DEFLATE_MAX_LITLEN_CODEWORD_LEN);
-		entry = d->litlen_decode_table[BITS(LITLEN_TABLEBITS)];
+		entry = d->u.litlen_decode_table[BITS(LITLEN_TABLEBITS)];
 		if (entry & HUFFDEC_SUBTABLE_POINTER) {
 			/* Litlen subtable required (uncommon case)  */
 			REMOVE_BITS(LITLEN_TABLEBITS);
-			entry = d->litlen_decode_table[
+			entry = d->u.litlen_decode_table[
 				((entry >> HUFFDEC_RESULT_SHIFT) & 0xFFFF) +
 				BITS(entry & HUFFDEC_LENGTH_MASK)];
 		}
@@ -252,7 +273,7 @@ next_block:
 		if (entry & HUFFDEC_LITERAL) {
 			/* Literal  */
 			if (unlikely(out_next == out_end))
-				return DECOMPRESS_INSUFFICIENT_SPACE;
+				return LIBDEFLATE_INSUFFICIENT_SPACE;
 			*out_next++ = (u8)(entry >> HUFFDEC_RESULT_SHIFT);
 			continue;
 		}
@@ -275,7 +296,7 @@ next_block:
 		STATIC_ASSERT(HUFFDEC_END_OF_BLOCK_LENGTH == 0);
 		if (unlikely((size_t)length - 1 >= out_end - out_next)) {
 			if (unlikely(length != HUFFDEC_END_OF_BLOCK_LENGTH))
-				return DECOMPRESS_INSUFFICIENT_SPACE;
+				return LIBDEFLATE_INSUFFICIENT_SPACE;
 			goto block_done;
 		}
 
@@ -309,65 +330,66 @@ next_block:
 		 * output buffer.  */
 		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
 
-		/* Copy the match: 'length' bytes at 'out_next - offset' to
-		 * 'out_next'.  */
+		/*
+		 * Copy the match: 'length' bytes at 'out_next - offset' to
+		 * 'out_next', possibly overlapping.  If the match doesn't end
+		 * too close to the end of the buffer and offset >= WORDBYTES ||
+		 * offset == 1, take a fast path which copies a word at a time
+		 * -- potentially more than the length of the match, but that's
+		 * fine as long as we check for enough extra space.
+		 *
+		 * The remaining cases are not performance-critical so are
+		 * handled by a simple byte-by-byte copy.
+		 */
+
+		src = out_next - offset;
+		dst = out_next;
+		out_next += length;
 
 		if (UNALIGNED_ACCESS_IS_FAST &&
-		    length <= (3 * WORDBYTES) &&
-		    offset >= WORDBYTES &&
-		    length + (3 * WORDBYTES) <= out_end - out_next)
-		{
-			/* Fast case: short length, no overlaps if we copy one
-			 * word at a time, and we aren't getting too close to
-			 * the end of the output array.  */
-			copy_word_unaligned(out_next - offset + (0 * WORDBYTES),
-					    out_next + (0 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (1 * WORDBYTES),
-					    out_next + (1 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (2 * WORDBYTES),
-					    out_next + (2 * WORDBYTES));
-		} else {
-			const u8 *src = out_next - offset;
-			u8 *dst = out_next;
-			u8 *end = out_next + length;
-
-			if (UNALIGNED_ACCESS_IS_FAST &&
-			    likely(out_end - end >= WORDBYTES - 1)) {
-				if (offset >= WORDBYTES) {
+		    /* max overrun is writing 3 words for a min length match */
+		    likely(out_end - out_next >=
+			   3 * WORDBYTES - DEFLATE_MIN_MATCH_LEN)) {
+			if (offset >= WORDBYTES) { /* words don't overlap? */
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				do {
 					copy_word_unaligned(src, dst);
 					src += WORDBYTES;
 					dst += WORDBYTES;
-					if (dst < end) {
-						do {
-							copy_word_unaligned(src, dst);
-							src += WORDBYTES;
-							dst += WORDBYTES;
-						} while (dst < end);
-					}
-				} else if (offset == 1) {
-					machine_word_t v = repeat_byte(*(dst - 1));
-					do {
-						store_word_unaligned(v, dst);
-						src += WORDBYTES;
-						dst += WORDBYTES;
-					} while (dst < end);
-				} else {
-					*dst++ = *src++;
-					*dst++ = *src++;
-					do {
-						*dst++ = *src++;
-					} while (dst < end);
-				}
+				} while (dst < out_next);
+			} else if (offset == 1) {
+				/* RLE encoding of previous byte, common if the
+				 * data contains many repeated bytes */
+				machine_word_t v = repeat_byte(*src);
+
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				do {
+					store_word_unaligned(v, dst);
+					dst += WORDBYTES;
+				} while (dst < out_next);
 			} else {
 				*dst++ = *src++;
 				*dst++ = *src++;
 				do {
 					*dst++ = *src++;
-				} while (dst < end);
+				} while (dst < out_next);
 			}
+		} else {
+			STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
+			*dst++ = *src++;
+			*dst++ = *src++;
+			do {
+				*dst++ = *src++;
+			} while (dst < out_next);
 		}
-
-		out_next += length;
 	}
 
 block_done:
@@ -378,11 +400,22 @@ block_done:
 
 	/* That was the last block.  */
 
+	/* Discard any readahead bits and check for excessive overread */
+	ALIGN_INPUT();
+
+	/* Optionally return the actual number of bytes read */
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	/* Optionally return the actual number of bytes written */
 	if (actual_out_nbytes_ret) {
 		*actual_out_nbytes_ret = out_next - (u8 *)out;
 	} else {
 		if (out_next != out_end)
-			return DECOMPRESS_SHORT_OUTPUT;
+			return LIBDEFLATE_SHORT_OUTPUT;
 	}
-	return DECOMPRESS_SUCCESS;
+	return LIBDEFLATE_SUCCESS;
 }
+
+#undef FUNCNAME
+#undef ATTRIBUTES

lib/deflate_compress.h

@@ -1,11 +1,15 @@
-#ifndef _LIB_DEFLATE_COMPRESS_H
-#define _LIB_DEFLATE_COMPRESS_H
+#ifndef LIB_DEFLATE_COMPRESS_H
+#define LIB_DEFLATE_COMPRESS_H
 
-/* DEFLATE compression is private to deflate_compress.c, but we do need to be
- * able to query the compression level for zlib and gzip header generation.  */
+#include "lib_common.h"
 
-struct deflate_compressor;
+/*
+ * DEFLATE compression is private to deflate_compress.c, but we do need to be
+ * able to query the compression level for zlib and gzip header generation.
+ */
 
-extern unsigned int deflate_get_compression_level(struct deflate_compressor *c);
+struct libdeflate_compressor;
 
-#endif /* _LIB_DEFLATE_COMPRESS_H */
+unsigned int deflate_get_compression_level(struct libdeflate_compressor *c);
+
+#endif /* LIB_DEFLATE_COMPRESS_H */

lib/deflate_constants.h

@@ -2,8 +2,8 @@
  * deflate_constants.h - constants for the DEFLATE compression format
  */
 
-#ifndef _LIB_DEFLATE_CONSTANTS_H
-#define _LIB_DEFLATE_CONSTANTS_H
+#ifndef LIB_DEFLATE_CONSTANTS_H
+#define LIB_DEFLATE_CONSTANTS_H
 
 /* Valid block types  */
 #define DEFLATE_BLOCKTYPE_UNCOMPRESSED		0
@@ -14,11 +14,11 @@
 #define DEFLATE_MIN_MATCH_LEN			3
 #define DEFLATE_MAX_MATCH_LEN			258
 
-/* Minimum and maximum supported match offsets (in bytes)  */
-#define DEFLATE_MIN_MATCH_OFFSET		1
+/* Maximum supported match offset (in bytes) */
 #define DEFLATE_MAX_MATCH_OFFSET		32768
 
-#define DEFLATE_MAX_WINDOW_SIZE			32768
+/* log2 of DEFLATE_MAX_MATCH_OFFSET */
+#define DEFLATE_WINDOW_ORDER			15
 
 /* Number of symbols in each Huffman code.  Note: for the literal/length
  * and offset codes, these are actually the maximum values; a given block
@@ -33,7 +33,7 @@
 /* Division of symbols in the literal/length code  */
 #define DEFLATE_NUM_LITERALS			256
 #define DEFLATE_END_OF_BLOCK			256
-#define DEFLATE_NUM_LEN_SYMS			31
+#define DEFLATE_FIRST_LEN_SYM			257
 
 /* Maximum codeword length, in bits, within each Huffman code  */
 #define DEFLATE_MAX_PRE_CODEWORD_LEN		7
@@ -56,11 +56,4 @@
 #define DEFLATE_MAX_EXTRA_LENGTH_BITS		5
 #define DEFLATE_MAX_EXTRA_OFFSET_BITS		14
 
-/* The maximum number of bits in which a match can be represented.  This
- * is the absolute worst case, which assumes the longest possible Huffman
- * codewords and the maximum numbers of extra bits.  */
-#define DEFLATE_MAX_MATCH_BITS	\
-	(DEFLATE_MAX_LITLEN_CODEWORD_LEN + DEFLATE_MAX_EXTRA_LENGTH_BITS + \
-	DEFLATE_MAX_OFFSET_CODEWORD_LEN + DEFLATE_MAX_EXTRA_OFFSET_BITS)
-
-#endif /* _LIB_DEFLATE_CONSTANTS_H */
+#endif /* LIB_DEFLATE_CONSTANTS_H */

lib/gzip_compress.c

@@ -1,27 +1,40 @@
 /*
  * gzip_compress.c - compress with a gzip wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "crc32.h"
 #include "deflate_compress.h"
 #include "gzip_constants.h"
 #include "unaligned.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT size_t
-gzip_compress(struct deflate_compressor *c, const void *in, size_t in_size,
-	      void *out, size_t out_nbytes_avail)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress(struct libdeflate_compressor *c,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
 	unsigned compression_level;
@@ -46,33 +59,35 @@ gzip_compress(struct deflate_compressor *c, const void *in, size_t in_size,
 	xfl = 0;
 	compression_level = deflate_get_compression_level(c);
 	if (compression_level < 2)
-		xfl |= GZIP_XFL_FASTEST_COMRESSION;
+		xfl |= GZIP_XFL_FASTEST_COMPRESSION;
 	else if (compression_level >= 8)
-		xfl |= GZIP_XFL_SLOWEST_COMRESSION;
+		xfl |= GZIP_XFL_SLOWEST_COMPRESSION;
 	*out_next++ = xfl;
 	/* OS */
 	*out_next++ = GZIP_OS_UNKNOWN;	/* OS  */
 
 	/* Compressed data  */
-	deflate_size = deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - GZIP_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* CRC32 */
-	put_unaligned_le32(crc32_gzip(in, in_size), out_next);
+	put_unaligned_le32(libdeflate_crc32(0, in, in_nbytes), out_next);
 	out_next += 4;
 
 	/* ISIZE */
-	put_unaligned_le32((u32)in_size, out_next);
+	put_unaligned_le32((u32)in_nbytes, out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBEXPORT size_t
-gzip_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *c,
+			       size_t in_nbytes)
 {
-	return GZIP_MIN_OVERHEAD + deflate_compress_bound(c, in_nbytes);
+	return GZIP_MIN_OVERHEAD +
+	       libdeflate_deflate_compress_bound(c, in_nbytes);
 }

lib/gzip_constants.h

@@ -2,8 +2,8 @@
  * gzip_constants.h - constants for the gzip wrapper format
  */
 
-#ifndef _LIB_GZIP_CONSTANTS_H
-#define _LIB_GZIP_CONSTANTS_H
+#ifndef LIB_GZIP_CONSTANTS_H
+#define LIB_GZIP_CONSTANTS_H
 
 #define GZIP_MIN_HEADER_SIZE	10
 #define GZIP_FOOTER_SIZE	8
@@ -23,8 +23,8 @@
 
 #define GZIP_MTIME_UNAVAILABLE	0
 
-#define GZIP_XFL_SLOWEST_COMRESSION	0x02
-#define GZIP_XFL_FASTEST_COMRESSION	0x04
+#define GZIP_XFL_SLOWEST_COMPRESSION	0x02
+#define GZIP_XFL_FASTEST_COMPRESSION	0x04
 
 #define GZIP_OS_FAT		0
 #define GZIP_OS_AMIGA		1
@@ -42,4 +42,4 @@
 #define GZIP_OS_RISCOS		13
 #define GZIP_OS_UNKNOWN		255
 
-#endif /* _LIB_GZIP_CONSTANTS_H */
+#endif /* LIB_GZIP_CONSTANTS_H */

lib/gzip_decompress.c

@@ -1,47 +1,61 @@
 /*
  * gzip_decompress.c - decompress with a gzip wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "crc32.h"
 #include "gzip_constants.h"
 #include "unaligned.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT enum decompress_result
-gzip_decompress(struct deflate_decompressor *d,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret)
 {
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
 	u8 flg;
+	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
-	enum decompress_result result;
+	enum libdeflate_result result;
 
 	if (in_nbytes < GZIP_MIN_OVERHEAD)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* ID1 */
 	if (*in_next++ != GZIP_ID1)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	/* ID2 */
 	if (*in_next++ != GZIP_ID2)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	/* CM */
 	if (*in_next++ != GZIP_CM_DEFLATE)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	flg = *in_next++;
 	/* MTIME */
 	in_next += 4;
@@ -51,7 +65,7 @@ gzip_decompress(struct deflate_decompressor *d,
 	in_next += 1;
 
 	if (flg & GZIP_FRESERVED)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* Extra field */
 	if (flg & GZIP_FEXTRA) {
@@ -59,7 +73,7 @@ gzip_decompress(struct deflate_decompressor *d,
 		in_next += 2;
 
 		if (in_end - in_next < (u32)xlen + GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 
 		in_next += xlen;
 	}
@@ -69,7 +83,7 @@ gzip_decompress(struct deflate_decompressor *d,
 		while (*in_next++ != 0 && in_next != in_end)
 			;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* File comment (zero terminated) */
@@ -77,22 +91,23 @@ gzip_decompress(struct deflate_decompressor *d,
 		while (*in_next++ != 0 && in_next != in_end)
 			;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* CRC16 for gzip header */
 	if (flg & GZIP_FHCRC) {
 		in_next += 2;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* Compressed data  */
-	result = deflate_decompress(d, in_next,
-				    in_end - GZIP_FOOTER_SIZE - in_next,
-				    out, out_nbytes_avail,
-				    actual_out_nbytes_ret);
-	if (result != DECOMPRESS_SUCCESS)
+	result = libdeflate_deflate_decompress_ex(d, in_next,
+					in_end - GZIP_FOOTER_SIZE - in_next,
+					out, out_nbytes_avail,
+					&actual_in_nbytes,
+					actual_out_nbytes_ret);
+	if (result != LIBDEFLATE_SUCCESS)
 		return result;
 
 	if (actual_out_nbytes_ret)
@@ -100,16 +115,32 @@ gzip_decompress(struct deflate_decompressor *d,
 	else
 		actual_out_nbytes = out_nbytes_avail;
 
-	in_next = in_end - GZIP_FOOTER_SIZE;
+	in_next += actual_in_nbytes;
 
 	/* CRC32 */
-	if (crc32_gzip(out, actual_out_nbytes) != get_unaligned_le32(in_next))
-		return DECOMPRESS_BAD_DATA;
+	if (libdeflate_crc32(0, out, actual_out_nbytes) !=
+	    get_unaligned_le32(in_next))
+		return LIBDEFLATE_BAD_DATA;
 	in_next += 4;
 
 	/* ISIZE */
 	if ((u32)actual_out_nbytes != get_unaligned_le32(in_next))
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
+	in_next += 4;
 
-	return DECOMPRESS_SUCCESS;
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	return LIBDEFLATE_SUCCESS;
+}
+
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress(struct libdeflate_decompressor *d,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret)
+{
+	return libdeflate_gzip_decompress_ex(d, in, in_nbytes,
+					     out, out_nbytes_avail,
+					     NULL, actual_out_nbytes_ret);
 }

lib/hc_matchfinder.h

@@ -1,15 +1,28 @@
 /*
  * hc_matchfinder.h - Lempel-Ziv matchfinding with a hash table of linked lists
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
  * ---------------------------------------------------------------------------
  *
@@ -60,7 +73,7 @@
  * chain for length 3+ matches, the algorithm just checks for one close length 3
  * match, then focuses on finding length 4+ matches.
  *
- * The longest_match() and skip_positions() functions are inlined into the
+ * The longest_match() and skip_bytes() functions are inlined into the
  * compressors that use them.  This isn't just about saving the overhead of a
  * function call.  These functions are intended to be called from the inner
  * loops of compressors, where giving the compiler more control over register
@@ -91,14 +104,17 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_HC_MATCHFINDER_H
+#define LIB_HC_MATCHFINDER_H
+
 #include "matchfinder_common.h"
 
 #define HC_MATCHFINDER_HASH3_ORDER	15
 #define HC_MATCHFINDER_HASH4_ORDER	16
 
-#define HC_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
-	 (1UL << HC_MATCHFINDER_HASH4_ORDER))
+#define HC_MATCHFINDER_TOTAL_HASH_SIZE			\
+	(((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
+	  (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 struct hc_matchfinder {
 
@@ -113,24 +129,24 @@ struct hc_matchfinder {
 	 * the node for the sequence with position 'pos' is 'next_tab[pos]'.  */
 	mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE];
 
-}
-#ifdef _aligned_attribute
-  _aligned_attribute(MATCHFINDER_ALIGNMENT)
-#endif
-;
+} MATCHFINDER_ALIGNED;
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void
 hc_matchfinder_init(struct hc_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 hc_matchfinder_slide_window(struct hc_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct hc_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 /*
@@ -142,9 +158,9 @@ hc_matchfinder_slide_window(struct hc_matchfinder *mf)
  *	Location of a pointer which points to the place in the input data the
  *	matchfinder currently stores positions relative to.  This may be updated
  *	by this function.
- * @cur_pos
- *	The current position in the input buffer relative to @in_base (the
- *	position of the sequence being matched against).
+ * @in_next
+ *	Pointer to the next position in the input buffer, i.e. the sequence
+ *	being matched against.
  * @best_len
  *	Require a match longer than this length.
  * @max_len
@@ -179,7 +195,7 @@ hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf,
 	const u8 *best_matchptr = in_next;
 	mf_pos_t cur_node3, cur_node4;
 	u32 hash3, hash4;
-	u32 next_seq3, next_seq4;
+	u32 next_hashseq;
 	u32 seq4;
 	const u8 *matchptr;
 	u32 len;
@@ -217,10 +233,9 @@ hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf,
 	mf->next_tab[cur_pos] = cur_node4;
 
 	/* Compute the next hash codes.  */
-	next_seq4 = load_u32_unaligned(in_next + 1);
-	next_seq3 = loaded_u32_to_u24(next_seq4);
-	next_hashes[0] = lz_hash(next_seq3, HC_MATCHFINDER_HASH3_ORDER);
-	next_hashes[1] = lz_hash(next_seq4, HC_MATCHFINDER_HASH4_ORDER);
+	next_hashseq = get_unaligned_le32(in_next + 1);
+	next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+	next_hashes[1] = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
 	prefetchw(&mf->hash3_tab[next_hashes[0]]);
 	prefetchw(&mf->hash4_tab[next_hashes[1]]);
 
@@ -332,34 +347,32 @@ out:
  *	Location of a pointer which points to the place in the input data the
  *	matchfinder currently stores positions relative to.  This may be updated
  *	by this function.
- * @cur_pos
- *	The current position in the input buffer relative to @in_base.
- * @end_pos
- *	The end position of the input buffer, relative to @in_base.
+ * @in_next
+ *	Pointer to the next position in the input buffer.
+ * @in_end
+ *	Pointer to the end of the input buffer.
+ * @count
+ *	The number of bytes to advance.  Must be > 0.
  * @next_hashes
  *	The precomputed hash codes for the sequence beginning at @in_next.
  *	These will be used and then updated with the precomputed hashcodes for
  *	the sequence beginning at @in_next + @count.
- * @count
- *	The number of bytes to advance.  Must be > 0.
- *
- * Returns @in_next + @count.
  */
-static forceinline const u8 *
-hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
-			      const u8 ** const restrict in_base_p,
-			      const u8 *in_next,
-			      const u8 * const in_end,
-			      const u32 count,
-			      u32 * const restrict next_hashes)
+static forceinline void
+hc_matchfinder_skip_bytes(struct hc_matchfinder * const restrict mf,
+			  const u8 ** const restrict in_base_p,
+			  const u8 *in_next,
+			  const u8 * const in_end,
+			  const u32 count,
+			  u32 * const restrict next_hashes)
 {
 	u32 cur_pos;
 	u32 hash3, hash4;
-	u32 next_seq3, next_seq4;
+	u32 next_hashseq;
 	u32 remaining = count;
 
 	if (unlikely(count + 5 > in_end - in_next))
-		return &in_next[count];
+		return;
 
 	cur_pos = in_next - *in_base_p;
 	hash3 = next_hashes[0];
@@ -374,10 +387,9 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 		mf->next_tab[cur_pos] = mf->hash4_tab[hash4];
 		mf->hash4_tab[hash4] = cur_pos;
 
-		next_seq4 = load_u32_unaligned(++in_next);
-		next_seq3 = loaded_u32_to_u24(next_seq4);
-		hash3 = lz_hash(next_seq3, HC_MATCHFINDER_HASH3_ORDER);
-		hash4 = lz_hash(next_seq4, HC_MATCHFINDER_HASH4_ORDER);
+		next_hashseq = get_unaligned_le32(++in_next);
+		hash3 = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+		hash4 = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
 		cur_pos++;
 	} while (--remaining);
 
@@ -385,6 +397,6 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 	prefetchw(&mf->hash4_tab[hash4]);
 	next_hashes[0] = hash3;
 	next_hashes[1] = hash4;
-
-	return in_next;
 }
+
+#endif /* LIB_HC_MATCHFINDER_H */

lib/ht_matchfinder.h

@@ -0,0 +1,234 @@
+/*
+ * ht_matchfinder.h - Lempel-Ziv matchfinding with a hash table
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ---------------------------------------------------------------------------
+ *
+ * This is a Hash Table (ht) matchfinder.
+ *
+ * This is a variant of the Hash Chains (hc) matchfinder that is optimized for
+ * very fast compression.  The ht_matchfinder stores the hash chains inline in
+ * the hash table, whereas the hc_matchfinder stores them in a separate array.
+ * Storing the hash chains inline is the faster method when max_search_depth
+ * (the maximum chain length) is very small.  It is not appropriate when
+ * max_search_depth is larger, as then it uses too much memory.
+ *
+ * Due to its focus on speed, the ht_matchfinder doesn't support length 3
+ * matches.  It also doesn't allow max_search_depth to vary at runtime; it is
+ * fixed at build time as HT_MATCHFINDER_BUCKET_SIZE.
+ *
+ * See hc_matchfinder.h for more information.
+ */
+
+#ifndef LIB_HT_MATCHFINDER_H
+#define LIB_HT_MATCHFINDER_H
+
+#include "matchfinder_common.h"
+
+#define HT_MATCHFINDER_HASH_ORDER	15
+#define HT_MATCHFINDER_BUCKET_SIZE	2
+
+#define HT_MATCHFINDER_MIN_MATCH_LEN	4
+/* Minimum value of max_len for ht_matchfinder_longest_match() */
+#define HT_MATCHFINDER_REQUIRED_NBYTES	5
+
+struct ht_matchfinder {
+	mf_pos_t hash_tab[1UL << HT_MATCHFINDER_HASH_ORDER]
+			 [HT_MATCHFINDER_BUCKET_SIZE];
+} MATCHFINDER_ALIGNED;
+
+static forceinline void
+ht_matchfinder_init(struct ht_matchfinder *mf)
+{
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, sizeof(*mf));
+}
+
+static forceinline void
+ht_matchfinder_slide_window(struct ht_matchfinder *mf)
+{
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
+}
+
+/* Note: max_len must be >= HT_MATCHFINDER_REQUIRED_NBYTES */
+static forceinline u32
+ht_matchfinder_longest_match(struct ht_matchfinder * const restrict mf,
+			     const u8 ** const restrict in_base_p,
+			     const u8 * const restrict in_next,
+			     const u32 max_len,
+			     const u32 nice_len,
+			     u32 * const restrict next_hash,
+			     u32 * const restrict offset_ret)
+{
+	u32 best_len = 0;
+	const u8 *best_matchptr = in_next;
+	u32 cur_pos = in_next - *in_base_p;
+	const u8 *in_base;
+	mf_pos_t cutoff;
+	u32 hash;
+	u32 seq;
+	mf_pos_t cur_node;
+	const u8 *matchptr;
+#if HT_MATCHFINDER_BUCKET_SIZE > 1
+	mf_pos_t to_insert;
+	u32 len;
+#endif
+#if HT_MATCHFINDER_BUCKET_SIZE > 2
+	int i;
+#endif
+
+	/* This is assumed throughout this function. */
+	STATIC_ASSERT(HT_MATCHFINDER_MIN_MATCH_LEN == 4);
+
+	if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
+		ht_matchfinder_slide_window(mf);
+		*in_base_p += MATCHFINDER_WINDOW_SIZE;
+		cur_pos = 0;
+	}
+	in_base = *in_base_p;
+	cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
+
+	hash = *next_hash;
+	STATIC_ASSERT(HT_MATCHFINDER_REQUIRED_NBYTES == 5);
+	*next_hash = lz_hash(get_unaligned_le32(in_next + 1),
+			     HT_MATCHFINDER_HASH_ORDER);
+	seq = load_u32_unaligned(in_next);
+	prefetchw(&mf->hash_tab[*next_hash]);
+#if HT_MATCHFINDER_BUCKET_SIZE == 1
+	/* Hand-unrolled version for BUCKET_SIZE == 1 */
+	cur_node = mf->hash_tab[hash][0];
+	mf->hash_tab[hash][0] = cur_pos;
+	if (cur_node <= cutoff)
+		goto out;
+	matchptr = &in_base[cur_node];
+	if (load_u32_unaligned(matchptr) == seq) {
+		best_len = lz_extend(in_next, matchptr, 4, max_len);
+		best_matchptr = matchptr;
+	}
+#elif HT_MATCHFINDER_BUCKET_SIZE == 2
+	/*
+	 * Hand-unrolled version for BUCKET_SIZE == 2.  The logic here also
+	 * differs slightly in that it copies the first entry to the second even
+	 * if nice_len is reached on the first, as this can be slightly faster.
+	 */
+	cur_node = mf->hash_tab[hash][0];
+	mf->hash_tab[hash][0] = cur_pos;
+	if (cur_node <= cutoff)
+		goto out;
+	matchptr = &in_base[cur_node];
+
+	to_insert = cur_node;
+	cur_node = mf->hash_tab[hash][1];
+	mf->hash_tab[hash][1] = to_insert;
+
+	if (load_u32_unaligned(matchptr) == seq) {
+		best_len = lz_extend(in_next, matchptr, 4, max_len);
+		best_matchptr = matchptr;
+		if (cur_node <= cutoff || best_len >= nice_len)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq &&
+		    load_u32_unaligned(matchptr + best_len - 3) ==
+		    load_u32_unaligned(in_next + best_len - 3)) {
+			len = lz_extend(in_next, matchptr, 4, max_len);
+			if (len > best_len) {
+				best_len = len;
+				best_matchptr = matchptr;
+			}
+		}
+	} else {
+		if (cur_node <= cutoff)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq) {
+			best_len = lz_extend(in_next, matchptr, 4, max_len);
+			best_matchptr = matchptr;
+		}
+	}
+#else
+	/* Generic version for HT_MATCHFINDER_BUCKET_SIZE > 2 */
+	to_insert = cur_pos;
+	for (i = 0; i < HT_MATCHFINDER_BUCKET_SIZE; i++) {
+		cur_node = mf->hash_tab[hash][i];
+		mf->hash_tab[hash][i] = to_insert;
+		if (cur_node <= cutoff)
+			goto out;
+		matchptr = &in_base[cur_node];
+		if (load_u32_unaligned(matchptr) == seq) {
+			len = lz_extend(in_next, matchptr, 4, max_len);
+			if (len > best_len) {
+				best_len = len;
+				best_matchptr = matchptr;
+				if (best_len >= nice_len)
+					goto out;
+			}
+		}
+		to_insert = cur_node;
+	}
+#endif
+out:
+	*offset_ret = in_next - best_matchptr;
+	return best_len;
+}
+
+static forceinline void
+ht_matchfinder_skip_bytes(struct ht_matchfinder * const restrict mf,
+			  const u8 ** const restrict in_base_p,
+			  const u8 *in_next,
+			  const u8 * const in_end,
+			  const u32 count,
+			  u32 * const restrict next_hash)
+{
+	s32 cur_pos = in_next - *in_base_p;
+	u32 hash;
+	u32 remaining = count;
+	int i;
+
+	if (unlikely(count + HT_MATCHFINDER_REQUIRED_NBYTES > in_end - in_next))
+		return;
+
+	if (cur_pos + count - 1 >= MATCHFINDER_WINDOW_SIZE) {
+		ht_matchfinder_slide_window(mf);
+		*in_base_p += MATCHFINDER_WINDOW_SIZE;
+		cur_pos -= MATCHFINDER_WINDOW_SIZE;
+	}
+
+	hash = *next_hash;
+	do {
+		for (i = HT_MATCHFINDER_BUCKET_SIZE - 1; i > 0; i--)
+			mf->hash_tab[hash][i] = mf->hash_tab[hash][i - 1];
+		mf->hash_tab[hash][0] = cur_pos;
+
+		hash = lz_hash(get_unaligned_le32(++in_next),
+			       HT_MATCHFINDER_HASH_ORDER);
+		cur_pos++;
+	} while (--remaining);
+
+	prefetchw(&mf->hash_tab[hash]);
+	*next_hash = hash;
+}
+
+#endif /* LIB_HT_MATCHFINDER_H */

lib/lib_common.h

@@ -0,0 +1,67 @@
+/*
+ * lib_common.h - internal header included by all library code
+ */
+
+#ifndef LIB_LIB_COMMON_H
+#define LIB_LIB_COMMON_H
+
+#ifdef LIBDEFLATE_H
+#  error "lib_common.h must always be included before libdeflate.h"
+   /* because BUILDING_LIBDEFLATE must be set first */
+#endif
+
+#define BUILDING_LIBDEFLATE
+
+#include "../common/common_defs.h"
+
+/*
+ * Prefix with "_libdeflate_" all global symbols which are not part of the API
+ * and don't already have a "libdeflate" prefix.  This avoids exposing overly
+ * generic names when libdeflate is built as a static library.
+ *
+ * Note that the chosen prefix is not really important and can be changed
+ * without breaking library users.  It was just chosen so that the resulting
+ * symbol names are unlikely to conflict with those from any other software.
+ * Also note that this fixup has no useful effect when libdeflate is built as a
+ * shared library, since these symbols are not exported.
+ */
+#define SYM_FIXUP(sym)			_libdeflate_##sym
+#define deflate_get_compression_level	SYM_FIXUP(deflate_get_compression_level)
+#define _cpu_features			SYM_FIXUP(_cpu_features)
+#define setup_cpu_features		SYM_FIXUP(setup_cpu_features)
+
+void *libdeflate_malloc(size_t size);
+void libdeflate_free(void *ptr);
+
+void *libdeflate_aligned_malloc(size_t alignment, size_t size);
+void libdeflate_aligned_free(void *ptr);
+
+#ifdef FREESTANDING
+/*
+ * With -ffreestanding, <string.h> may be missing, and we must provide
+ * implementations of memset(), memcpy(), memmove(), and memcmp().
+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+ *
+ * Also, -ffreestanding disables interpreting calls to these functions as
+ * built-ins.  E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
+ * not be optimized to a single load instruction.  For performance reasons we
+ * don't want that.  So, declare these functions as macros that expand to the
+ * corresponding built-ins.  This approach is recommended in the gcc man page.
+ * We still need the actual function definitions in case gcc calls them.
+ */
+void *memset(void *s, int c, size_t n);
+#define memset(s, c, n)		__builtin_memset((s), (c), (n))
+
+void *memcpy(void *dest, const void *src, size_t n);
+#define memcpy(dest, src, n)	__builtin_memcpy((dest), (src), (n))
+
+void *memmove(void *dest, const void *src, size_t n);
+#define memmove(dest, src, n)	__builtin_memmove((dest), (src), (n))
+
+int memcmp(const void *s1, const void *s2, size_t n);
+#define memcmp(s1, s2, n)	__builtin_memcmp((s1), (s2), (n))
+#else
+#include <string.h>
+#endif
+
+#endif /* LIB_LIB_COMMON_H */

lib/matchfinder_avx2.h

@@ -1,53 +0,0 @@
-/*
- * matchfinder_avx2.h - matchfinding routines optimized for Intel AVX2 (Advanced
- * Vector Extensions)
- */
-
-#include <immintrin.h>
-
-static forceinline bool
-matchfinder_init_avx2(mf_pos_t *data, size_t size)
-{
-	__m256i v, *p;
-	size_t n;
-
-	if (size % sizeof(__m256i) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
-{
-	__m256i v, *p;
-	size_t n;
-
-	if ((size % sizeof(__m256i) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
-	do {
-		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
-		p[0] = _mm256_adds_epi16(p[0], v);
-		p[1] = _mm256_adds_epi16(p[1], v);
-		p[2] = _mm256_adds_epi16(p[2], v);
-		p[3] = _mm256_adds_epi16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/matchfinder_common.h

@@ -2,10 +2,10 @@
  * matchfinder_common.h - common code for Lempel-Ziv matchfinding
  */
 
-#ifndef _LIB_MATCHFINDER_COMMON_H
-#define _LIB_MATCHFINDER_COMMON_H
+#ifndef LIB_MATCHFINDER_COMMON_H
+#define LIB_MATCHFINDER_COMMON_H
 
-#include "common_defs.h"
+#include "lib_common.h"
 #include "unaligned.h"
 
 #ifndef MATCHFINDER_WINDOW_ORDER
@@ -18,30 +18,24 @@ typedef s16 mf_pos_t;
 
 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
 
-#define MATCHFINDER_ALIGNMENT 8
+/*
+ * Required alignment of the matchfinder buffer pointer and size.  The values
+ * here come from the AVX-2 implementation, which is the worst case.
+ */
+#define MATCHFINDER_MEM_ALIGNMENT	32
+#define MATCHFINDER_SIZE_ALIGNMENT	128
 
-#ifdef __AVX2__
-#  include "matchfinder_avx2.h"
-#  if MATCHFINDER_ALIGNMENT < 32
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 32
-#  endif
-#endif
-
-#ifdef __SSE2__
-#  include "matchfinder_sse2.h"
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
-#  endif
-#endif
-
-#ifdef __ARM_NEON__
-#  include "matchfinder_neon.h"
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
+#undef matchfinder_init
+#undef matchfinder_rebase
+#ifdef _aligned_attribute
+#  define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
+#  if defined(__arm__) || defined(__aarch64__)
+#    include "arm/matchfinder_impl.h"
+#  elif defined(__i386__) || defined(__x86_64__)
+#    include "x86/matchfinder_impl.h"
 #  endif
+#else
+#  define MATCHFINDER_ALIGNED
 #endif
 
 /*
@@ -49,31 +43,20 @@ typedef s16 mf_pos_t;
  *
  * Essentially, this is an optimized memset().
  *
- * 'data' must be aligned to a MATCHFINDER_ALIGNMENT boundary.
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_init
 static forceinline void
-matchfinder_init(mf_pos_t *data, size_t num_entries)
+matchfinder_init(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-#if defined(__AVX2__) && defined(_aligned_attribute)
-	if (matchfinder_init_avx2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__SSE2__) && defined(_aligned_attribute)
-	if (matchfinder_init_sse2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__ARM_NEON__) && defined(_aligned_attribute)
-	if (matchfinder_init_neon(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
 	for (i = 0; i < num_entries; i++)
 		data[i] = MATCHFINDER_INITVAL;
 }
+#endif
 
 /*
  * Slide the matchfinder by WINDOW_SIZE bytes.
@@ -95,27 +78,17 @@ matchfinder_init(mf_pos_t *data, size_t num_entries)
  * hash bucket.  Note that in the latter table, the links are 1-ary in the case
  * of "hash chains", and 2-ary in the case of "binary trees".  In either case,
  * the links need to be rebased in the same way.
+ *
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_rebase
 static forceinline void
-matchfinder_rebase(mf_pos_t *data, size_t num_entries)
+matchfinder_rebase(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-#if defined(__AVX2__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_avx2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__SSE2__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_sse2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__ARM_NEON__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_neon(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
 	if (MATCHFINDER_WINDOW_SIZE == 32768) {
 		/* Branchless version for 32768 byte windows.  If the value was
 		 * already negative, clear all bits except the sign bit; this
@@ -138,6 +111,7 @@ matchfinder_rebase(mf_pos_t *data, size_t num_entries)
 			data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
 	}
 }
+#endif
 
 /*
  * The hash function: given a sequence prefix held in the low-order bits of a
@@ -202,4 +176,4 @@ word_differs:
 	return len;
 }
 
-#endif /* _LIB_MATCHFINDER_COMMON_H */
+#endif /* LIB_MATCHFINDER_COMMON_H */

lib/matchfinder_neon.h

@@ -1,61 +0,0 @@
-/*
- * matchfinder_neon.h - matchfinding routines optimized for ARM NEON (Advanced
- * SIMD) instructions
- */
-
-#include <arm_neon.h>
-
-static forceinline bool
-matchfinder_init_neon(mf_pos_t *data, size_t size)
-{
-	int16x8_t v, *p;
-	size_t n;
-
-	if (size % sizeof(int16x8_t) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_neon(mf_pos_t *data, size_t size)
-{
-	int16x8_t v, *p;
-	size_t n;
-
-	if ((size % sizeof(int16x8_t) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
-	do {
-		p[0] = vqaddq_s16(p[0], v);
-		p[1] = vqaddq_s16(p[1], v);
-		p[2] = vqaddq_s16(p[2], v);
-		p[3] = vqaddq_s16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/matchfinder_sse2.h

@@ -1,53 +0,0 @@
-/*
- * matchfinder_sse2.h - matchfinding routines optimized for Intel SSE2
- * (Streaming SIMD Extensions).
- */
-
-#include <emmintrin.h>
-
-static forceinline bool
-matchfinder_init_sse2(mf_pos_t *data, size_t size)
-{
-	__m128i v, *p;
-	size_t n;
-
-	if (size % sizeof(__m128i) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
-{
-	__m128i v, *p;
-	size_t n;
-
-	if ((size % sizeof(__m128i) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
-	do {
-		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
-		p[0] = _mm_adds_epi16(p[0], v);
-		p[1] = _mm_adds_epi16(p[1], v);
-		p[2] = _mm_adds_epi16(p[2], v);
-		p[3] = _mm_adds_epi16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/unaligned.h

@@ -5,15 +5,41 @@
 #ifndef LIB_UNALIGNED_H
 #define LIB_UNALIGNED_H
 
-#include "common_defs.h"
+#include "lib_common.h"
+
+/***** Unaligned loads and stores without endianness conversion *****/
 
 /*
- * Naming note:
+ * memcpy() is portable, and it usually gets optimized appropriately by modern
+ * compilers.  I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled
+ * to a load or store instruction, not to an actual function call.
  *
- * {load,store}_*_unaligned() deal with raw bytes without endianness conversion.
- * {get,put}_unaligned_*() deal with a specific endianness.
+ * We no longer use the "packed struct" approach, as that is nonstandard, has
+ * unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366).  However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
  */
 
+#define DEFINE_UNALIGNED_TYPE(type)				\
+static forceinline type						\
+load_##type##_unaligned(const void *p)				\
+{								\
+	type v;							\
+	memcpy(&v, p, sizeof(v));				\
+	return v;						\
+}								\
+								\
+static forceinline void						\
+store_##type##_unaligned(type v, void *p)			\
+{								\
+	memcpy(p, &v, sizeof(v));				\
+}
+
 DEFINE_UNALIGNED_TYPE(u16)
 DEFINE_UNALIGNED_TYPE(u32)
 DEFINE_UNALIGNED_TYPE(u64)
@@ -22,7 +48,7 @@ DEFINE_UNALIGNED_TYPE(machine_word_t)
 #define load_word_unaligned	load_machine_word_t_unaligned
 #define store_word_unaligned	store_machine_word_t_unaligned
 
-/***** Unaligned loads  *****/
+/***** Unaligned loads with endianness conversion *****/
 
 static forceinline u16
 get_unaligned_le16(const u8 *p)
@@ -84,7 +110,7 @@ get_unaligned_leword(const u8 *p)
 		return get_unaligned_le64(p);
 }
 
-/***** Unaligned stores  *****/
+/***** Unaligned stores with endianness conversion *****/
 
 static forceinline void
 put_unaligned_le16(u16 v, u8 *p)

lib/utils.c

@@ -0,0 +1,142 @@
+/*
+ * utils.c - utility functions for libdeflate
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+#include "libdeflate.h"
+
+#ifdef FREESTANDING
+#  define malloc NULL
+#  define free NULL
+#else
+#  include <stdlib.h>
+#endif
+
+static void *(*libdeflate_malloc_func)(size_t) = malloc;
+static void (*libdeflate_free_func)(void *) = free;
+
+void *
+libdeflate_malloc(size_t size)
+{
+	return (*libdeflate_malloc_func)(size);
+}
+
+void
+libdeflate_free(void *ptr)
+{
+	(*libdeflate_free_func)(ptr);
+}
+
+void *
+libdeflate_aligned_malloc(size_t alignment, size_t size)
+{
+	void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size);
+	if (ptr) {
+		void *orig_ptr = ptr;
+		ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
+		((void **)ptr)[-1] = orig_ptr;
+	}
+	return ptr;
+}
+
+void
+libdeflate_aligned_free(void *ptr)
+{
+	if (ptr)
+		libdeflate_free(((void **)ptr)[-1]);
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *))
+{
+	libdeflate_malloc_func = malloc_func;
+	libdeflate_free_func = free_func;
+}
+
+/*
+ * Implementations of libc functions for freestanding library builds.
+ * Normal library builds don't use these.  Not optimized yet; usually the
+ * compiler expands these functions and doesn't actually call them anyway.
+ */
+#ifdef FREESTANDING
+#undef memset
+void * __attribute__((weak))
+memset(void *s, int c, size_t n)
+{
+	u8 *p = s;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		p[i] = c;
+	return s;
+}
+
+#undef memcpy
+void * __attribute__((weak))
+memcpy(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		d[i] = s[i];
+	return dest;
+}
+
+#undef memmove
+void * __attribute__((weak))
+memmove(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	if (d <= s)
+		return memcpy(d, s, n);
+
+	for (i = n; i > 0; i--)
+		d[i - 1] = s[i - 1];
+	return dest;
+}
+
+#undef memcmp
+int __attribute__((weak))
+memcmp(const void *s1, const void *s2, size_t n)
+{
+	const u8 *p1 = s1;
+	const u8 *p2 = s2;
+	size_t i;
+
+	for (i = 0; i < n; i++) {
+		if (p1[i] != p2[i])
+			return (int)p1[i] - (int)p2[i];
+	}
+	return 0;
+}
+#endif /* FREESTANDING */

lib/x86/adler32_impl.h

@@ -0,0 +1,337 @@
+/*
+ * x86/adler32_impl.h - x86 implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_ADLER32_IMPL_H
+#define LIB_X86_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * The following macros horizontally sum the s1 counters and add them to the
+ * real s1, and likewise for s2.  They do this via a series of reductions, each
+ * of which halves the vector length, until just one counter remains.
+ *
+ * The s1 reductions don't depend on the s2 reductions and vice versa, so for
+ * efficiency they are interleaved.  Also, every other s1 counter is 0 due to
+ * the 'psadbw' instruction (_mm_sad_epu8) summing groups of 8 bytes rather than
+ * 4; hence, one of the s1 reductions is skipped when going from 128 => 32 bits.
+ */
+
+#define ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_last = (v_s1), s2_last = (v_s2);			    \
+									    \
+	/* 128 => 32 bits */						    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x31);	    \
+	s1_last += (__v4su)_mm_shuffle_epi32((__m128i)s1_last, 0x02);	    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x02);	    \
+									    \
+	*(s1) += (u32)_mm_cvtsi128_si32((__m128i)s1_last);		    \
+	*(s2) += (u32)_mm_cvtsi128_si32((__m128i)s2_last);		    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_128bit, s2_128bit;					    \
+									    \
+	/* 256 => 128 bits */						    \
+	s1_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 1);   \
+	s2_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 1);   \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_128((s1), (s2), s1_128bit, s2_128bit);	    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v8su s1_256bit, s2_256bit;					    \
+									    \
+	/* 512 => 256 bits */						    \
+	s1_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 1);  \
+	s2_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 1);  \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_256((s1), (s2), s1_256bit, s2_256bit);	    \
+}
+
+/* AVX-512BW implementation: like the AVX2 one, but does 64 bytes at a time */
+#undef DISPATCH_AVX512BW
+#if !defined(DEFAULT_IMPL) &&						\
+    /*
+     * clang before v3.9 is missing some AVX-512BW intrinsics including
+     * _mm512_sad_epu8(), a.k.a. __builtin_ia32_psadbw512.  So just make using
+     * AVX-512BW, even when __AVX512BW__ is defined, conditional on
+     * COMPILER_SUPPORTS_AVX512BW_TARGET where we check for that builtin.
+     */									\
+    COMPILER_SUPPORTS_AVX512BW_TARGET &&				\
+    (defined(__AVX512BW__) || (X86_CPU_FEATURES_ENABLED &&		\
+			       COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_avx512bw
+#  define FUNCNAME_CHUNK	adler32_avx512bw_chunk
+#  define IMPL_ALIGNMENT	64
+#  define IMPL_SEGMENT_SIZE	64
+#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
+#  ifdef __AVX512BW__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_avx512bw
+#  else
+#    define ATTRIBUTES		__attribute__((target("avx512bw")))
+#    define DISPATCH		1
+#    define DISPATCH_AVX512BW	1
+#  endif
+#  include <immintrin.h>
+static forceinline ATTRIBUTES void
+adler32_avx512bw_chunk(const __m512i *p, const __m512i *const end,
+		       u32 *s1, u32 *s2)
+{
+	const __m512i zeroes = _mm512_setzero_si512();
+	const __v64qi multipliers = (__v64qi){
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v32hi ones = (__v32hi)_mm512_set1_epi16(1);
+	__v16si v_s1 = (__v16si)zeroes;
+	__v16si v_s1_sums = (__v16si)zeroes;
+	__v16si v_s2 = (__v16si)zeroes;
+
+	do {
+		/* Load the next 64-byte segment */
+		__m512i bytes = *p++;
+
+		/* Multiply the bytes by 64...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v32hi sums = (__v32hi)_mm512_maddubs_epi16(
+						bytes, (__m512i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 64 to the end. */
+		v_s1_sums += v_s1;
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v16si)_mm512_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 64...1 to the corresponding s2 counter */
+		v_s2 += (__v16si)_mm512_madd_epi16((__m512i)sums,
+						   (__m512i)ones);
+	} while (p != end);
+
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (64) */
+	v_s2 += (__v16si)_mm512_slli_epi32((__m512i)v_s1_sums, 6);
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* AVX-512BW implementation */
+
+/* AVX2 implementation: like the AVX-512BW one, but does 32 bytes at a time */
+#undef DISPATCH_AVX2
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__AVX2__) || (X86_CPU_FEATURES_ENABLED &&	\
+			       COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_avx2
+#  define FUNCNAME_CHUNK	adler32_avx2_chunk
+#  define IMPL_ALIGNMENT	32
+#  define IMPL_SEGMENT_SIZE	32
+#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
+#  ifdef __AVX2__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_avx2
+#  else
+#    define ATTRIBUTES		__attribute__((target("avx2")))
+#    define DISPATCH		1
+#    define DISPATCH_AVX2	1
+#  endif
+#  include <immintrin.h>
+static forceinline ATTRIBUTES void
+adler32_avx2_chunk(const __m256i *p, const __m256i *const end, u32 *s1, u32 *s2)
+{
+	const __m256i zeroes = _mm256_setzero_si256();
+	const __v32qu multipliers = (__v32qu){
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v16hu ones = (__v16hu)_mm256_set1_epi16(1);
+	__v8su v_s1 = (__v8su)zeroes;
+	__v8su v_s1_sums = (__v8su)zeroes;
+	__v8su v_s2 = (__v8su)zeroes;
+
+	do {
+		/* Load the next 32-byte segment */
+		__m256i bytes = *p++;
+
+		/* Multiply the bytes by 32...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v16hu sums = (__v16hu)_mm256_maddubs_epi16(
+						bytes, (__m256i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 32 to the end. */
+		v_s1_sums += v_s1;
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v8su)_mm256_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 32...1 to the corresponding s2 counter */
+		v_s2 += (__v8su)_mm256_madd_epi16((__m256i)sums, (__m256i)ones);
+	} while (p != end);
+
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (32) */
+	v_s2 += (__v8su)_mm256_slli_epi32((__m256i)v_s1_sums, 5);
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* AVX2 implementation */
+
+/* SSE2 implementation */
+#undef DISPATCH_SSE2
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__SSE2__) || (X86_CPU_FEATURES_ENABLED &&	\
+			       COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_sse2
+#  define FUNCNAME_CHUNK	adler32_sse2_chunk
+#  define IMPL_ALIGNMENT	16
+#  define IMPL_SEGMENT_SIZE	32
+/*
+ * The 16-bit precision byte counters must not be allowed to undergo *signed*
+ * overflow, otherwise the signed multiplications at the end (_mm_madd_epi16)
+ * would behave incorrectly.
+ */
+#  define IMPL_MAX_CHUNK_SIZE	(32 * (0x7FFF / 0xFF))
+#  ifdef __SSE2__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_sse2
+#  else
+#    define ATTRIBUTES		__attribute__((target("sse2")))
+#    define DISPATCH		1
+#    define DISPATCH_SSE2	1
+#  endif
+#  include <emmintrin.h>
+static forceinline ATTRIBUTES void
+adler32_sse2_chunk(const __m128i *p, const __m128i *const end, u32 *s1, u32 *s2)
+{
+	const __m128i zeroes = _mm_setzero_si128();
+
+	/* s1 counters: 32-bit, sum of bytes */
+	__v4su v_s1 = (__v4su)zeroes;
+
+	/* s2 counters: 32-bit, sum of s1 values */
+	__v4su v_s2 = (__v4su)zeroes;
+
+	/*
+	 * Thirty-two 16-bit counters for byte sums.  Each accumulates the bytes
+	 * that eventually need to be multiplied by a number 32...1 for addition
+	 * into s2.
+	 */
+	__v8hu v_byte_sums_a = (__v8hu)zeroes;
+	__v8hu v_byte_sums_b = (__v8hu)zeroes;
+	__v8hu v_byte_sums_c = (__v8hu)zeroes;
+	__v8hu v_byte_sums_d = (__v8hu)zeroes;
+
+	do {
+		/* Load the next 32 bytes */
+		const __m128i bytes1 = *p++;
+		const __m128i bytes2 = *p++;
+
+		/*
+		 * Accumulate the previous s1 counters into the s2 counters.
+		 * Logically, this really should be v_s2 += v_s1 * 32, but we
+		 * can do the multiplication (or left shift) later.
+		 */
+		v_s2 += v_s1;
+
+		/*
+		 * s1 update: use "Packed Sum of Absolute Differences" to add
+		 * the bytes horizontally with 8 bytes per sum.  Then add the
+		 * sums to the s1 counters.
+		 */
+		v_s1 += (__v4su)_mm_sad_epu8(bytes1, zeroes);
+		v_s1 += (__v4su)_mm_sad_epu8(bytes2, zeroes);
+
+		/*
+		 * Also accumulate the bytes into 32 separate counters that have
+		 * 16-bit precision.
+		 */
+		v_byte_sums_a += (__v8hu)_mm_unpacklo_epi8(bytes1, zeroes);
+		v_byte_sums_b += (__v8hu)_mm_unpackhi_epi8(bytes1, zeroes);
+		v_byte_sums_c += (__v8hu)_mm_unpacklo_epi8(bytes2, zeroes);
+		v_byte_sums_d += (__v8hu)_mm_unpackhi_epi8(bytes2, zeroes);
+
+	} while (p != end);
+
+	/* Finish calculating the s2 counters */
+	v_s2 = (__v4su)_mm_slli_epi32((__m128i)v_s2, 5);
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_a,
+				       (__m128i)(__v8hu){ 32, 31, 30, 29, 28, 27, 26, 25 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_b,
+				       (__m128i)(__v8hu){ 24, 23, 22, 21, 20, 19, 18, 17 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_c,
+				       (__m128i)(__v8hu){ 16, 15, 14, 13, 12, 11, 10, 9 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_d,
+				       (__m128i)(__v8hu){ 8,  7,  6,  5,  4,  3,  2,  1 });
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* SSE2 implementation */
+
+#ifdef DISPATCH
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_AVX512BW
+	if (features & X86_CPU_FEATURE_AVX512BW)
+		return adler32_avx512bw;
+#endif
+#ifdef DISPATCH_AVX2
+	if (features & X86_CPU_FEATURE_AVX2)
+		return adler32_avx2;
+#endif
+#ifdef DISPATCH_SSE2
+	if (features & X86_CPU_FEATURE_SSE2)
+		return adler32_sse2;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_ADLER32_IMPL_H */

lib/x86/cpu_features.c

@@ -0,0 +1,152 @@
+/*
+ * x86/cpu_features.c - feature detection for x86 processors
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#if X86_CPU_FEATURES_ENABLED
+
+volatile u32 _cpu_features = 0;
+
+/* With old GCC versions we have to manually save and restore the x86_32 PIC
+ * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
+#if defined(__i386__) && defined(__PIC__)
+#  define EBX_CONSTRAINT "=&r"
+#else
+#  define EBX_CONSTRAINT "=b"
+#endif
+
+/* Execute the CPUID instruction.  */
+static inline void
+cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
+{
+	__asm__(".ifnc %%ebx, %1; mov  %%ebx, %1; .endif\n"
+		"cpuid                                  \n"
+		".ifnc %%ebx, %1; xchg %%ebx, %1; .endif\n"
+		: "=a" (*a), EBX_CONSTRAINT (*b), "=c" (*c), "=d" (*d)
+		: "a" (leaf), "c" (subleaf));
+}
+
+/* Read an extended control register.  */
+static inline u64
+read_xcr(u32 index)
+{
+	u32 edx, eax;
+
+	/* Execute the "xgetbv" instruction.  Old versions of binutils do not
+	 * recognize this instruction, so list the raw bytes instead.  */
+	__asm__ (".byte 0x0f, 0x01, 0xd0" : "=d" (edx), "=a" (eax) : "c" (index));
+
+	return ((u64)edx << 32) | eax;
+}
+
+#undef BIT
+#define BIT(nr)			(1UL << (nr))
+
+#define XCR0_BIT_SSE		BIT(1)
+#define XCR0_BIT_AVX		BIT(2)
+#define XCR0_BIT_OPMASK		BIT(5)
+#define XCR0_BIT_ZMM_HI256	BIT(6)
+#define XCR0_BIT_HI16_ZMM	BIT(7)
+
+#define IS_SET(reg, nr)		((reg) & BIT(nr))
+#define IS_ALL_SET(reg, mask)	(((reg) & (mask)) == (mask))
+
+static const struct cpu_feature x86_cpu_feature_table[] = {
+	{X86_CPU_FEATURE_SSE2,		"sse2"},
+	{X86_CPU_FEATURE_PCLMUL,	"pclmul"},
+	{X86_CPU_FEATURE_AVX,		"avx"},
+	{X86_CPU_FEATURE_AVX2,		"avx2"},
+	{X86_CPU_FEATURE_BMI2,		"bmi2"},
+	{X86_CPU_FEATURE_AVX512BW,	"avx512bw"},
+};
+
+/* Initialize _cpu_features with bits for interesting processor features. */
+void setup_cpu_features(void)
+{
+	u32 features = 0;
+	u32 dummy1, dummy2, dummy3, dummy4;
+	u32 max_function;
+	u32 features_1, features_2, features_3, features_4;
+	bool os_avx_support = false;
+	bool os_avx512_support = false;
+
+	/* Get maximum supported function  */
+	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
+	if (max_function < 1)
+		goto out;
+
+	/* Standard feature flags  */
+	cpuid(1, 0, &dummy1, &dummy2, &features_2, &features_1);
+
+	if (IS_SET(features_1, 26))
+		features |= X86_CPU_FEATURE_SSE2;
+
+	if (IS_SET(features_2, 1))
+		features |= X86_CPU_FEATURE_PCLMUL;
+
+	if (IS_SET(features_2, 27)) { /* OSXSAVE set? */
+		u64 xcr0 = read_xcr(0);
+
+		os_avx_support = IS_ALL_SET(xcr0,
+					    XCR0_BIT_SSE |
+					    XCR0_BIT_AVX);
+
+		os_avx512_support = IS_ALL_SET(xcr0,
+					       XCR0_BIT_SSE |
+					       XCR0_BIT_AVX |
+					       XCR0_BIT_OPMASK |
+					       XCR0_BIT_ZMM_HI256 |
+					       XCR0_BIT_HI16_ZMM);
+	}
+
+	if (os_avx_support && IS_SET(features_2, 28))
+		features |= X86_CPU_FEATURE_AVX;
+
+	if (max_function < 7)
+		goto out;
+
+	/* Extended feature flags  */
+	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
+
+	if (os_avx_support && IS_SET(features_3, 5))
+		features |= X86_CPU_FEATURE_AVX2;
+
+	if (IS_SET(features_3, 8))
+		features |= X86_CPU_FEATURE_BMI2;
+
+	if (os_avx512_support && IS_SET(features_3, 30))
+		features |= X86_CPU_FEATURE_AVX512BW;
+
+out:
+	disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
+					 ARRAY_LEN(x86_cpu_feature_table));
+
+	_cpu_features = features | X86_CPU_FEATURES_KNOWN;
+}
+
+#endif /* X86_CPU_FEATURES_ENABLED */

lib/x86/cpu_features.h

@@ -0,0 +1,41 @@
+/*
+ * x86/cpu_features.h - feature detection for x86 processors
+ */
+
+#ifndef LIB_X86_CPU_FEATURES_H
+#define LIB_X86_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if (defined(__i386__) || defined(__x86_64__)) && \
+	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+#  define X86_CPU_FEATURES_ENABLED 1
+#else
+#  define X86_CPU_FEATURES_ENABLED 0
+#endif
+
+#if X86_CPU_FEATURES_ENABLED
+
+#define X86_CPU_FEATURE_SSE2		0x00000001
+#define X86_CPU_FEATURE_PCLMUL		0x00000002
+#define X86_CPU_FEATURE_AVX		0x00000004
+#define X86_CPU_FEATURE_AVX2		0x00000008
+#define X86_CPU_FEATURE_BMI2		0x00000010
+#define X86_CPU_FEATURE_AVX512BW	0x00000020
+
+#define X86_CPU_FEATURES_KNOWN		0x80000000
+
+extern volatile u32 _cpu_features;
+
+void setup_cpu_features(void);
+
+static inline u32 get_cpu_features(void)
+{
+	if (_cpu_features == 0)
+		setup_cpu_features();
+	return _cpu_features;
+}
+
+#endif /* X86_CPU_FEATURES_ENABLED */
+
+#endif /* LIB_X86_CPU_FEATURES_H */

lib/x86/crc32_impl.h

@@ -0,0 +1,92 @@
+/*
+ * x86/crc32_impl.h - x86 implementations of CRC-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_CRC32_IMPL_H
+#define LIB_X86_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * Include the PCLMUL/AVX implementation?  Although our PCLMUL-optimized CRC-32
+ * function doesn't use any AVX intrinsics specifically, it can benefit a lot
+ * from being compiled for an AVX target: on Skylake, ~16700 MB/s vs. ~10100
+ * MB/s.  I expect this is related to the PCLMULQDQ instructions being assembled
+ * in the newer three-operand form rather than the older two-operand form.
+ *
+ * Note: this is only needed if __AVX__ is *not* defined, since otherwise the
+ * "regular" PCLMUL implementation would already be AVX enabled.
+ */
+#undef DISPATCH_PCLMUL_AVX
+#if !defined(DEFAULT_IMPL) && !defined(__AVX__) &&	\
+	X86_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_AVX_TARGET &&	\
+	(defined(__PCLMUL__) || COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS)
+#  define FUNCNAME		crc32_pclmul_avx
+#  define FUNCNAME_ALIGNED	crc32_pclmul_avx_aligned
+#  define ATTRIBUTES		__attribute__((target("pclmul,avx")))
+#  define DISPATCH		1
+#  define DISPATCH_PCLMUL_AVX	1
+#  include "crc32_pclmul_template.h"
+#endif
+
+/* PCLMUL implementation */
+#undef DISPATCH_PCLMUL
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__PCLMUL__) || (X86_CPU_FEATURES_ENABLED &&	\
+				 COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS))
+#  define FUNCNAME		crc32_pclmul
+#  define FUNCNAME_ALIGNED	crc32_pclmul_aligned
+#  ifdef __PCLMUL__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_pclmul
+#  else
+#    define ATTRIBUTES		__attribute__((target("pclmul")))
+#    define DISPATCH		1
+#    define DISPATCH_PCLMUL	1
+#  endif
+#  include "crc32_pclmul_template.h"
+#endif
+
+#ifdef DISPATCH
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_PCLMUL_AVX
+	if ((features & X86_CPU_FEATURE_PCLMUL) &&
+	    (features & X86_CPU_FEATURE_AVX))
+		return crc32_pclmul_avx;
+#endif
+#ifdef DISPATCH_PCLMUL
+	if (features & X86_CPU_FEATURE_PCLMUL)
+		return crc32_pclmul;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_CRC32_IMPL_H */

lib/x86/crc32_pclmul_template.h

@@ -0,0 +1,262 @@
+/*
+ * x86/crc32_pclmul_template.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <immintrin.h>
+
+/*
+ * CRC-32 folding with PCLMULQDQ.
+ *
+ * The basic idea is to repeatedly "fold" each 512 bits into the next 512 bits,
+ * producing an abbreviated message which is congruent the original message
+ * modulo the generator polynomial G(x).
+ *
+ * Folding each 512 bits is implemented as eight 64-bit folds, each of which
+ * uses one carryless multiplication instruction.  It's expected that CPUs may
+ * be able to execute some of these multiplications in parallel.
+ *
+ * Explanation of "folding": let A(x) be 64 bits from the message, and let B(x)
+ * be 95 bits from a constant distance D later in the message.  The relevant
+ * portion of the message can be written as:
+ *
+ *	M(x) = A(x)*x^D + B(x)
+ *
+ * ... where + and * represent addition and multiplication, respectively, of
+ * polynomials over GF(2).  Note that when implemented on a computer, these
+ * operations are equivalent to XOR and carryless multiplication, respectively.
+ *
+ * For the purpose of CRC calculation, only the remainder modulo the generator
+ * polynomial G(x) matters:
+ *
+ *	M(x) mod G(x) = (A(x)*x^D + B(x)) mod G(x)
+ *
+ * Since the modulo operation can be applied anywhere in a sequence of additions
+ * and multiplications without affecting the result, this is equivalent to:
+ *
+ *	M(x) mod G(x) = (A(x)*(x^D mod G(x)) + B(x)) mod G(x)
+ *
+ * For any D, 'x^D mod G(x)' will be a polynomial with maximum degree 31, i.e.
+ * a 32-bit quantity.  So 'A(x) * (x^D mod G(x))' is equivalent to a carryless
+ * multiplication of a 64-bit quantity by a 32-bit quantity, producing a 95-bit
+ * product.  Then, adding (XOR-ing) the product to B(x) produces a polynomial
+ * with the same length as B(x) but with the same remainder as 'A(x)*x^D +
+ * B(x)'.  This is the basic fold operation with 64 bits.
+ *
+ * Note that the carryless multiplication instruction PCLMULQDQ actually takes
+ * two 64-bit inputs and produces a 127-bit product in the low-order bits of a
+ * 128-bit XMM register.  This works fine, but care must be taken to account for
+ * "bit endianness".  With the CRC version implemented here, bits are always
+ * ordered such that the lowest-order bit represents the coefficient of highest
+ * power of x and the highest-order bit represents the coefficient of the lowest
+ * power of x.  This is backwards from the more intuitive order.  Still,
+ * carryless multiplication works essentially the same either way.  It just must
+ * be accounted for that when we XOR the 95-bit product in the low-order 95 bits
+ * of a 128-bit XMM register into 128-bits of later data held in another XMM
+ * register, we'll really be XOR-ing the product into the mathematically higher
+ * degree end of those later bits, not the lower degree end as may be expected.
+ *
+ * So given that caveat and the fact that we process 512 bits per iteration, the
+ * 'D' values we need for the two 64-bit halves of each 128 bits of data are:
+ *
+ *	D = (512 + 95) - 64	 for the higher-degree half of each 128 bits,
+ *				 i.e. the lower order bits in the XMM register
+ *
+ *	D = (512 + 95) - 128	 for the lower-degree half of each 128 bits,
+ *				 i.e. the higher order bits in the XMM register
+ *
+ * The required 'x^D mod G(x)' values were precomputed.
+ *
+ * When <= 512 bits remain in the message, we finish up by folding across
+ * smaller distances.  This works similarly; the distance D is just different,
+ * so different constant multipliers must be used.  Finally, once the remaining
+ * message is just 64 bits, it is reduced to the CRC-32 using Barrett reduction
+ * (explained later).
+ *
+ * For more information see the original paper from Intel:
+ *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
+ *	December 2009
+ *	http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+ */
+static u32 ATTRIBUTES
+FUNCNAME_ALIGNED(u32 remainder, const __m128i *p, size_t nr_segs)
+{
+	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
+	const __v2di multipliers_4 = (__v2di){ 0x8F352D95, 0x1D9513D7 };
+	const __v2di multipliers_2 = (__v2di){ 0xF1DA05AA, 0x81256527 };
+	const __v2di multipliers_1 = (__v2di){ 0xAE689191, 0xCCAA009E };
+	const __v2di final_multiplier = (__v2di){ 0xB8BC6765 };
+	const __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };
+	const __v2di barrett_reduction_constants =
+			(__v2di){ 0x00000001F7011641, 0x00000001DB710641 };
+
+	const __m128i * const end = p + nr_segs;
+	const __m128i * const end512 = p + (nr_segs & ~3);
+	__m128i x0, x1, x2, x3;
+
+	/*
+	 * Account for the current 'remainder', i.e. the CRC of the part of the
+	 * message already processed.  Explanation: rewrite the message
+	 * polynomial M(x) in terms of the first part A(x), the second part
+	 * B(x), and the length of the second part in bits |B(x)| >= 32:
+	 *
+	 *	M(x) = A(x)*x^|B(x)| + B(x)
+	 *
+	 * Then the CRC of M(x) is:
+	 *
+	 *	CRC(M(x)) = CRC(A(x)*x^|B(x)| + B(x))
+	 *	          = CRC(A(x)*x^32*x^(|B(x)| - 32) + B(x))
+	 *	          = CRC(CRC(A(x))*x^(|B(x)| - 32) + B(x))
+	 *
+	 * Note: all arithmetic is modulo G(x), the generator polynomial; that's
+	 * why A(x)*x^32 can be replaced with CRC(A(x)) = A(x)*x^32 mod G(x).
+	 *
+	 * So the CRC of the full message is the CRC of the second part of the
+	 * message where the first 32 bits of the second part of the message
+	 * have been XOR'ed with the CRC of the first part of the message.
+	 */
+	x0 = *p++;
+	x0 ^= (__m128i)(__v4si){ remainder };
+
+	if (p > end512) /* only 128, 256, or 384 bits of input? */
+		goto _128_bits_at_a_time;
+	x1 = *p++;
+	x2 = *p++;
+	x3 = *p++;
+
+	/* Fold 512 bits at a time */
+	for (; p != end512; p += 4) {
+		__m128i y0, y1, y2, y3;
+
+		y0 = p[0];
+		y1 = p[1];
+		y2 = p[2];
+		y3 = p[3];
+
+		/*
+		 * Note: the immediate constant for PCLMULQDQ specifies which
+		 * 64-bit halves of the 128-bit vectors to multiply:
+		 *
+		 * 0x00 means low halves (higher degree polynomial terms for us)
+		 * 0x11 means high halves (lower degree polynomial terms for us)
+		 */
+		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x00);
+		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x00);
+		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x00);
+		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x00);
+		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x11);
+		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x11);
+		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x11);
+		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x11);
+
+		x0 = y0;
+		x1 = y1;
+		x2 = y2;
+		x3 = y3;
+	}
+
+	/* Fold 512 bits => 128 bits */
+	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x00);
+	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x00);
+	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x11);
+	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x11);
+	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x00);
+	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x11);
+	x0 = x3;
+
+_128_bits_at_a_time:
+	while (p != end) {
+		/* Fold 128 bits into next 128 bits */
+		x1 = *p++;
+		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x00);
+		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x11);
+		x0 = x1;
+	}
+
+	/* Now there are just 128 bits left, stored in 'x0'. */
+
+	/*
+	 * Fold 128 => 96 bits.  This also implicitly appends 32 zero bits,
+	 * which is equivalent to multiplying by x^32.  This is needed because
+	 * the CRC is defined as M(x)*x^32 mod G(x), not just M(x) mod G(x).
+	 */
+	x0 = _mm_srli_si128(x0, 8) ^
+	     _mm_clmulepi64_si128(x0, multipliers_1, 0x10);
+
+	/* Fold 96 => 64 bits */
+	x0 = _mm_srli_si128(x0, 4) ^
+	     _mm_clmulepi64_si128(x0 & mask32, final_multiplier, 0x00);
+
+        /*
+	 * Finally, reduce 64 => 32 bits using Barrett reduction.
+	 *
+	 * Let M(x) = A(x)*x^32 + B(x) be the remaining message.  The goal is to
+	 * compute R(x) = M(x) mod G(x).  Since degree(B(x)) < degree(G(x)):
+	 *
+	 *	R(x) = (A(x)*x^32 + B(x)) mod G(x)
+	 *	     = (A(x)*x^32) mod G(x) + B(x)
+	 *
+	 * Then, by the Division Algorithm there exists a unique q(x) such that:
+	 *
+	 *	A(x)*x^32 mod G(x) = A(x)*x^32 - q(x)*G(x)
+	 *
+	 * Since the left-hand side is of maximum degree 31, the right-hand side
+	 * must be too.  This implies that we can apply 'mod x^32' to the
+	 * right-hand side without changing its value:
+	 *
+	 *	(A(x)*x^32 - q(x)*G(x)) mod x^32 = q(x)*G(x) mod x^32
+	 *
+	 * Note that '+' is equivalent to '-' in polynomials over GF(2).
+	 *
+	 * We also know that:
+	 *
+	 *	              / A(x)*x^32 \
+	 *	q(x) = floor (  ---------  )
+	 *	              \    G(x)   /
+	 *
+	 * To compute this efficiently, we can multiply the top and bottom by
+	 * x^32 and move the division by G(x) to the top:
+	 *
+	 *	              / A(x) * floor(x^64 / G(x)) \
+	 *	q(x) = floor (  -------------------------  )
+	 *	              \           x^32            /
+	 *
+	 * Note that floor(x^64 / G(x)) is a constant.
+	 *
+	 * So finally we have:
+	 *
+	 *	                          / A(x) * floor(x^64 / G(x)) \
+	 *	R(x) = B(x) + G(x)*floor (  -------------------------  )
+	 *	                          \           x^32            /
+	 */
+	x1 = x0;
+	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x00);
+	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x10);
+	return _mm_cvtsi128_si32(_mm_srli_si128(x0 ^ x1, 4));
+}
+
+#define IMPL_ALIGNMENT		16
+#define IMPL_SEGMENT_SIZE	16
+#include "../crc32_vec_template.h"

lib/x86/decompress_impl.h

@@ -0,0 +1,31 @@
+#ifndef LIB_X86_DECOMPRESS_IMPL_H
+#define LIB_X86_DECOMPRESS_IMPL_H
+
+#include "cpu_features.h"
+
+/* Include the BMI2-optimized version? */
+#undef DISPATCH_BMI2
+#if !defined(__BMI2__) && X86_CPU_FEATURES_ENABLED && \
+	COMPILER_SUPPORTS_BMI2_TARGET
+#  define FUNCNAME	deflate_decompress_bmi2
+#  define ATTRIBUTES	__attribute__((target("bmi2")))
+#  define DISPATCH	1
+#  define DISPATCH_BMI2	1
+#  include "../decompress_template.h"
+#endif
+
+#ifdef DISPATCH
+static inline decompress_func_t
+arch_select_decompress_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_BMI2
+	if (features & X86_CPU_FEATURE_BMI2)
+		return deflate_decompress_bmi2;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_DECOMPRESS_IMPL_H */

lib/x86/matchfinder_impl.h

@@ -0,0 +1,122 @@
+/*
+ * x86/matchfinder_impl.h - x86 implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_MATCHFINDER_IMPL_H
+#define LIB_X86_MATCHFINDER_IMPL_H
+
+#ifdef __AVX2__
+#  include <immintrin.h>
+static forceinline void
+matchfinder_init_avx2(mf_pos_t *data, size_t size)
+{
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_avx2
+
+static forceinline void
+matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
+{
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
+		p[0] = _mm256_adds_epi16(p[0], v);
+		p[1] = _mm256_adds_epi16(p[1], v);
+		p[2] = _mm256_adds_epi16(p[2], v);
+		p[3] = _mm256_adds_epi16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_avx2
+
+#elif defined(__SSE2__)
+#  include <emmintrin.h>
+static forceinline void
+matchfinder_init_sse2(mf_pos_t *data, size_t size)
+{
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_sse2
+
+static forceinline void
+matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
+{
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
+		p[0] = _mm_adds_epi16(p[0], v);
+		p[1] = _mm_adds_epi16(p[1], v);
+		p[2] = _mm_adds_epi16(p[2], v);
+		p[3] = _mm_adds_epi16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_sse2
+#endif /* __SSE2__ */
+
+#endif /* LIB_X86_MATCHFINDER_IMPL_H */

lib/x86_cpu_features.c

@@ -1,149 +0,0 @@
-/*
- * x86_cpu_features.c - feature detection for x86 processors
- *
- * Written in 2015 by Eric Biggers <ebiggers3@gmail.com>
- *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
- *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
- */
-
-#include "x86_cpu_features.h"
-
-#if X86_CPU_FEATURES_ENABLED
-
-#define DEBUG 0
-
-#if DEBUG
-#  include <stdio.h>
-#endif
-
-u32 _x86_cpu_features = 0;
-
-/* With old GCC versions we have to manually save and restore the x86_32 PIC
- * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
-#if defined(__i386__) && defined(__PIC__)
-#  define EBX_CONSTRAINT "=r"
-#else
-#  define EBX_CONSTRAINT "=b"
-#endif
-
-/* Execute the CPUID instruction.  */
-static inline void
-cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
-{
-	__asm__(".ifnc %%ebx, %1; mov  %%ebx, %1; .endif\n"
-		"cpuid                                  \n"
-		".ifnc %%ebx, %1; xchg %%ebx, %1; .endif\n"
-		: "=a" (*a), EBX_CONSTRAINT (*b), "=c" (*c), "=d" (*d)
-		: "a" (leaf), "c" (subleaf));
-}
-
-/* Read an extended control register.  */
-static inline u64
-read_xcr(u32 index)
-{
-	u32 edx, eax;
-
-	/* Execute the "xgetbv" instruction.  Old versions of binutils do not
-	 * recognize this instruction, so list the raw bytes instead.  */
-	__asm__ (".byte 0x0f, 0x01, 0xd0" : "=d" (edx), "=a" (eax) : "c" (index));
-
-	return ((u64)edx << 32) | eax;
-}
-
-#define IS_SET(reg, bit) ((reg) & ((u32)1 << (bit)))
-
-/* Initialize _x86_cpu_features with bits for interesting processor features. */
-void
-x86_setup_cpu_features(void)
-{
-	u32 features = 0;
-	u32 dummy1, dummy2, dummy3, dummy4;
-	u32 max_function;
-	u32 features_1, features_2, features_3, features_4;
-	bool os_saves_ymm_regs = false;
-
-	/* Get maximum supported function  */
-	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
-	if (max_function < 1)
-		goto out;
-
-	/* Standard feature flags  */
-	cpuid(1, 0, &dummy1, &dummy2, &features_2, &features_1);
-
-	if (IS_SET(features_1, 25))
-		features |= X86_CPU_FEATURE_SSE;
-
-	if (IS_SET(features_1, 26))
-		features |= X86_CPU_FEATURE_SSE2;
-
-	if (IS_SET(features_2, 0))
-		features |= X86_CPU_FEATURE_SSE3;
-
-	if (IS_SET(features_2, 9))
-		features |= X86_CPU_FEATURE_SSSE3;
-
-	if (IS_SET(features_2, 19))
-		features |= X86_CPU_FEATURE_SSE4_1;
-
-	if (IS_SET(features_2, 20))
-		features |= X86_CPU_FEATURE_SSE4_2;
-
-	if (IS_SET(features_2, 27)) /* OSXSAVE set?  */
-		if ((read_xcr(0) & 0x6) == 0x6)
-			os_saves_ymm_regs = true;
-
-	if (os_saves_ymm_regs && IS_SET(features_2, 28))
-		features |= X86_CPU_FEATURE_AVX;
-
-	if (max_function < 7)
-		goto out;
-
-	/* Extended feature flags  */
-	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
-
-	if (IS_SET(features_3, 3))
-		features |= X86_CPU_FEATURE_BMI;
-
-	if (os_saves_ymm_regs && IS_SET(features_3, 5))
-		features |= X86_CPU_FEATURE_AVX2;
-
-	if (IS_SET(features_3, 8))
-		features |= X86_CPU_FEATURE_BMI2;
-
-out:
-
-#if DEBUG
-	printf("Detected x86 CPU features: ");
-	if (features & X86_CPU_FEATURE_SSE)
-		printf("SSE ");
-	if (features & X86_CPU_FEATURE_SSE2)
-		printf("SSE2 ");
-	if (features & X86_CPU_FEATURE_SSE3)
-		printf("SSE3 ");
-	if (features & X86_CPU_FEATURE_SSSE3)
-		printf("SSSE3 ");
-	if (features & X86_CPU_FEATURE_SSE4_1)
-		printf("SSE4.1 ");
-	if (features & X86_CPU_FEATURE_SSE4_2)
-		printf("SSE4.2 ");
-	if (features & X86_CPU_FEATURE_BMI)
-		printf("BMI ");
-	if (features & X86_CPU_FEATURE_AVX)
-		printf("AVX ");
-	if (features & X86_CPU_FEATURE_BMI2)
-		printf("BMI2 ");
-	if (features & X86_CPU_FEATURE_AVX2)
-		printf("AVX2 ");
-	printf("\n");
-#endif /* DEBUG */
-
-	_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
-}
-
-#endif /* X86_CPU_FEATURES_ENABLED */

lib/x86_cpu_features.h

@@ -1,47 +0,0 @@
-/*
- * x86_cpu_features.h - feature detection for x86 processors
- */
-
-#ifndef _LIB_X86_CPU_FEATURES_H
-#define _LIB_X86_CPU_FEATURES_H
-
-#include "common_defs.h"
-
-#if defined(__x86_64__) && COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-#  define X86_CPU_FEATURES_ENABLED 1
-#else
-#  define X86_CPU_FEATURES_ENABLED 0
-#endif
-
-#if X86_CPU_FEATURES_ENABLED
-
-#define X86_CPU_FEATURE_SSE		0x00000001
-#define X86_CPU_FEATURE_SSE2		0x00000002
-#define X86_CPU_FEATURE_SSE3		0x00000004
-#define X86_CPU_FEATURE_SSSE3		0x00000008
-#define X86_CPU_FEATURE_SSE4_1		0x00000010
-#define X86_CPU_FEATURE_SSE4_2		0x00000020
-#define X86_CPU_FEATURE_AVX		0x00000040
-#define X86_CPU_FEATURE_BMI		0x00000080
-#define X86_CPU_FEATURE_AVX2		0x00000100
-#define X86_CPU_FEATURE_BMI2		0x00000200
-
-#define X86_CPU_FEATURES_KNOWN		0x80000000
-
-extern u32 _x86_cpu_features;
-
-extern void
-x86_setup_cpu_features(void);
-
-/* Does the processor have the specified feature?  */
-static inline bool
-x86_have_cpu_feature(u32 feature)
-{
-	if (_x86_cpu_features == 0)
-		x86_setup_cpu_features();
-	return _x86_cpu_features & feature;
-}
-
-#endif /* X86_CPU_FEATURES_ENABLED */
-
-#endif /* _LIB_X86_CPU_FEATURES_H */

lib/zlib_compress.c

@@ -1,27 +1,40 @@
 /*
  * zlib_compress.c - compress with a zlib wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
 #include "deflate_compress.h"
 #include "unaligned.h"
 #include "zlib_constants.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT size_t
-zlib_compress(struct deflate_compressor *c, const void *in, size_t in_size,
-	      void *out, size_t out_nbytes_avail)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress(struct libdeflate_compressor *c,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
 	u16 hdr;
@@ -50,21 +63,23 @@ zlib_compress(struct deflate_compressor *c, const void *in, size_t in_size,
 	out_next += 2;
 
 	/* Compressed data  */
-	deflate_size = deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - ZLIB_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* ADLER32  */
-	put_unaligned_be32(adler32(in, in_size), out_next);
+	put_unaligned_be32(libdeflate_adler32(1, in, in_nbytes), out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBEXPORT size_t
-zlib_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *c,
+			       size_t in_nbytes)
 {
-	return ZLIB_MIN_OVERHEAD + deflate_compress_bound(c, in_nbytes);
+	return ZLIB_MIN_OVERHEAD +
+	       libdeflate_deflate_compress_bound(c, in_nbytes);
 }

lib/zlib_constants.h

@@ -2,8 +2,8 @@
  * zlib_constants.h - constants for the zlib wrapper format
  */
 
-#ifndef _LIB_ZLIB_CONSTANTS_H
-#define _LIB_ZLIB_CONSTANTS_H
+#ifndef LIB_ZLIB_CONSTANTS_H
+#define LIB_ZLIB_CONSTANTS_H
 
 #define ZLIB_MIN_HEADER_SIZE	2
 #define ZLIB_FOOTER_SIZE	4
@@ -18,4 +18,4 @@
 #define ZLIB_DEFAULT_COMPRESSION	2
 #define ZLIB_SLOWEST_COMPRESSION	3
 
-#endif /* _LIB_ZLIB_CONSTANTS_H */
+#endif /* LIB_ZLIB_CONSTANTS_H */

lib/zlib_decompress.c

@@ -1,37 +1,51 @@
 /*
  * zlib_decompress.c - decompress with a zlib wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
 #include "unaligned.h"
 #include "zlib_constants.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT enum decompress_result
-zlib_decompress(struct deflate_decompressor *d,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret)
 {
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
 	u16 hdr;
+	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
-	enum decompress_result result;
+	enum libdeflate_result result;
 
 	if (in_nbytes < ZLIB_MIN_OVERHEAD)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* 2 byte header: CMF and FLG  */
 	hdr = get_unaligned_be16(in_next);
@@ -39,26 +53,26 @@ zlib_decompress(struct deflate_decompressor *d,
 
 	/* FCHECK */
 	if ((hdr % 31) != 0)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* CM */
 	if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* CINFO */
 	if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* FDICT */
 	if ((hdr >> 5) & 1)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* Compressed data  */
-	result = deflate_decompress(d, in_next,
-				    in_end - ZLIB_FOOTER_SIZE - in_next,
-				    out, out_nbytes_avail,
-				    actual_out_nbytes_ret);
-	if (result != DECOMPRESS_SUCCESS)
+	result = libdeflate_deflate_decompress_ex(d, in_next,
+					in_end - ZLIB_FOOTER_SIZE - in_next,
+					out, out_nbytes_avail,
+					&actual_in_nbytes, actual_out_nbytes_ret);
+	if (result != LIBDEFLATE_SUCCESS)
 		return result;
 
 	if (actual_out_nbytes_ret)
@@ -66,11 +80,27 @@ zlib_decompress(struct deflate_decompressor *d,
 	else
 		actual_out_nbytes = out_nbytes_avail;
 
-	in_next = in_end - ZLIB_FOOTER_SIZE;
+	in_next += actual_in_nbytes;
 
 	/* ADLER32  */
-	if (adler32(out, actual_out_nbytes) != get_unaligned_be32(in_next))
-		return DECOMPRESS_BAD_DATA;
+	if (libdeflate_adler32(1, out, actual_out_nbytes) !=
+	    get_unaligned_be32(in_next))
+		return LIBDEFLATE_BAD_DATA;
+	in_next += 4;
 
-	return DECOMPRESS_SUCCESS;
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	return LIBDEFLATE_SUCCESS;
+}
+
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret)
+{
+	return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
+					     out, out_nbytes_avail,
+					     NULL, actual_out_nbytes_ret);
 }

libdeflate.h

@@ -9,69 +9,100 @@
 extern "C" {
 #endif
 
-#define LIBDEFLATE_VERSION_MAJOR	0
-#define LIBDEFLATE_VERSION_MINOR	2
-#define LIBDEFLATE_VERSION_STRING	"0.2"
+#define LIBDEFLATE_VERSION_MAJOR	1
+#define LIBDEFLATE_VERSION_MINOR	10
+#define LIBDEFLATE_VERSION_STRING	"1.10"
 
 #include <stddef.h>
+#include <stdint.h>
 
-/* Microsoft C / Visual Studio garbage.  If you want to link to the DLL version
- * of libdeflate, then #define LIBDEFLATE_DLL.  */
-#ifdef _MSC_VER
+/*
+ * On Windows, if you want to link to the DLL version of libdeflate, then
+ * #define LIBDEFLATE_DLL.  Note that the calling convention is "stdcall".
+ */
+#ifdef LIBDEFLATE_DLL
 #  ifdef BUILDING_LIBDEFLATE
-#    define LIBDEFLATEAPI __declspec(dllexport)
-#  elif defined(LIBDEFLATE_DLL)
-#    define LIBDEFLATEAPI __declspec(dllimport)
+#    define LIBDEFLATEEXPORT	LIBEXPORT
+#  elif defined(_WIN32) || defined(__CYGWIN__)
+#    define LIBDEFLATEEXPORT	__declspec(dllimport)
 #  endif
 #endif
-#ifndef LIBDEFLATEAPI
-#  define LIBDEFLATEAPI
+#ifndef LIBDEFLATEEXPORT
+#  define LIBDEFLATEEXPORT
 #endif
 
+#if defined(_WIN32) && !defined(_WIN64) && defined(LIBDEFLATE_DLL)
+#  define LIBDEFLATEAPI_ABI	__stdcall
+#else
+#  define LIBDEFLATEAPI_ABI
+#endif
+
+#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \
+	defined(_WIN32) && !defined(_WIN64)
+    /*
+     * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4.
+     * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX
+     * code when called from an MSVC-compiled application.
+     */
+#  define LIBDEFLATEAPI_STACKALIGN	__attribute__((force_align_arg_pointer))
+#else
+#  define LIBDEFLATEAPI_STACKALIGN
+#endif
+
+#define LIBDEFLATEAPI	LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN
+
 /* ========================================================================== */
 /*                             Compression                                    */
 /* ========================================================================== */
 
-struct deflate_compressor;
+struct libdeflate_compressor;
 
 /*
- * deflate_alloc_compressor() allocates a new DEFLATE compressor.
- * 'compression_level' is the compression level on a zlib-like scale but with a
- * higher maximum value (1 = fastest, 6 = medium/default, 9 = slow, 12 =
- * slowest).  The return value is a pointer to the new DEFLATE compressor, or
- * NULL if out of memory.
+ * libdeflate_alloc_compressor() allocates a new compressor that supports
+ * DEFLATE, zlib, and gzip compression.  'compression_level' is the compression
+ * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
+ * medium/default, 9 = slow, 12 = slowest).  Level 0 is also supported and means
+ * "no compression", specifically "create a valid stream, but only emit
+ * uncompressed blocks" (this will expand the data slightly).
+ *
+ * The return value is a pointer to the new compressor, or NULL if out of memory
+ * or if the compression level is invalid (i.e. outside the range [0, 12]).
  *
  * Note: for compression, the sliding window size is defined at compilation time
  * to 32768, the largest size permissible in the DEFLATE format.  It cannot be
  * changed at runtime.
+ *
+ * A single compressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different compressors concurrently.
  */
-LIBDEFLATEAPI struct deflate_compressor *
-deflate_alloc_compressor(unsigned int compression_level);
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+libdeflate_alloc_compressor(int compression_level);
 
 /*
- * deflate_compress() performs DEFLATE compression on a buffer of data.  The
- * function attempts to compress 'in_nbytes' bytes of data located at 'in' and
- * write the results to 'out', which has space for 'out_nbytes_avail' bytes.
- * The return value is the compressed size in bytes, or 0 if the data could not
- * be compressed to 'out_nbytes_avail' bytes or fewer.
+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
+ * data.  The function attempts to compress 'in_nbytes' bytes of data located at
+ * 'in' and write the results to 'out', which has space for 'out_nbytes_avail'
+ * bytes.  The return value is the compressed size in bytes, or 0 if the data
+ * could not be compressed to 'out_nbytes_avail' bytes or fewer.
  */
-LIBDEFLATEAPI size_t
-deflate_compress(struct deflate_compressor *compressor,
-		 const void *in, size_t in_nbytes,
-		 void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
+			    const void *in, size_t in_nbytes,
+			    void *out, size_t out_nbytes_avail);
 
 /*
- * deflate_compress_bound() returns a worst-case upper bound on the number of
- * bytes of compressed data that may be produced by compressing any buffer of
- * length less than or equal to 'in_nbytes' using deflate_compress() with the
- * specified compressor.  Mathematically, this bound will necessarily be a
- * number greater than or equal to 'in_nbytes'.  It may be an overestimate of
- * the true upper bound.  The return value is guaranteed to be the same for all
- * invocations with the same compressor and same 'in_nbytes'.
+ * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
+ * number of bytes of compressed data that may be produced by compressing any
+ * buffer of length less than or equal to 'in_nbytes' using
+ * libdeflate_deflate_compress() with the specified compressor.  Mathematically,
+ * this bound will necessarily be a number greater than or equal to 'in_nbytes'.
+ * It may be an overestimate of the true upper bound.  The return value is
+ * guaranteed to be the same for all invocations with the same compressor and
+ * same 'in_nbytes'.
  *
  * As a special case, 'compressor' may be NULL.  This causes the bound to be
- * taken across *any* deflate_compressor that could ever be allocated with this
- * build of the library, with any options.
+ * taken across *any* libdeflate_compressor that could ever be allocated with
+ * this build of the library, with any options.
  *
  * Note that this function is not necessary in many applications.  With
  * block-based compression, it is usually preferable to separately store the
@@ -79,154 +110,254 @@ deflate_compress(struct deflate_compressor *compressor,
  * to less than their original size uncompressed.  In that scenario, there is no
  * need to know the worst-case compressed size, since the maximum number of
  * bytes of compressed data that may be used would always be one less than the
- * input length.  You can just pass a buffer of that size to deflate_compress()
- * and store the data uncompressed if deflate_compress() returns 0, indicating
- * that the compressed data did not fit into the provided output buffer.
+ * input length.  You can just pass a buffer of that size to
+ * libdeflate_deflate_compress() and store the data uncompressed if
+ * libdeflate_deflate_compress() returns 0, indicating that the compressed data
+ * did not fit into the provided output buffer.
  */
-LIBDEFLATEAPI size_t
-deflate_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
+				  size_t in_nbytes);
 
 /*
- * Like deflate_compress(), but stores the data in the zlib wrapper format.
+ * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper
+ * format.
  */
-LIBDEFLATEAPI size_t
-zlib_compress(struct deflate_compressor *compressor,
-	      const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail);
 
 /*
- * Like deflate_compress_bound(), but assumes the data will be compressed with
- * zlib_compress() rather than with deflate_compress().
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_zlib_compress() rather than with
+ * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
-zlib_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
+			       size_t in_nbytes);
 
 /*
- * Like deflate_compress(), but stores the data in the gzip wrapper format.
+ * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper
+ * format.
  */
-LIBDEFLATEAPI size_t
-gzip_compress(struct deflate_compressor *compressor,
-	      const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail);
 
 /*
- * Like deflate_compress_bound(), but assumes the data will be compressed with
- * gzip_compress() rather than with deflate_compress().
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_gzip_compress() rather than with
+ * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
-gzip_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
+			       size_t in_nbytes);
 
 /*
- * deflate_free_compressor() frees a DEFLATE compressor that was allocated with
- * deflate_alloc_compressor().  If a NULL pointer is passed in, no action is
+ * libdeflate_free_compressor() frees a compressor that was allocated with
+ * libdeflate_alloc_compressor().  If a NULL pointer is passed in, no action is
  * taken.
  */
-LIBDEFLATEAPI void
-deflate_free_compressor(struct deflate_compressor *compressor);
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_compressor(struct libdeflate_compressor *compressor);
 
 /* ========================================================================== */
 /*                             Decompression                                  */
 /* ========================================================================== */
 
-struct deflate_decompressor;
+struct libdeflate_decompressor;
 
 /*
- * deflate_alloc_decompressor() allocates a new DEFLATE decompressor.  The
- * return value is a pointer to the new DEFLATE decompressor, or NULL if out of
- * memory.
+ * libdeflate_alloc_decompressor() allocates a new decompressor that can be used
+ * for DEFLATE, zlib, and gzip decompression.  The return value is a pointer to
+ * the new decompressor, or NULL if out of memory.
  *
  * This function takes no parameters, and the returned decompressor is valid for
  * decompressing data that was compressed at any compression level and with any
- * sliding window size.  It can also be used for any wrapper format (raw
- * DEFLATE, zlib, or gzip); however, the appropriate decompression function must
- * be called.
+ * sliding window size.
+ *
+ * A single decompressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different decompressors concurrently.
  */
-LIBDEFLATEAPI struct deflate_decompressor *
-deflate_alloc_decompressor(void);
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
+libdeflate_alloc_decompressor(void);
 
 /*
- * Result of a call to deflate_decompress(), zlib_decompress(), or
- * gzip_decompress().
+ * Result of a call to libdeflate_deflate_decompress(),
+ * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
  */
-enum decompress_result {
+enum libdeflate_result {
 	/* Decompression was successful.  */
-	DECOMPRESS_SUCCESS = 0,
+	LIBDEFLATE_SUCCESS = 0,
 
 	/* Decompressed failed because the compressed data was invalid, corrupt,
 	 * or otherwise unsupported.  */
-	DECOMPRESS_BAD_DATA = 1,
+	LIBDEFLATE_BAD_DATA = 1,
 
 	/* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
 	 * decompressed to fewer than 'out_nbytes_avail' bytes.  */
-	DECOMPRESS_SHORT_OUTPUT = 2,
+	LIBDEFLATE_SHORT_OUTPUT = 2,
 
 	/* The data would have decompressed to more than 'out_nbytes_avail'
 	 * bytes.  */
-	DECOMPRESS_INSUFFICIENT_SPACE = 3,
+	LIBDEFLATE_INSUFFICIENT_SPACE = 3,
 };
 
 /*
- * deflate_decompress() decompresses 'in_nbytes' bytes of DEFLATE-compressed
- * data at 'in' and writes the uncompressed data to 'out', which is a buffer of
- * at least 'out_nbytes_avail' bytes.  If decompression was successful, then 0
- * (DECOMPRESS_SUCCESS) is returned; otherwise, a nonzero result code such as
- * DECOMPRESS_BAD_DATA is returned.  If a nonzero result code is returned, then
- * the contents of the output buffer are undefined.
+ * libdeflate_deflate_decompress() decompresses the DEFLATE-compressed stream
+ * from the buffer 'in' with compressed size up to 'in_nbytes' bytes.  The
+ * uncompressed data is written to 'out', a buffer with size 'out_nbytes_avail'
+ * bytes.  If decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned.
+ * Otherwise, a nonzero result code such as LIBDEFLATE_BAD_DATA is returned.  If
+ * a nonzero result code is returned, then the contents of the output buffer are
+ * undefined.
  *
- * deflate_decompress() can be used in cases where the actual uncompressed size
- * is known (recommended) or unknown (not recommended):
+ * Decompression stops at the end of the DEFLATE stream (as indicated by the
+ * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
+ *
+ * libdeflate_deflate_decompress() can be used in cases where the actual
+ * uncompressed size is known (recommended) or unknown (not recommended):
  *
  *   - If the actual uncompressed size is known, then pass the actual
  *     uncompressed size as 'out_nbytes_avail' and pass NULL for
- *     'actual_out_nbytes_ret'.  This makes deflate_decompress() fail with
- *     DECOMPRESS_SHORT_OUTPUT if the data decompressed to fewer than the
+ *     'actual_out_nbytes_ret'.  This makes libdeflate_deflate_decompress() fail
+ *     with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
  *     specified number of bytes.
  *
  *   - If the actual uncompressed size is unknown, then provide a non-NULL
  *     'actual_out_nbytes_ret' and provide a buffer with some size
  *     'out_nbytes_avail' that you think is large enough to hold all the
  *     uncompressed data.  In this case, if the data decompresses to less than
- *     or equal to 'out_nbytes_avail' bytes, then deflate_decompress() will
- *     write the actual uncompressed size to *actual_out_nbytes_ret and return 0
- *     (DECOMPRESS_SUCCESS).  Otherwise, it will return
- *     DECOMPRESS_INSUFFICIENT_SPACE if the provided buffer was not large enough
- *     but no other problems were encountered, or another nonzero result code if
- *     decompression failed for another reason.
+ *     or equal to 'out_nbytes_avail' bytes, then
+ *     libdeflate_deflate_decompress() will write the actual uncompressed size
+ *     to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS).  Otherwise,
+ *     it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
+ *     not large enough but no other problems were encountered, or another
+ *     nonzero result code if decompression failed for another reason.
  */
-LIBDEFLATEAPI enum decompress_result
-deflate_decompress(struct deflate_decompressor *decompressor,
-		   const void *in, size_t in_nbytes,
-		   void *out, size_t out_nbytes_avail,
-		   size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_out_nbytes_ret);
 
 /*
- * Like deflate_decompress(), but assumes the zlib wrapper format instead of raw
- * DEFLATE.
+ * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
+ * then the actual compressed size of the DEFLATE stream (aligned to the next
+ * byte boundary) is written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEAPI enum decompress_result
-zlib_decompress(struct deflate_decompressor *decompressor,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
+				 const void *in, size_t in_nbytes,
+				 void *out, size_t out_nbytes_avail,
+				 size_t *actual_in_nbytes_ret,
+				 size_t *actual_out_nbytes_ret);
 
 /*
- * Like deflate_decompress(), but assumes the gzip wrapper format instead of raw
- * DEFLATE.
+ * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
+ * instead of raw DEFLATE.
+ *
+ * Decompression will stop at the end of the zlib stream, even if it is shorter
+ * than 'in_nbytes'.  If you need to know exactly where the zlib stream ended,
+ * use libdeflate_zlib_decompress_ex().
  */
-LIBDEFLATEAPI enum decompress_result
-gzip_decompress(struct deflate_decompressor *decompressor,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret);
 
 /*
- * deflate_free_decompressor() frees a DEFLATE decompressor that was allocated
- * with deflate_alloc_decompressor().  If a NULL pointer is passed in, no action
+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first zlib-compressed stream in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
+ * instead of raw DEFLATE.
+ *
+ * If multiple gzip-compressed members are concatenated, then only the first
+ * will be decompressed.  Use libdeflate_gzip_decompress_ex() if you need
+ * multi-member support.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first gzip-compressed member in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret);
+
+/*
+ * libdeflate_free_decompressor() frees a decompressor that was allocated with
+ * libdeflate_alloc_decompressor().  If a NULL pointer is passed in, no action
  * is taken.
  */
-LIBDEFLATEAPI void
-deflate_free_decompressor(struct deflate_decompressor *decompressor);
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
 
+/* ========================================================================== */
+/*                                Checksums                                   */
+/* ========================================================================== */
+
+/*
+ * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
+ * data and returns the updated checksum.  When starting a new checksum, the
+ * required initial value for 'adler' is 1.  This value is also returned when
+ * 'buffer' is specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
+
+
+/*
+ * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
+ * and returns the updated checksum.  When starting a new checksum, the required
+ * initial value for 'crc' is 0.  This value is also returned when 'buffer' is
+ * specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
+
+/* ========================================================================== */
+/*                           Custom memory allocator                          */
+/* ========================================================================== */
+
+/*
+ * Install a custom memory allocator which libdeflate will use for all memory
+ * allocations.  'malloc_func' is a function that must behave like malloc(), and
+ * 'free_func' is a function that must behave like free().
+ *
+ * There must not be any libdeflate_compressor or libdeflate_decompressor
+ * structures in existence when calling this function.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *));
 
 #ifdef __cplusplus
 }

libdeflate.pc.in

@@ -0,0 +1,10 @@
+prefix=@PREFIX@
+exec_prefix=${prefix}
+includedir=@INCDIR@
+libdir=@LIBDIR@
+
+Name: libdeflate
+Description: Fast implementation of DEFLATE, zlib, and gzip
+Version: @VERSION@
+Libs: -L${libdir} -ldeflate
+Cflags: -I${includedir}

programs/benchmark.c

@@ -25,31 +25,390 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <zlib.h> /* for comparison purposes */
+#include "test_util.h"
 
-#include "prog_util.h"
+static const tchar *const optstring = T("0::1::2::3::4::5::6::7::8::9::C:D:eghs:VYZz");
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::ghs:VYZz");
+enum format {
+	DEFLATE_FORMAT,
+	ZLIB_FORMAT,
+	GZIP_FORMAT,
+};
+
+struct compressor {
+	int level;
+	enum format format;
+	const struct engine *engine;
+	void *private;
+};
+
+struct decompressor {
+	enum format format;
+	const struct engine *engine;
+	void *private;
+};
+
+struct engine {
+	const tchar *name;
+
+	bool (*init_compressor)(struct compressor *);
+	size_t (*compress_bound)(struct compressor *, size_t);
+	size_t (*compress)(struct compressor *, const void *, size_t,
+			   void *, size_t);
+	void (*destroy_compressor)(struct compressor *);
+
+	bool (*init_decompressor)(struct decompressor *);
+	bool (*decompress)(struct decompressor *, const void *, size_t,
+			   void *, size_t);
+	void (*destroy_decompressor)(struct decompressor *);
+};
+
+/******************************************************************************/
+
+static bool
+libdeflate_engine_init_compressor(struct compressor *c)
+{
+	c->private = alloc_compressor(c->level);
+	return c->private != NULL;
+}
+
+static size_t
+libdeflate_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	switch (c->format) {
+	case ZLIB_FORMAT:
+		return libdeflate_zlib_compress_bound(c->private, in_nbytes);
+	case GZIP_FORMAT:
+		return libdeflate_gzip_compress_bound(c->private, in_nbytes);
+	default:
+		return libdeflate_deflate_compress_bound(c->private, in_nbytes);
+	}
+}
+
+static size_t
+libdeflate_engine_compress(struct compressor *c, const void *in,
+			   size_t in_nbytes, void *out, size_t out_nbytes_avail)
+{
+	switch (c->format) {
+	case ZLIB_FORMAT:
+		return libdeflate_zlib_compress(c->private, in, in_nbytes,
+						out, out_nbytes_avail);
+	case GZIP_FORMAT:
+		return libdeflate_gzip_compress(c->private, in, in_nbytes,
+						out, out_nbytes_avail);
+	default:
+		return libdeflate_deflate_compress(c->private, in, in_nbytes,
+						   out, out_nbytes_avail);
+	}
+}
+
+static void
+libdeflate_engine_destroy_compressor(struct compressor *c)
+{
+	libdeflate_free_compressor(c->private);
+}
+
+static bool
+libdeflate_engine_init_decompressor(struct decompressor *d)
+{
+	d->private = alloc_decompressor();
+	return d->private != NULL;
+}
+
+static bool
+libdeflate_engine_decompress(struct decompressor *d, const void *in,
+			     size_t in_nbytes, void *out, size_t out_nbytes)
+{
+	switch (d->format) {
+	case ZLIB_FORMAT:
+		return !libdeflate_zlib_decompress(d->private, in, in_nbytes,
+						   out, out_nbytes, NULL);
+	case GZIP_FORMAT:
+		return !libdeflate_gzip_decompress(d->private, in, in_nbytes,
+						   out, out_nbytes, NULL);
+	default:
+		return !libdeflate_deflate_decompress(d->private, in, in_nbytes,
+						      out, out_nbytes, NULL);
+	}
+}
+
+static void
+libdeflate_engine_destroy_decompressor(struct decompressor *d)
+{
+	libdeflate_free_decompressor(d->private);
+}
+
+static const struct engine libdeflate_engine = {
+	.name			= T("libdeflate"),
+
+	.init_compressor	= libdeflate_engine_init_compressor,
+	.compress_bound		= libdeflate_engine_compress_bound,
+	.compress		= libdeflate_engine_compress,
+	.destroy_compressor	= libdeflate_engine_destroy_compressor,
+
+	.init_decompressor	= libdeflate_engine_init_decompressor,
+	.decompress		= libdeflate_engine_decompress,
+	.destroy_decompressor	= libdeflate_engine_destroy_decompressor,
+};
+
+/******************************************************************************/
+
+static int
+get_libz_window_bits(enum format format)
+{
+	const int windowBits = 15;
+	switch (format) {
+	case ZLIB_FORMAT:
+		return windowBits;
+	case GZIP_FORMAT:
+		return windowBits + 16;
+	default:
+		return -windowBits;
+	}
+}
+
+static bool
+libz_engine_init_compressor(struct compressor *c)
+{
+	z_stream *z;
+
+	if (c->level > 9) {
+		msg("libz only supports up to compression level 9");
+		return false;
+	}
+
+	z = xmalloc(sizeof(*z));
+	if (z == NULL)
+		return false;
+
+	z->next_in = NULL;
+	z->avail_in = 0;
+	z->zalloc = NULL;
+	z->zfree = NULL;
+	z->opaque = NULL;
+	if (deflateInit2(z, c->level, Z_DEFLATED,
+			 get_libz_window_bits(c->format),
+			 8, Z_DEFAULT_STRATEGY) != Z_OK)
+	{
+		msg("unable to initialize deflater");
+		free(z);
+		return false;
+	}
+
+	c->private = z;
+	return true;
+}
+
+static size_t
+libz_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return deflateBound(c->private, in_nbytes);
+}
+
+static size_t
+libz_engine_compress(struct compressor *c, const void *in, size_t in_nbytes,
+		     void *out, size_t out_nbytes_avail)
+{
+	z_stream *z = c->private;
+
+	deflateReset(z);
+
+	z->next_in = (void *)in;
+	z->avail_in = in_nbytes;
+	z->next_out = out;
+	z->avail_out = out_nbytes_avail;
+
+	if (deflate(z, Z_FINISH) != Z_STREAM_END)
+		return 0;
+
+	return out_nbytes_avail - z->avail_out;
+}
+
+static void
+libz_engine_destroy_compressor(struct compressor *c)
+{
+	z_stream *z = c->private;
+
+	deflateEnd(z);
+	free(z);
+}
+
+static bool
+libz_engine_init_decompressor(struct decompressor *d)
+{
+	z_stream *z;
+
+	z = xmalloc(sizeof(*z));
+	if (z == NULL)
+		return false;
+
+	z->next_in = NULL;
+	z->avail_in = 0;
+	z->zalloc = NULL;
+	z->zfree = NULL;
+	z->opaque = NULL;
+	if (inflateInit2(z, get_libz_window_bits(d->format)) != Z_OK) {
+		msg("unable to initialize inflater");
+		free(z);
+		return false;
+	}
+
+	d->private = z;
+	return true;
+}
+
+static bool
+libz_engine_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
+		       void *out, size_t out_nbytes)
+{
+	z_stream *z = d->private;
+
+	inflateReset(z);
+
+	z->next_in = (void *)in;
+	z->avail_in = in_nbytes;
+	z->next_out = out;
+	z->avail_out = out_nbytes;
+
+	return inflate(z, Z_FINISH) == Z_STREAM_END && z->avail_out == 0;
+}
+
+static void
+libz_engine_destroy_decompressor(struct decompressor *d)
+{
+	z_stream *z = d->private;
+
+	inflateEnd(z);
+	free(z);
+}
+
+static const struct engine libz_engine = {
+	.name			= T("libz"),
+
+	.init_compressor	= libz_engine_init_compressor,
+	.compress_bound		= libz_engine_compress_bound,
+	.compress		= libz_engine_compress,
+	.destroy_compressor	= libz_engine_destroy_compressor,
+
+	.init_decompressor	= libz_engine_init_decompressor,
+	.decompress		= libz_engine_decompress,
+	.destroy_decompressor	= libz_engine_destroy_decompressor,
+};
+
+/******************************************************************************/
+
+static const struct engine * const all_engines[] = {
+	&libdeflate_engine,
+	&libz_engine,
+};
+
+#define DEFAULT_ENGINE libdeflate_engine
+
+static const struct engine *
+name_to_engine(const tchar *name)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_LEN(all_engines); i++)
+		if (tstrcmp(all_engines[i]->name, name) == 0)
+			return all_engines[i];
+	return NULL;
+}
+
+/******************************************************************************/
+
+static bool
+compressor_init(struct compressor *c, int level, enum format format,
+		const struct engine *engine)
+{
+	c->level = level;
+	c->format = format;
+	c->engine = engine;
+	return engine->init_compressor(c);
+}
+
+static size_t
+compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return c->engine->compress_bound(c, in_nbytes);
+}
+
+static size_t
+do_compress(struct compressor *c, const void *in, size_t in_nbytes,
+	    void *out, size_t out_nbytes_avail)
+{
+	return c->engine->compress(c, in, in_nbytes, out, out_nbytes_avail);
+}
+
+static void
+compressor_destroy(struct compressor *c)
+{
+	if (c->engine != NULL)
+		c->engine->destroy_compressor(c);
+}
+
+static bool
+decompressor_init(struct decompressor *d, enum format format,
+		  const struct engine *engine)
+{
+	d->format = format;
+	d->engine = engine;
+	return engine->init_decompressor(d);
+}
+
+static bool
+do_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
+	      void *out, size_t out_nbytes)
+{
+	return d->engine->decompress(d, in, in_nbytes, out, out_nbytes);
+}
+
+static void
+decompressor_destroy(struct decompressor *d)
+{
+	if (d->engine != NULL)
+		d->engine->destroy_decompressor(d);
+}
+
+/******************************************************************************/
+
+static void
+show_available_engines(FILE *fp)
+{
+	size_t i;
+
+	fprintf(fp, "Available ENGINEs are: ");
+	for (i = 0; i < ARRAY_LEN(all_engines); i++) {
+		fprintf(fp, "%"TS, all_engines[i]->name);
+		if (i < ARRAY_LEN(all_engines) - 1)
+			fprintf(fp, ", ");
+	}
+	fprintf(fp, ".  Default is %"TS"\n", DEFAULT_ENGINE.name);
+}
 
 static void
 show_usage(FILE *fp)
 {
 	fprintf(fp,
-"Usage: %"TS" [-LVL] [-ghVYZz] [-s SIZE] [FILE]...\n"
+"Usage: %"TS" [-LVL] [-C ENGINE] [-D ENGINE] [-ghVz] [-s SIZE] [FILE]...\n"
 "Benchmark DEFLATE compression and decompression on the specified FILEs.\n"
 "\n"
 "Options:\n"
+"  -0        no compression\n"
 "  -1        fastest (worst) compression\n"
 "  -6        medium compression (default)\n"
 "  -12       slowest (best) compression\n"
-"  -g        use gzip wrapper\n"
+"  -C ENGINE compression engine\n"
+"  -D ENGINE decompression engine\n"
+"  -e        allow chunks to be expanded (implied by -0)\n"
+"  -g        use gzip format instead of raw DEFLATE\n"
 "  -h        print this help\n"
 "  -s SIZE   chunk size\n"
 "  -V        show version and legal information\n"
-"  -Y        compress with libz, not libdeflate\n"
-"  -Z        decompress with libz, not libdeflate\n"
-"  -z        use zlib wrapper\n",
-	program_invocation_name);
+"  -z        use zlib format instead of raw DEFLATE\n"
+"\n", prog_invocation_name);
+
+	show_available_engines(fp);
 }
 
 static void
@@ -65,273 +424,13 @@ show_version(void)
 	);
 }
 
-enum wrapper {
-	NO_WRAPPER,
-	ZLIB_WRAPPER,
-	GZIP_WRAPPER,
-};
 
-struct compressor {
-	void *private;
-	size_t (*compress)(void *, const void *, size_t, void *, size_t);
-	void (*free_private)(void *);
-};
-
-struct decompressor {
-	void *private;
-	bool (*decompress)(void *, const void *, size_t, void *, size_t);
-	void (*free_private)(void *);
-};
-
-static int
-get_libz_window_bits(enum wrapper wrapper)
-{
-	const int windowBits = 15;
-	switch (wrapper) {
-	case ZLIB_WRAPPER:
-		return windowBits;
-	case GZIP_WRAPPER:
-		return windowBits + 16;
-	default:
-		return -windowBits;
-	}
-}
-
-static size_t
-libz_compress(void *private, const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail)
-{
-	z_stream *z = private;
-
-	deflateReset(z);
-
-	z->next_in = (void *)in;
-	z->avail_in = in_nbytes;
-	z->next_out = out;
-	z->avail_out = out_nbytes_avail;
-
-	if (deflate(z, Z_FINISH) != Z_STREAM_END)
-		return 0;
-
-	return out_nbytes_avail - z->avail_out;
-}
-
-static bool
-libz_decompress(void *private, const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail)
-{
-	z_stream *z = private;
-
-	inflateReset(z);
-
-	z->next_in = (void *)in;
-	z->avail_in = in_nbytes;
-	z->next_out = out;
-	z->avail_out = out_nbytes_avail;
-
-	return (inflate(z, Z_FINISH) == Z_STREAM_END && z->avail_out == 0);
-}
-
-static void
-libz_free_compressor(void *private)
-{
-	deflateEnd((z_stream *)private);
-	free(private);
-}
-
-static void
-libz_free_decompressor(void *private)
-{
-	inflateEnd((z_stream *)private);
-	free(private);
-}
-
-static size_t
-libdeflate_deflate_compress(void *private, const void *in, size_t in_nbytes,
-			    void *out, size_t out_nbytes_avail)
-{
-	return deflate_compress(private, in, in_nbytes,
-				out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_deflate_decompress(void *private, const void *in, size_t in_nbytes,
-			      void *out, size_t out_nbytes_avail)
-{
-	return 0 == deflate_decompress(private, in, in_nbytes,
-				       out, out_nbytes_avail, NULL);
-}
-
-static size_t
-libdeflate_zlib_compress(void *private, const void *in, size_t in_nbytes,
-			 void *out, size_t out_nbytes_avail)
-{
-	return zlib_compress(private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_zlib_decompress(void *private, const void *in, size_t in_nbytes,
-			   void *out, size_t out_nbytes_avail)
-{
-	return 0 == zlib_decompress(private, in, in_nbytes,
-				    out, out_nbytes_avail, NULL);
-}
-
-static size_t
-libdeflate_gzip_compress(void *private, const void *in, size_t in_nbytes,
-			 void *out, size_t out_nbytes_avail)
-{
-	return gzip_compress(private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_gzip_decompress(void *private, const void *in, size_t in_nbytes,
-			   void *out, size_t out_nbytes_avail)
-{
-	return 0 == gzip_decompress(private, in, in_nbytes,
-				    out, out_nbytes_avail, NULL);
-}
-
-static void
-libdeflate_free_compressor(void *private)
-{
-	deflate_free_compressor(private);
-}
-
-static void
-libdeflate_free_decompressor(void *private)
-{
-	deflate_free_decompressor(private);
-}
-
-static int
-compressor_init(struct compressor *c, int level,
-		enum wrapper wrapper, bool use_libz)
-{
-	if (use_libz) {
-		z_stream *z;
-
-		if (level > 9) {
-			msg("libz only supports up to compression level 9");
-			return -1;
-		}
-
-		z = xmalloc(sizeof(z_stream));
-		if (z == NULL)
-			return -1;
-
-		c->private = z;
-
-		z->next_in = NULL;
-		z->avail_in = 0;
-		z->zalloc = NULL;
-		z->zfree = NULL;
-		z->opaque = NULL;
-		if (deflateInit2(z, level, Z_DEFLATED,
-				 get_libz_window_bits(wrapper),
-				 8, Z_DEFAULT_STRATEGY) != Z_OK)
-		{
-			msg("unable to initialize deflater");
-			free(z);
-			return -1;
-		}
-		c->compress = libz_compress;
-		c->free_private = libz_free_compressor;
-	} else {
-		c->private = alloc_compressor(level);
-		if (c->private == NULL)
-			return -1;
-		switch (wrapper) {
-		case ZLIB_WRAPPER:
-			c->compress = libdeflate_zlib_compress;
-			break;
-		case GZIP_WRAPPER:
-			c->compress = libdeflate_gzip_compress;
-			break;
-		default:
-			c->compress = libdeflate_deflate_compress;
-			break;
-		}
-		c->free_private = libdeflate_free_compressor;
-	}
-	return 0;
-}
-
-static size_t
-do_compress(struct compressor *c, const void *in, size_t in_nbytes,
-	    void *out, size_t out_nbytes_avail)
-{
-	return (*c->compress)(c->private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static void
-compressor_destroy(struct compressor *c)
-{
-	(*c->free_private)(c->private);
-}
-
-static int
-decompressor_init(struct decompressor *d, enum wrapper wrapper, bool use_libz)
-{
-	if (use_libz) {
-		z_stream *z;
-
-		z = xmalloc(sizeof(z_stream));
-		if (z == NULL)
-			return -1;
-
-		d->private = z;
-
-		z->next_in = NULL;
-		z->avail_in = 0;
-		z->zalloc = NULL;
-		z->zfree = NULL;
-		z->opaque = NULL;
-		if (inflateInit2(z, get_libz_window_bits(wrapper)) != Z_OK) {
-			msg("unable to initialize inflater");
-			free(z);
-			return -1;
-		}
-
-		d->decompress = libz_decompress;
-		d->free_private = libz_free_decompressor;
-	} else {
-		d->private = alloc_decompressor();
-		if (d->private == NULL)
-			return -1;
-		switch (wrapper) {
-		case ZLIB_WRAPPER:
-			d->decompress = libdeflate_zlib_decompress;
-			break;
-		case GZIP_WRAPPER:
-			d->decompress = libdeflate_gzip_decompress;
-			break;
-		default:
-			d->decompress = libdeflate_deflate_decompress;
-			break;
-		}
-		d->free_private = libdeflate_free_decompressor;
-	}
-	return 0;
-}
-
-static bool
-do_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail)
-{
-	return (*d->decompress)(d->private, in, in_nbytes,
-				out, out_nbytes_avail);
-}
-
-static void
-decompressor_destroy(struct decompressor *d)
-{
-	(*d->free_private)(d->private);
-}
+/******************************************************************************/
 
 static int
 do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 	     void *decompressed_buf, u32 chunk_size,
+	     bool allow_expansion, size_t compressed_buf_size,
 	     struct compressor *compressor,
 	     struct decompressor *decompressor)
 {
@@ -343,35 +442,45 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 	while ((ret = xread(in, original_buf, chunk_size)) > 0) {
 		u32 original_size = ret;
+		size_t out_nbytes_avail;
 		u32 compressed_size;
 		u64 start_time;
-		bool result;
+		bool ok;
 
 		total_uncompressed_size += original_size;
 
+		if (allow_expansion) {
+			out_nbytes_avail = compress_bound(compressor,
+							  original_size);
+			if (out_nbytes_avail > compressed_buf_size) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
+		} else {
+			out_nbytes_avail = original_size - 1;
+		}
+
 		/* Compress the chunk of data. */
-		start_time = current_time();
+		start_time = timer_ticks();
 		compressed_size = do_compress(compressor,
 					      original_buf,
 					      original_size,
 					      compressed_buf,
-					      original_size - 1);
-		total_compress_time += current_time() - start_time;
+					      out_nbytes_avail);
+		total_compress_time += timer_ticks() - start_time;
 
 		if (compressed_size) {
 			/* Successfully compressed the chunk of data. */
 
 			/* Decompress the data we just compressed and compare
 			 * the result with the original. */
-			start_time = current_time();
-			result = do_decompress(decompressor,
-					       compressed_buf,
-					       compressed_size,
-					       decompressed_buf,
-					       original_size);
-			total_decompress_time += current_time() - start_time;
+			start_time = timer_ticks();
+			ok = do_decompress(decompressor,
+					   compressed_buf, compressed_size,
+					   decompressed_buf, original_size);
+			total_decompress_time += timer_ticks() - start_time;
 
-			if (!result) {
+			if (!ok) {
 				msg("%"TS": failed to decompress data",
 				    in->name);
 				return -1;
@@ -387,7 +496,14 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 			total_compressed_size += compressed_size;
 		} else {
-			/* Compression did not make the chunk smaller. */
+			/*
+			 * The chunk would have compressed to more than
+			 * out_nbytes_avail bytes.
+			 */
+			if (allow_expansion) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
 			total_compressed_size += original_size;
 		}
 	}
@@ -412,11 +528,11 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 	       (unsigned int)(total_compressed_size * 100000 /
 				total_uncompressed_size % 1000));
 	printf("\tCompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n",
-	       total_compress_time / 1000000,
-	       1000 * total_uncompressed_size / total_compress_time);
+	       timer_ticks_to_ms(total_compress_time),
+	       timer_MB_per_s(total_uncompressed_size, total_compress_time));
 	printf("\tDecompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n",
-	       total_decompress_time / 1000000,
-	       1000 * total_uncompressed_size / total_decompress_time);
+	       timer_ticks_to_ms(total_decompress_time),
+	       timer_MB_per_s(total_uncompressed_size, total_decompress_time));
 
 	return 0;
 }
@@ -426,23 +542,26 @@ tmain(int argc, tchar *argv[])
 {
 	u32 chunk_size = 1048576;
 	int level = 6;
-	enum wrapper wrapper = NO_WRAPPER;
-	bool compress_with_libz = false;
-	bool decompress_with_libz = false;
+	enum format format = DEFLATE_FORMAT;
+	const struct engine *compress_engine = &DEFAULT_ENGINE;
+	const struct engine *decompress_engine = &DEFAULT_ENGINE;
+	bool allow_expansion = false;
+	struct compressor compressor = { 0 };
+	struct decompressor decompressor = { 0 };
+	size_t compressed_buf_size;
 	void *original_buf = NULL;
 	void *compressed_buf = NULL;
 	void *decompressed_buf = NULL;
-	struct compressor compressor;
-	struct decompressor decompressor;
 	tchar *default_file_list[] = { NULL };
 	int opt_char;
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
+		case '0':
 		case '1':
 		case '2':
 		case '3':
@@ -453,11 +572,30 @@ tmain(int argc, tchar *argv[])
 		case '8':
 		case '9':
 			level = parse_compression_level(opt_char, toptarg);
-			if (level == 0)
-				return -1;
+			if (level < 0)
+				return 1;
+			break;
+		case 'C':
+			compress_engine = name_to_engine(toptarg);
+			if (compress_engine == NULL) {
+				msg("invalid compression engine: \"%"TS"\"", toptarg);
+				show_available_engines(stderr);
+				return 1;
+			}
+			break;
+		case 'D':
+			decompress_engine = name_to_engine(toptarg);
+			if (decompress_engine == NULL) {
+				msg("invalid decompression engine: \"%"TS"\"", toptarg);
+				show_available_engines(stderr);
+				return 1;
+			}
+			break;
+		case 'e':
+			allow_expansion = true;
 			break;
 		case 'g':
-			wrapper = GZIP_WRAPPER;
+			format = GZIP_FORMAT;
 			break;
 		case 'h':
 			show_usage(stdout);
@@ -472,14 +610,14 @@ tmain(int argc, tchar *argv[])
 		case 'V':
 			show_version();
 			return 0;
-		case 'Y':
-			compress_with_libz = true;
+		case 'Y': /* deprecated, use '-C libz' instead */
+			compress_engine = &libz_engine;
 			break;
-		case 'Z':
-			decompress_with_libz = true;
+		case 'Z': /* deprecated, use '-D libz' instead */
+			decompress_engine = &libz_engine;
 			break;
 		case 'z':
-			wrapper = ZLIB_WRAPPER;
+			format = ZLIB_FORMAT;
 			break;
 		default:
 			show_usage(stderr);
@@ -490,22 +628,28 @@ tmain(int argc, tchar *argv[])
 	argc -= toptind;
 	argv += toptind;
 
+	if (level == 0)
+		allow_expansion = true;
+
+	ret = -1;
+	if (!compressor_init(&compressor, level, format, compress_engine))
+		goto out;
+	if (!decompressor_init(&decompressor, format, decompress_engine))
+		goto out;
+
+	if (allow_expansion)
+		compressed_buf_size = compress_bound(&compressor, chunk_size);
+	else
+		compressed_buf_size = chunk_size - 1;
+
 	original_buf = xmalloc(chunk_size);
-	compressed_buf = xmalloc(chunk_size - 1);
+	compressed_buf = xmalloc(compressed_buf_size);
 	decompressed_buf = xmalloc(chunk_size);
 
 	ret = -1;
 	if (original_buf == NULL || compressed_buf == NULL ||
 	    decompressed_buf == NULL)
-		goto out0;
-
-	ret = compressor_init(&compressor, level, wrapper, compress_with_libz);
-	if (ret)
-		goto out0;
-
-	ret = decompressor_init(&decompressor, wrapper, decompress_with_libz);
-	if (ret)
-		goto out1;
+		goto out;
 
 	if (argc == 0) {
 		argv = default_file_list;
@@ -516,41 +660,37 @@ tmain(int argc, tchar *argv[])
 				argv[i] = NULL;
 	}
 
-	printf("Benchmarking DEFLATE compression:\n");
+	printf("Benchmarking %s compression:\n",
+	       format == DEFLATE_FORMAT ? "DEFLATE" :
+	       format == ZLIB_FORMAT ? "zlib" : "gzip");
 	printf("\tCompression level: %d\n", level);
 	printf("\tChunk size: %"PRIu32"\n", chunk_size);
-	printf("\tWrapper: %s\n",
-	       wrapper == NO_WRAPPER ? "None" :
-	       wrapper == ZLIB_WRAPPER ? "zlib" : "gzip");
-	printf("\tCompression engine: %s\n",
-	       compress_with_libz ? "libz" : "libdeflate");
-	printf("\tDecompression engine: %s\n",
-	       decompress_with_libz ? "libz" : "libdeflate");
+	printf("\tCompression engine: %"TS"\n", compress_engine->name);
+	printf("\tDecompression engine: %"TS"\n", decompress_engine->name);
 
 	for (i = 0; i < argc; i++) {
 		struct file_stream in;
 
-		ret = xopen_for_read(argv[i], &in);
+		ret = xopen_for_read(argv[i], true, &in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 
 		printf("Processing %"TS"...\n", in.name);
 
 		ret = do_benchmark(&in, original_buf, compressed_buf,
-				   decompressed_buf, chunk_size, &compressor,
-				   &decompressor);
+				   decompressed_buf, chunk_size,
+				   allow_expansion, compressed_buf_size,
+				   &compressor, &decompressor);
 		xclose(&in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 	}
 	ret = 0;
-out2:
-	decompressor_destroy(&decompressor);
-out1:
-	compressor_destroy(&compressor);
-out0:
+out:
 	free(decompressed_buf);
 	free(compressed_buf);
 	free(original_buf);
+	decompressor_destroy(&decompressor);
+	compressor_destroy(&compressor);
 	return -ret;
 }

programs/checksum.c

@@ -0,0 +1,207 @@
+/*
+ * checksum.c - Adler-32 and CRC-32 checksumming program
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "test_util.h"
+
+static const tchar *const optstring = T("Ahs:tZ");
+
+static void
+show_usage(FILE *fp)
+{
+	fprintf(fp,
+"Usage: %"TS" [-A] [-h] [-s SIZE] [-t] [-Z] [FILE]...\n"
+"Calculate Adler-32 or CRC-32 checksums of the specified FILEs.\n"
+"\n"
+"Options:\n"
+"  -A        use Adler-32 (default is CRC-32)\n"
+"  -h        print this help\n"
+"  -s SIZE   chunk size\n"
+"  -t        show checksum speed, excluding I/O\n"
+"  -Z        use zlib implementation instead of libdeflate\n",
+	prog_invocation_name);
+}
+
+typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
+
+static u32
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
+}
+
+static u32
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
+{
+	return adler32(adler, buf, len);
+}
+
+static u32
+crc32_zlib(u32 crc, const void *buf, size_t len)
+{
+	return crc32(crc, buf, len);
+}
+
+static int
+checksum_stream(struct file_stream *in, cksum_fn_t cksum, u32 *sum,
+		void *buf, size_t bufsize, u64 *size_ret, u64 *elapsed_ret)
+{
+	u64 size = 0;
+	u64 elapsed = 0;
+
+	for (;;) {
+		ssize_t ret;
+		u64 start_time;
+
+		ret = xread(in, buf, bufsize);
+		if (ret < 0)
+			return ret;
+		if (ret == 0)
+			break;
+
+		size += ret;
+		start_time = timer_ticks();
+		*sum = cksum(*sum, buf, ret);
+		elapsed += timer_ticks() - start_time;
+	}
+
+	if (elapsed == 0)
+		elapsed = 1;
+	*size_ret = size;
+	*elapsed_ret = elapsed;
+	return 0;
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	bool use_adler32 = false;
+	bool use_zlib_impl = false;
+	bool do_timing = false;
+	void *buf;
+	size_t bufsize = 131072;
+	tchar *default_file_list[] = { NULL };
+	cksum_fn_t cksum;
+	int opt_char;
+	int i;
+	int ret;
+
+	begin_program(argv);
+
+	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
+		switch (opt_char) {
+		case 'A':
+			use_adler32 = true;
+			break;
+		case 'h':
+			show_usage(stdout);
+			return 0;
+		case 's':
+			bufsize = tstrtoul(toptarg, NULL, 10);
+			if (bufsize == 0) {
+				msg("invalid chunk size: \"%"TS"\"", toptarg);
+				return 1;
+			}
+			break;
+		case 't':
+			do_timing = true;
+			break;
+		case 'Z':
+			use_zlib_impl = true;
+			break;
+		default:
+			show_usage(stderr);
+			return 1;
+		}
+	}
+
+	argc -= toptind;
+	argv += toptind;
+
+	if (use_adler32) {
+		if (use_zlib_impl)
+			cksum = adler32_zlib;
+		else
+			cksum = adler32_libdeflate;
+	} else {
+		if (use_zlib_impl)
+			cksum = crc32_zlib;
+		else
+			cksum = crc32_libdeflate;
+	}
+
+	buf = xmalloc(bufsize);
+	if (buf == NULL)
+		return 1;
+
+	if (argc == 0) {
+		argv = default_file_list;
+		argc = ARRAY_LEN(default_file_list);
+	} else {
+		for (i = 0; i < argc; i++)
+			if (argv[i][0] == '-' && argv[i][1] == '\0')
+				argv[i] = NULL;
+	}
+
+	for (i = 0; i < argc; i++) {
+		struct file_stream in;
+		u32 sum = cksum(0, NULL, 0);
+		u64 size = 0;
+		u64 elapsed = 0;
+
+		ret = xopen_for_read(argv[i], true, &in);
+		if (ret != 0)
+			goto out;
+
+		ret = checksum_stream(&in, cksum, &sum, buf, bufsize,
+				      &size, &elapsed);
+		if (ret == 0) {
+			if (do_timing) {
+				printf("%08"PRIx32"\t%"TS"\t"
+				       "%"PRIu64" ms\t%"PRIu64" MB/s\n",
+				       sum, in.name, timer_ticks_to_ms(elapsed),
+				       timer_MB_per_s(size, elapsed));
+			} else {
+				printf("%08"PRIx32"\t%"TS"\t\n", sum, in.name);
+			}
+		}
+
+		xclose(&in);
+
+		if (ret != 0)
+			goto out;
+	}
+	ret = 0;
+out:
+	free(buf);
+	return -ret;
+}

programs/detect.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-if [ -z "$CC" ]; then
-	CC=cc
-fi
-
-echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
-echo "#ifndef _CONFIG_H"
-echo "#define _CONFIG_H"
-
-tmpfile="$(mktemp -t libdeflate_config.XXXXXXXX)"
-trap "rm -f \"$tmpfile\"" EXIT
-
-check_function() {
-	funcname="$1"
-	macro="HAVE_$(echo $funcname | tr a-z A-Z)"
-	echo "int main() { $funcname(); }" > "$tmpfile"
-	echo
-	echo "/* Is the $funcname() function available? */"
-	if $CC -x c $tmpfile -o /dev/null > /dev/null 2>&1; then
-		echo "#define $macro 1"
-	else
-		echo "/* $macro is not set */"
-	fi
-}
-
-check_function clock_gettime
-check_function futimens
-check_function futimes
-
-echo
-echo "#endif /* _CONFIG_H */"

programs/gzip.c

@@ -27,6 +27,7 @@
 
 #include "prog_util.h"
 
+#include <errno.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #ifdef _WIN32
@@ -37,22 +38,29 @@
 #  include <utime.h>
 #endif
 
+#define GZIP_MIN_HEADER_SIZE	10
+#define GZIP_FOOTER_SIZE	8
+#define GZIP_MIN_OVERHEAD	(GZIP_MIN_HEADER_SIZE + GZIP_FOOTER_SIZE)
+#define GZIP_ID1		0x1F
+#define GZIP_ID2		0x8B
+
 struct options {
 	bool to_stdout;
 	bool decompress;
 	bool force;
 	bool keep;
+	bool test;
 	int compression_level;
 	const tchar *suffix;
 };
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhkL:s:S:V");
+static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknqS:tV");
 
 static void
 show_usage(FILE *fp)
 {
 	fprintf(fp,
-"Usage: %"TS" [-LEVEL] [-cdfhkV] [-s SIZE] [-S SUF] FILE...\n"
+"Usage: %"TS" [-LEVEL] [-cdfhkqtV] [-S SUF] FILE...\n"
 "Compress or decompress the specified FILEs.\n"
 "\n"
 "Options:\n"
@@ -61,12 +69,16 @@ show_usage(FILE *fp)
 "  -12       slowest (best) compression\n"
 "  -c        write to standard output\n"
 "  -d        decompress\n"
-"  -f        overwrite existing output files\n"
+"  -f        overwrite existing output files; (de)compress hard-linked files;\n"
+"            allow reading/writing compressed data from/to terminal;\n"
+"            with gunzip -c, pass through non-gzipped data\n"
 "  -h        print this help\n"
 "  -k        don't delete input files\n"
-"  -S SUF    use suffix .SUF instead of .gz\n"
+"  -q        suppress warnings\n"
+"  -S SUF    use suffix SUF instead of .gz\n"
+"  -t        test file integrity\n"
 "  -V        show version and legal information\n",
-	program_invocation_name);
+	prog_invocation_name);
 }
 
 static void
@@ -86,10 +98,14 @@ show_version(void)
 static bool
 is_gunzip(void)
 {
-	if (tstrxcmp(program_invocation_name, T("gunzip")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip")) == 0)
+		return true;
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip")) == 0)
 		return true;
 #ifdef _WIN32
-	if (tstrxcmp(program_invocation_name, T("gunzip.exe")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip.exe")) == 0)
+		return true;
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip.exe")) == 0)
 		return true;
 #endif
 	return false;
@@ -98,10 +114,15 @@ is_gunzip(void)
 static const tchar *
 get_suffix(const tchar *path, const tchar *suffix)
 {
-	const tchar *dot = tstrrchr(get_filename(path), '.');
+	size_t path_len = tstrlen(path);
+	size_t suffix_len = tstrlen(suffix);
+	const tchar *p;
 
-	if (dot != NULL && tstrxcmp(dot + 1, suffix) == 0)
-		return dot;
+	if (path_len <= suffix_len)
+		return NULL;
+	p = &path[path_len - suffix_len];
+	if (tstrxcmp(p, suffix) == 0)
+		return p;
 	return NULL;
 }
 
@@ -111,8 +132,23 @@ has_suffix(const tchar *path, const tchar *suffix)
 	return get_suffix(path, suffix) != NULL;
 }
 
+static tchar *
+append_suffix(const tchar *path, const tchar *suffix)
+{
+	size_t path_len = tstrlen(path);
+	size_t suffix_len = tstrlen(suffix);
+	tchar *suffixed_path;
+
+	suffixed_path = xmalloc((path_len + suffix_len + 1) * sizeof(tchar));
+	if (suffixed_path == NULL)
+		return NULL;
+	tmemcpy(suffixed_path, path, path_len);
+	tmemcpy(&suffixed_path[path_len], suffix, suffix_len + 1);
+	return suffixed_path;
+}
+
 static int
-do_compress(struct deflate_compressor *compressor,
+do_compress(struct libdeflate_compressor *compressor,
 	    struct file_stream *in, struct file_stream *out)
 {
 	const void *uncompressed_data = in->mmap_mem;
@@ -122,7 +158,8 @@ do_compress(struct deflate_compressor *compressor,
 	size_t max_compressed_size;
 	int ret;
 
-	max_compressed_size = gzip_compress_bound(compressor, uncompressed_size);
+	max_compressed_size = libdeflate_gzip_compress_bound(compressor,
+							     uncompressed_size);
 	compressed_data = xmalloc(max_compressed_size);
 	if (compressed_data == NULL) {
 		msg("%"TS": file is probably too large to be processed by this "
@@ -131,12 +168,13 @@ do_compress(struct deflate_compressor *compressor,
 		goto out;
 	}
 
-	actual_compressed_size =
-		gzip_compress(compressor, uncompressed_data, uncompressed_size,
-			      compressed_data, max_compressed_size);
-
+	actual_compressed_size = libdeflate_gzip_compress(compressor,
+							  uncompressed_data,
+							  uncompressed_size,
+							  compressed_data,
+							  max_compressed_size);
 	if (actual_compressed_size == 0) {
-		msg("Bug in gzip_compress_bound()!");
+		msg("Bug in libdeflate_gzip_compress_bound()!");
 		ret = -1;
 		goto out;
 	}
@@ -155,72 +193,140 @@ load_u32_gzip(const u8 *p)
 }
 
 static int
-do_decompress(struct deflate_decompressor *decompressor,
-	      struct file_stream *in, struct file_stream *out)
+do_decompress(struct libdeflate_decompressor *decompressor,
+	      struct file_stream *in, struct file_stream *out,
+	      const struct options *options)
 {
 	const u8 *compressed_data = in->mmap_mem;
 	size_t compressed_size = in->mmap_size;
 	void *uncompressed_data = NULL;
 	size_t uncompressed_size;
-	enum decompress_result result;
-	int ret;
+	size_t max_uncompressed_size;
+	size_t actual_in_nbytes;
+	size_t actual_out_nbytes;
+	enum libdeflate_result result;
+	int ret = 0;
 
-	if (compressed_size < sizeof(u32)) {
-	       msg("%"TS": not in gzip format", in->name);
-	       ret = -1;
-	       goto out;
+	if (compressed_size < GZIP_MIN_OVERHEAD ||
+	    compressed_data[0] != GZIP_ID1 ||
+	    compressed_data[1] != GZIP_ID2) {
+		if (options->force)
+			return full_write(out, compressed_data, compressed_size);
+		msg("%"TS": not in gzip format", in->name);
+		return -1;
 	}
 
+	/*
+	 * Use the ISIZE field as a hint for the decompressed data size.  It may
+	 * need to be increased later, however, because the file may contain
+	 * multiple gzip members and the particular ISIZE we happen to use may
+	 * not be the largest; or the real size may be >= 4 GiB, causing ISIZE
+	 * to overflow.  In any case, make sure to allocate at least one byte.
+	 */
 	uncompressed_size = load_u32_gzip(&compressed_data[compressed_size - 4]);
+	if (uncompressed_size == 0)
+		uncompressed_size = 1;
 
-	uncompressed_data = xmalloc(uncompressed_size);
-	if (uncompressed_data == NULL) {
-		msg("%"TS": file is probably too large to be processed by this "
-		    "program", in->name);
-		ret = -1;
-		goto out;
-	}
+	/*
+	 * DEFLATE cannot expand data more than 1032x, so there's no need to
+	 * ever allocate a buffer more than 1032 times larger than the
+	 * compressed data.  This is a fail-safe, albeit not a very good one, if
+	 * ISIZE becomes corrupted on a small file.  (The 1032x number comes
+	 * from each 2 bits generating a 258-byte match.  This is a hard upper
+	 * bound; the real upper bound is slightly smaller due to overhead.)
+	 */
+	if (compressed_size <= SIZE_MAX / 1032)
+		max_uncompressed_size = compressed_size * 1032;
+	else
+		max_uncompressed_size = SIZE_MAX;
 
-	result = gzip_decompress(decompressor, compressed_data, compressed_size,
-				 uncompressed_data, uncompressed_size, NULL);
+	do {
+		if (uncompressed_data == NULL) {
+			uncompressed_size = MIN(uncompressed_size,
+						max_uncompressed_size);
+			uncompressed_data = xmalloc(uncompressed_size);
+			if (uncompressed_data == NULL) {
+				msg("%"TS": file is probably too large to be "
+				    "processed by this program", in->name);
+				ret = -1;
+				goto out;
+			}
+		}
 
-	if (result == DECOMPRESS_INSUFFICIENT_SPACE) {
-		msg("%"TS": file corrupt or too large to be processed by this "
-		    "program", in->name);
-		ret = -1;
-		goto out;
-	}
+		result = libdeflate_gzip_decompress_ex(decompressor,
+						       compressed_data,
+						       compressed_size,
+						       uncompressed_data,
+						       uncompressed_size,
+						       &actual_in_nbytes,
+						       &actual_out_nbytes);
 
-	if (result != DECOMPRESS_SUCCESS) {
-		msg("%"TS": file corrupt or not in gzip format", in->name);
-		ret = -1;
-		goto out;
-	}
+		if (result == LIBDEFLATE_INSUFFICIENT_SPACE) {
+			if (uncompressed_size >= max_uncompressed_size) {
+				msg("Bug in libdeflate_gzip_decompress_ex(): data expanded too much!");
+				ret = -1;
+				goto out;
+			}
+			if (uncompressed_size * 2 <= uncompressed_size) {
+				msg("%"TS": file corrupt or too large to be "
+				    "processed by this program", in->name);
+				ret = -1;
+				goto out;
+			}
+			uncompressed_size *= 2;
+			free(uncompressed_data);
+			uncompressed_data = NULL;
+			continue;
+		}
 
-	ret = full_write(out, uncompressed_data, uncompressed_size);
+		if (result != LIBDEFLATE_SUCCESS) {
+			msg("%"TS": file corrupt or not in gzip format",
+			    in->name);
+			ret = -1;
+			goto out;
+		}
+
+		if (actual_in_nbytes == 0 ||
+		    actual_in_nbytes > compressed_size ||
+		    actual_out_nbytes > uncompressed_size) {
+			msg("Bug in libdeflate_gzip_decompress_ex(): impossible actual_nbytes value!");
+			ret = -1;
+			goto out;
+		}
+
+		if (!options->test) {
+			ret = full_write(out, uncompressed_data, actual_out_nbytes);
+			if (ret != 0)
+				goto out;
+		}
+
+		compressed_data += actual_in_nbytes;
+		compressed_size -= actual_in_nbytes;
+
+	} while (compressed_size != 0);
 out:
 	free(uncompressed_data);
 	return ret;
 }
 
 static int
-stat_file(struct file_stream *in, struct stat *stbuf, bool allow_hard_links)
+stat_file(struct file_stream *in, stat_t *stbuf, bool allow_hard_links)
 {
-	if (fstat(in->fd, stbuf) != 0) {
+	if (tfstat(in->fd, stbuf) != 0) {
 		msg("%"TS": unable to stat file", in->name);
 		return -1;
 	}
 
 	if (!S_ISREG(stbuf->st_mode) && !in->is_standard_stream) {
-		msg("%"TS" is %s -- skipping",
-		    in->name, S_ISDIR(stbuf->st_mode) ? "a directory" :
-							"not a regular file");
+		warn("%"TS" is %s -- skipping",
+		     in->name, S_ISDIR(stbuf->st_mode) ? "a directory" :
+							 "not a regular file");
 		return -2;
 	}
 
 	if (stbuf->st_nlink > 1 && !allow_hard_links) {
-		msg("%"TS" has multiple hard links -- skipping "
-		    "(use -f to process anyway)", in->name);
+		warn("%"TS" has multiple hard links -- skipping (use -f to process anyway)",
+		     in->name);
 		return -2;
 	}
 
@@ -228,7 +334,7 @@ stat_file(struct file_stream *in, struct stat *stbuf, bool allow_hard_links)
 }
 
 static void
-restore_mode(struct file_stream *out, const struct stat *stbuf)
+restore_mode(struct file_stream *out, const stat_t *stbuf)
 {
 #ifndef _WIN32
 	if (fchmod(out->fd, stbuf->st_mode) != 0)
@@ -237,7 +343,7 @@ restore_mode(struct file_stream *out, const struct stat *stbuf)
 }
 
 static void
-restore_owner_and_group(struct file_stream *out, const struct stat *stbuf)
+restore_owner_and_group(struct file_stream *out, const stat_t *stbuf)
 {
 #ifndef _WIN32
 	if (fchown(out->fd, stbuf->st_uid, stbuf->st_gid) != 0) {
@@ -249,33 +355,33 @@ restore_owner_and_group(struct file_stream *out, const struct stat *stbuf)
 
 static void
 restore_timestamps(struct file_stream *out, const tchar *newpath,
-		   const struct stat *stbuf)
+		   const stat_t *stbuf)
 {
 	int ret;
-#if defined(HAVE_FUTIMENS)
+#if defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION)
 	struct timespec times[2] = {
 		stbuf->st_atim, stbuf->st_mtim,
 	};
 	ret = futimens(out->fd, times);
-#elif defined(HAVE_FUTIMES)
+#elif defined(HAVE_FUTIMES) && defined(HAVE_STAT_NANOSECOND_PRECISION)
 	struct timeval times[2] = {
 		{ stbuf->st_atim.tv_sec, stbuf->st_atim.tv_nsec / 1000, },
 		{ stbuf->st_mtim.tv_sec, stbuf->st_mtim.tv_nsec / 1000, },
 	};
 	ret = futimes(out->fd, times);
-#else /* HAVE_FUTIMES */
+#else
 	struct tutimbuf times = {
 		stbuf->st_atime, stbuf->st_mtime,
 	};
 	ret = tutime(newpath, &times);
-#endif /* !HAVE_FUTIMES */
+#endif
 	if (ret != 0)
 		msg_errno("%"TS": unable to preserve timestamps", out->name);
 }
 
 static void
 restore_metadata(struct file_stream *out, const tchar *newpath,
-		 const struct stat *stbuf)
+		 const stat_t *stbuf)
 {
 	restore_mode(out, stbuf);
 	restore_owner_and_group(out, stbuf);
@@ -283,32 +389,57 @@ restore_metadata(struct file_stream *out, const tchar *newpath,
 }
 
 static int
-decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
+decompress_file(struct libdeflate_decompressor *decompressor, const tchar *path,
 		const struct options *options)
 {
+	tchar *oldpath = (tchar *)path;
 	tchar *newpath = NULL;
 	struct file_stream in;
 	struct file_stream out;
-	struct stat stbuf;
+	stat_t stbuf;
 	int ret;
 	int ret2;
 
-	if (path != NULL && !options->to_stdout) {
+	if (path != NULL) {
 		const tchar *suffix = get_suffix(path, options->suffix);
 		if (suffix == NULL) {
-			msg("\"%"TS"\" does not end with the .%"TS" suffix -- "
-			    "skipping", path, options->suffix);
-			ret = -2;
-			goto out;
+			/*
+			 * Input file is unsuffixed.  If the file doesn't exist,
+			 * then try it suffixed.  Otherwise, if we're not
+			 * writing to stdout, skip the file with warning status.
+			 * Otherwise, go ahead and try to open the file anyway
+			 * (which will very likely fail).
+			 */
+			if (tstat(path, &stbuf) != 0 && errno == ENOENT) {
+				oldpath = append_suffix(path, options->suffix);
+				if (oldpath == NULL)
+					return -1;
+				if (!options->to_stdout)
+					newpath = (tchar *)path;
+			} else if (!options->to_stdout) {
+				warn("\"%"TS"\" does not end with the %"TS" suffix -- skipping",
+				     path, options->suffix);
+				return -2;
+			}
+		} else if (!options->to_stdout) {
+			/*
+			 * Input file is suffixed, and we're not writing to
+			 * stdout.  Strip the suffix to get the path to the
+			 * output file.
+			 */
+			newpath = xmalloc((suffix - oldpath + 1) *
+					  sizeof(tchar));
+			if (newpath == NULL)
+				return -1;
+			tmemcpy(newpath, oldpath, suffix - oldpath);
+			newpath[suffix - oldpath] = '\0';
 		}
-		newpath = xmalloc((suffix - path + 1) * sizeof(tchar));
-		tmemcpy(newpath, path, suffix - path);
-		newpath[suffix - path] = '\0';
 	}
 
-	ret = xopen_for_read(path, &in);
+	ret = xopen_for_read(oldpath, options->force || options->to_stdout,
+			     &in);
 	if (ret != 0)
-		goto out_free_newpath;
+		goto out_free_paths;
 
 	if (!options->force && isatty(in.fd)) {
 		msg("Refusing to read compressed data from terminal.  "
@@ -317,7 +448,8 @@ decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
 		goto out_close_in;
 	}
 
-	ret = stat_file(&in, &stbuf, options->force || newpath == NULL);
+	ret = stat_file(&in, &stbuf, options->force || options->keep ||
+			oldpath == NULL || newpath == NULL);
 	if (ret != 0)
 		goto out_close_in;
 
@@ -325,15 +457,16 @@ decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
 	if (ret != 0)
 		goto out_close_in;
 
+	/* TODO: need a streaming-friendly solution */
 	ret = map_file_contents(&in, stbuf.st_size);
 	if (ret != 0)
 		goto out_close_out;
 
-	ret = do_decompress(decompressor, &in, &out);
+	ret = do_decompress(decompressor, &in, &out, options);
 	if (ret != 0)
 		goto out_close_out;
 
-	if (path != NULL && newpath != NULL)
+	if (oldpath != NULL && newpath != NULL)
 		restore_metadata(&out, newpath, &stbuf);
 	ret = 0;
 out_close_out:
@@ -344,49 +477,44 @@ out_close_out:
 		tunlink(newpath);
 out_close_in:
 	xclose(&in);
-	if (ret == 0 && path != NULL && newpath != NULL && !options->keep)
-		tunlink(path);
-out_free_newpath:
-	free(newpath);
-out:
+	if (ret == 0 && oldpath != NULL && newpath != NULL && !options->keep)
+		tunlink(oldpath);
+out_free_paths:
+	if (newpath != path)
+		free(newpath);
+	if (oldpath != path)
+		free(oldpath);
 	return ret;
 }
 
 static int
-compress_file(struct deflate_compressor *compressor, const tchar *path,
+compress_file(struct libdeflate_compressor *compressor, const tchar *path,
 	      const struct options *options)
 {
 	tchar *newpath = NULL;
 	struct file_stream in;
 	struct file_stream out;
-	struct stat stbuf;
+	stat_t stbuf;
 	int ret;
 	int ret2;
 
 	if (path != NULL && !options->to_stdout) {
-		size_t path_nchars, suffix_nchars;
-
 		if (!options->force && has_suffix(path, options->suffix)) {
-			msg("%"TS": already has .%"TS" suffix -- skipping",
+			msg("%"TS": already has %"TS" suffix -- skipping",
 			    path, options->suffix);
-			ret = -2;
-			goto out;
+			return 0;
 		}
-		path_nchars = tstrlen(path);
-		suffix_nchars = tstrlen(options->suffix);
-		newpath = xmalloc((path_nchars + 1 + suffix_nchars + 1) *
-					sizeof(tchar));
-		tmemcpy(newpath, path, path_nchars);
-		newpath[path_nchars] = '.';
-		tmemcpy(&newpath[path_nchars + 1], options->suffix,
-			suffix_nchars + 1);
+		newpath = append_suffix(path, options->suffix);
+		if (newpath == NULL)
+			return -1;
 	}
 
-	ret = xopen_for_read(path, &in);
+	ret = xopen_for_read(path, options->force || options->to_stdout, &in);
 	if (ret != 0)
 		goto out_free_newpath;
 
-	ret = stat_file(&in, &stbuf, options->force || newpath == NULL);
+	ret = stat_file(&in, &stbuf, options->force || options->keep ||
+			path == NULL || newpath == NULL);
 	if (ret != 0)
 		goto out_close_in;
 
@@ -401,8 +529,9 @@ compress_file(struct deflate_compressor *compressor, const tchar *path,
 		goto out_close_out;
 	}
 
+	/* TODO: need a streaming-friendly solution */
 	ret = map_file_contents(&in, stbuf.st_size);
-	if (ret)
+	if (ret != 0)
 		goto out_close_out;
 
 	ret = do_compress(compressor, &in, &out);
@@ -424,26 +553,27 @@ out_close_in:
 		tunlink(path);
 out_free_newpath:
 	free(newpath);
-out:
 	return ret;
 }
 
 int
 tmain(int argc, tchar *argv[])
 {
+	tchar *default_file_list[] = { NULL };
 	struct options options;
 	int opt_char;
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	options.to_stdout = false;
 	options.decompress = is_gunzip();
 	options.force = false;
 	options.keep = false;
+	options.test = false;
 	options.compression_level = 6;
-	options.suffix = T("gz");
+	options.suffix = T(".gz");
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
@@ -458,8 +588,8 @@ tmain(int argc, tchar *argv[])
 		case '9':
 			options.compression_level =
 				parse_compression_level(opt_char, toptarg);
-			if (options.compression_level == 0)
-				return -1;
+			if (options.compression_level < 0)
+				return 1;
 			break;
 		case 'c':
 			options.to_stdout = true;
@@ -476,8 +606,34 @@ tmain(int argc, tchar *argv[])
 		case 'k':
 			options.keep = true;
 			break;
+		case 'n':
+			/*
+			 * -n means don't save or restore the original filename
+			 *  in the gzip header.  Currently this implementation
+			 *  already behaves this way by default, so accept the
+			 *  option as a no-op.
+			 */
+			break;
+		case 'q':
+			suppress_warnings = true;
+			break;
 		case 'S':
 			options.suffix = toptarg;
+			if (options.suffix[0] == T('\0')) {
+				msg("invalid suffix");
+				return 1;
+			}
+			break;
+		case 't':
+			options.test = true;
+			options.decompress = true;
+			options.to_stdout = true;
+			/*
+			 * -t behaves just like the more commonly used -c
+			 * option, except that -t doesn't actually write
+			 * anything.  For ease of implementation, just pretend
+			 * that -c was specified too.
+			 */
 			break;
 		case 'V':
 			show_version();
@@ -492,20 +648,17 @@ tmain(int argc, tchar *argv[])
 	argc -= toptind;
 
 	if (argc == 0) {
-		show_usage(stderr);
-		return 1;
-	}
-	for (i = 0; i < argc; i++) {
-		if (argv[i][0] == '-' && argv[i][1] == '\0') {
-			msg("This implementation of gzip does not yet "
-			    "support reading from standard input.");
-			return 1;
-		}
+		argv = default_file_list;
+		argc = ARRAY_LEN(default_file_list);
+	} else {
+		for (i = 0; i < argc; i++)
+			if (argv[i][0] == '-' && argv[i][1] == '\0')
+				argv[i] = NULL;
 	}
 
 	ret = 0;
 	if (options.decompress) {
-		struct deflate_decompressor *d;
+		struct libdeflate_decompressor *d;
 
 		d = alloc_decompressor();
 		if (d == NULL)
@@ -514,9 +667,9 @@ tmain(int argc, tchar *argv[])
 		for (i = 0; i < argc; i++)
 			ret |= -decompress_file(d, argv[i], &options);
 
-		deflate_free_decompressor(d);
+		libdeflate_free_decompressor(d);
 	} else {
-		struct deflate_compressor *c;
+		struct libdeflate_compressor *c;
 
 		c = alloc_compressor(options.compression_level);
 		if (c == NULL)
@@ -525,16 +678,20 @@ tmain(int argc, tchar *argv[])
 		for (i = 0; i < argc; i++)
 			ret |= -compress_file(c, argv[i], &options);
 
-		deflate_free_compressor(c);
+		libdeflate_free_compressor(c);
 	}
 
-	/*
-	 * If ret=0, there were no warnings or errors.  Exit with status 0.
-	 * If ret=2, there was at least one warning.  Exit with status 2.
-	 * Else, there was at least one error.  Exit with status 1.
-	 */
-	if (ret != 0 && ret != 2)
-		ret = 1;
-
-	return ret;
+	switch (ret) {
+	case 0:
+		/* No warnings or errors */
+		return 0;
+	case 2:
+		/* At least one warning, but no errors */
+		if (suppress_warnings)
+			return 0;
+		return 2;
+	default:
+		/* At least one error */
+		return 1;
+	}
 }

programs/prog_util.c

@@ -30,24 +30,41 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdarg.h>
-#include <time.h>
 #ifdef _WIN32
 #  include <windows.h>
 #else
 #  include <unistd.h>
 #  include <sys/mman.h>
-#  include <sys/time.h>
+#endif
+
+#ifndef O_BINARY
+#  define O_BINARY 0
+#endif
+#ifndef O_SEQUENTIAL
+#  define O_SEQUENTIAL 0
+#endif
+#ifndef O_NOFOLLOW
+#  define O_NOFOLLOW 0
+#endif
+#ifndef O_NONBLOCK
+#  define O_NONBLOCK 0
+#endif
+#ifndef O_NOCTTY
+#  define O_NOCTTY 0
 #endif
 
 /* The invocation name of the program (filename component only) */
-const tchar *program_invocation_name;
+const tchar *prog_invocation_name;
+
+/* Whether to suppress warning messages or not */
+bool suppress_warnings;
 
 static void
 do_msg(const char *format, bool with_errno, va_list va)
 {
 	int saved_errno = errno;
 
-	fprintf(stderr, "%"TS": ", program_invocation_name);
+	fprintf(stderr, "%"TS": ", prog_invocation_name);
 	vfprintf(stderr, format, va);
 	if (with_errno)
 		fprintf(stderr, ": %s\n", strerror(saved_errno));
@@ -79,6 +96,20 @@ msg_errno(const char *format, ...)
 	va_end(va);
 }
 
+
+/* Same as msg(), but do nothing if 'suppress_warnings' has been set. */
+void
+warn(const char *format, ...)
+{
+	if (!suppress_warnings) {
+		va_list va;
+
+		va_start(va, format);
+		do_msg(format, false, va);
+		va_end(va);
+	}
+}
+
 /* malloc() wrapper */
 void *
 xmalloc(size_t size)
@@ -91,28 +122,6 @@ xmalloc(size_t size)
 	return p;
 }
 
-/*
- * Retrieve the current time in nanoseconds since a start time which is fixed
- * for the duration of program execution but is otherwise unspecified
- */
-u64
-current_time(void)
-{
-#ifdef _WIN32
-	FILETIME ft;
-	GetSystemTimeAsFileTime(&ft);
-	return 100 * (((u64)ft.dwHighDateTime << 32) | ft.dwLowDateTime);
-#elif defined(HAVE_CLOCK_GETTIME)
-	struct timespec ts;
-	clock_gettime(CLOCK_MONOTONIC, &ts);
-	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
-#else
-	struct timeval tv;
-	gettimeofday(&tv, NULL);
-	return (1000000000 * (u64)tv.tv_sec) + (1000 * (u64)tv.tv_usec);
-#endif
-}
-
 /*
  * Retrieve a pointer to the filename component of the specified path.
  *
@@ -120,7 +129,7 @@ current_time(void)
  * properly for directories, since a path to a directory might have trailing
  * slashes.
  */
-const tchar *
+static const tchar *
 get_filename(const tchar *path)
 {
 	const tchar *slash = tstrrchr(path, '/');
@@ -134,6 +143,17 @@ get_filename(const tchar *path)
 	return path;
 }
 
+void
+begin_program(tchar *argv[])
+{
+	prog_invocation_name = get_filename(argv[0]);
+
+#ifdef FREESTANDING
+	/* This allows testing freestanding library builds. */
+	libdeflate_set_memory_allocator(malloc, free);
+#endif
+}
+
 /* Create a copy of 'path' surrounded by double quotes */
 static tchar *
 quote_path(const tchar *path)
@@ -153,8 +173,9 @@ quote_path(const tchar *path)
 
 /* Open a file for reading, or set up standard input for reading */
 int
-xopen_for_read(const tchar *path, struct file_stream *strm)
+xopen_for_read(const tchar *path, bool symlink_ok, struct file_stream *strm)
 {
+	strm->mmap_token = NULL;
 	strm->mmap_mem = NULL;
 
 	if (path == NULL) {
@@ -173,13 +194,18 @@ xopen_for_read(const tchar *path, struct file_stream *strm)
 	if (strm->name == NULL)
 		return -1;
 
-	strm->fd = topen(path, O_RDONLY | O_BINARY | O_NOFOLLOW);
+	strm->fd = topen(path, O_RDONLY | O_BINARY | O_NONBLOCK | O_NOCTTY |
+			 (symlink_ok ? 0 : O_NOFOLLOW) | O_SEQUENTIAL);
 	if (strm->fd < 0) {
 		msg_errno("Can't open %"TS" for reading", strm->name);
 		free(strm->name);
 		return -1;
 	}
 
+#if defined(HAVE_POSIX_FADVISE) && (O_SEQUENTIAL == 0)
+	(void)posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+
 	return 0;
 }
 
@@ -187,6 +213,9 @@ xopen_for_read(const tchar *path, struct file_stream *strm)
 int
 xopen_for_write(const tchar *path, bool overwrite, struct file_stream *strm)
 {
+	int ret = -1;
+
+	strm->mmap_token = NULL;
 	strm->mmap_mem = NULL;
 
 	if (path == NULL) {
@@ -214,13 +243,14 @@ retry:
 		}
 		if (!overwrite) {
 			if (!isatty(STDERR_FILENO) || !isatty(STDIN_FILENO)) {
-				msg("%"TS" already exists; use -f to overwrite",
-				    strm->name);
+				warn("%"TS" already exists; use -f to overwrite",
+				     strm->name);
+				ret = -2; /* warning only */
 				goto err;
 			}
 			fprintf(stderr, "%"TS": %"TS" already exists; "
 				"overwrite? (y/n) ",
-				program_invocation_name, strm->name);
+				prog_invocation_name, strm->name);
 			if (getchar() != 'y') {
 				msg("Not overwriting.");
 				goto err;
@@ -237,13 +267,60 @@ retry:
 
 err:
 	free(strm->name);
-	return -1;
+	return ret;
+}
+
+/* Read the full contents of a file into memory */
+static int
+read_full_contents(struct file_stream *strm)
+{
+	size_t filled = 0;
+	size_t capacity = 4096;
+	char *buf;
+	int ret;
+
+	buf = xmalloc(capacity);
+	if (buf == NULL)
+		return -1;
+	do {
+		if (filled == capacity) {
+			char *newbuf;
+
+			if (capacity == SIZE_MAX)
+				goto oom;
+			capacity += MIN(SIZE_MAX - capacity, capacity);
+			newbuf = realloc(buf, capacity);
+			if (newbuf == NULL)
+				goto oom;
+			buf = newbuf;
+		}
+		ret = xread(strm, &buf[filled], capacity - filled);
+		if (ret < 0)
+			goto err;
+		filled += ret;
+	} while (ret != 0);
+
+	strm->mmap_mem = buf;
+	strm->mmap_size = filled;
+	return 0;
+
+err:
+	free(buf);
+	return ret;
+oom:
+	msg("Out of memory!  %"TS" is too large to be processed by "
+	    "this program as currently implemented.", strm->name);
+	ret = -1;
+	goto err;
 }
 
 /* Map the contents of a file into memory */
 int
 map_file_contents(struct file_stream *strm, u64 size)
 {
+	if (size == 0) /* mmap isn't supported on empty files */
+		return read_full_contents(strm);
+
 	if (size > SIZE_MAX) {
 		msg("%"TS" is too large to be processed by this program",
 		    strm->name);
@@ -254,8 +331,11 @@ map_file_contents(struct file_stream *strm, u64 size)
 				(HANDLE)(intptr_t)_get_osfhandle(strm->fd),
 				NULL, PAGE_READONLY, 0, 0, NULL);
 	if (strm->mmap_token == NULL) {
+		DWORD err = GetLastError();
+		if (err == ERROR_BAD_EXE_FORMAT) /* mmap unsupported */
+			return read_full_contents(strm);
 		msg("Unable create file mapping for %"TS": Windows error %u",
-		    strm->name, (unsigned int)GetLastError());
+		    strm->name, (unsigned int)err);
 		return -1;
 	}
 
@@ -267,10 +347,12 @@ map_file_contents(struct file_stream *strm, u64 size)
 		CloseHandle((HANDLE)strm->mmap_token);
 		return -1;
 	}
-#else
+#else /* _WIN32 */
 	strm->mmap_mem = mmap(NULL, size, PROT_READ, MAP_SHARED, strm->fd, 0);
 	if (strm->mmap_mem == MAP_FAILED) {
 		strm->mmap_mem = NULL;
+		if (errno == ENODEV) /* mmap isn't supported on this file */
+			return read_full_contents(strm);
 		if (errno == ENOMEM) {
 			msg("%"TS" is too large to be processed by this "
 			    "program", strm->name);
@@ -280,7 +362,13 @@ map_file_contents(struct file_stream *strm, u64 size)
 		}
 		return -1;
 	}
+
+#ifdef HAVE_POSIX_MADVISE
+	(void)posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL);
 #endif
+	strm->mmap_token = strm; /* anything that's not NULL */
+
+#endif /* !_WIN32 */
 	strm->mmap_size = size;
 	return 0;
 }
@@ -300,6 +388,8 @@ xread(struct file_stream *strm, void *buf, size_t count)
 		if (res == 0)
 			break;
 		if (res < 0) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
 			msg_errno("Error reading from %"TS, strm->name);
 			return -1;
 		}
@@ -309,39 +399,6 @@ xread(struct file_stream *strm, void *buf, size_t count)
 	return orig_count - count;
 }
 
-/* Skip over 'count' bytes from a file, returning 0 on success or -1 on error */
-int
-skip_bytes(struct file_stream *strm, size_t count)
-{
-	size_t bufsize;
-	char *buffer;
-	ssize_t ret;
-
-	if (count == 0)
-		return 0;
-
-	bufsize = MIN(count, 4096);
-	buffer = xmalloc(bufsize);
-	if (buffer == NULL)
-		return -1;
-	do {
-		size_t n = MIN(count, bufsize);
-		ret = xread(strm, buffer, n);
-		if (ret < 0)
-			goto out;
-		if (ret != n) {
-			msg("%"TS": unexpected end-of-file", strm->name);
-			ret = -1;
-			goto out;
-		}
-		count -= ret;
-	} while (count != 0);
-	ret = 0;
-out:
-	free(buffer);
-	return ret;
-}
-
 /* Write to a file, returning 0 if all bytes were written or -1 on error */
 int
 full_write(struct file_stream *strm, const void *buf, size_t count)
@@ -365,23 +422,27 @@ int
 xclose(struct file_stream *strm)
 {
 	int ret = 0;
-	if (strm->fd >= 0 && !strm->is_standard_stream) {
+
+	if (!strm->is_standard_stream) {
 		if (close(strm->fd) != 0) {
 			msg_errno("Error closing %"TS, strm->name);
 			ret = -1;
 		}
 		free(strm->name);
-
-		if (strm->mmap_mem != NULL) {
-#ifdef _WIN32
-			UnmapViewOfFile(strm->mmap_mem);
-			CloseHandle((HANDLE)strm->mmap_token);
-#else
-			munmap(strm->mmap_mem, strm->mmap_size);
-#endif
-			strm->mmap_mem = NULL;
-		}
 	}
+
+	if (strm->mmap_token != NULL) {
+#ifdef _WIN32
+		UnmapViewOfFile(strm->mmap_mem);
+		CloseHandle((HANDLE)strm->mmap_token);
+#else
+		munmap(strm->mmap_mem, strm->mmap_size);
+#endif
+		strm->mmap_token = NULL;
+	} else {
+		free(strm->mmap_mem);
+	}
+	strm->mmap_mem = NULL;
 	strm->fd = -1;
 	strm->name = NULL;
 	return ret;
@@ -389,36 +450,48 @@ xclose(struct file_stream *strm)
 
 /*
  * Parse the compression level given on the command line, returning the
- * compression level on success or 0 on error
+ * compression level on success or -1 on error
  */
 int
 parse_compression_level(tchar opt_char, const tchar *arg)
 {
-	unsigned long level = opt_char - '0';
-	const tchar *p;
+	int level;
 
 	if (arg == NULL)
 		arg = T("");
 
-	for (p = arg; *p >= '0' && *p <= '9'; p++)
-		level = (level * 10) + (*p - '0');
+	if (opt_char < '0' || opt_char > '9')
+		goto invalid;
+	level = opt_char - '0';
 
-	if (level < 1 || level > 12 || *p != '\0') {
-		msg("Invalid compression level: \"%"TC"%"TS"\".  "
-		    "Must be an integer in the range [1, 12].", opt_char, arg);
-		return 0;
+	if (arg[0] != '\0') {
+		if (arg[0] < '0' || arg[0] > '9')
+			goto invalid;
+		if (arg[1] != '\0')	/* Levels are at most 2 digits */
+			goto invalid;
+		if (level == 0)		/* Don't allow arguments like "-01" */
+			goto invalid;
+		level = (level * 10) + (arg[0] - '0');
 	}
 
+	if (level < 0 || level > 12)
+		goto invalid;
+
 	return level;
+
+invalid:
+	msg("Invalid compression level: \"%"TC"%"TS"\".  "
+	    "Must be an integer in the range [0, 12].", opt_char, arg);
+	return -1;
 }
 
 /* Allocate a new DEFLATE compressor */
-struct deflate_compressor *
+struct libdeflate_compressor *
 alloc_compressor(int level)
 {
-	struct deflate_compressor *c;
+	struct libdeflate_compressor *c;
 
-	c = deflate_alloc_compressor(level);
+	c = libdeflate_alloc_compressor(level);
 	if (c == NULL) {
 		msg_errno("Unable to allocate compressor with "
 			  "compression level %d", level);
@@ -427,12 +500,12 @@ alloc_compressor(int level)
 }
 
 /* Allocate a new DEFLATE decompressor */
-struct deflate_decompressor *
+struct libdeflate_decompressor *
 alloc_decompressor(void)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 	if (d == NULL)
 		msg_errno("Unable to allocate decompressor");
 

programs/prog_util.h

@@ -38,8 +38,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#ifndef _WIN32
+#  include <sys/types.h>
+#endif
 
-#include "common_defs.h"
+#include "../common/common_defs.h"
 
 #ifdef __GNUC__
 # define _printf(str_idx, args_idx)	\
@@ -48,6 +51,27 @@
 # define _printf(str_idx, args_idx)
 #endif
 
+#ifdef _MSC_VER
+/*
+ * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
+ * inttypes.h.  Work around this by defining the PRI* macros ourselves.
+ */
+# define PRIu8  "hhu"
+# define PRIu16 "hu"
+# define PRIu32 "u"
+# define PRIu64 "llu"
+# define PRIi8  "hhi"
+# define PRIi16 "hi"
+# define PRIi32 "i"
+# define PRIi64 "lli"
+# define PRIx8  "hhx"
+# define PRIx16 "hx"
+# define PRIx32 "x"
+# define PRIx64 "llx"
+#else
+# include <inttypes.h>
+#endif
+
 #ifdef _WIN32
 
 /*
@@ -57,7 +81,7 @@
  */
 
 #include <wchar.h>
-extern int wmain(int argc, wchar_t **argv);
+int wmain(int argc, wchar_t **argv);
 #  define	tmain		wmain
 #  define	tchar		wchar_t
 #  define	_T(text)	L##text
@@ -68,27 +92,23 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		_wopen
 #  define	tstrchr		wcschr
 #  define	tstrcmp		wcscmp
-#  define	tstrcpy		wcscpy
 #  define	tstrlen		wcslen
 #  define	tstrrchr	wcsrchr
 #  define	tstrtoul	wcstoul
 #  define	tstrxcmp	wcsicmp
 #  define	tunlink		_wunlink
-#  define	tutimbuf	_utimbuf
-#  define	tutime		_wutime
+#  define	tutimbuf	__utimbuf64
+#  define	tutime		_wutime64
+#  define	tstat		_wstat64
+#  define	tfstat		_fstat64
+#  define	stat_t		struct _stat64
 #  ifdef _MSC_VER
-     /* Standard file descriptors?  What are those? */
 #    define	STDIN_FILENO	0
 #    define	STDOUT_FILENO	1
 #    define	STDERR_FILENO	2
-
-     /* Fix the broken stat-related definitions. */
-#    define	fstat		_fstat
-#    define	stat		_stat
 #    define	S_ISREG(m)      (((m) & S_IFMT) == S_IFREG)
 #    define	S_ISDIR(m)      (((m) & S_IFMT) == S_IFDIR)
 #  endif
-#  define	O_NOFOLLOW	0
 
 #else /* _WIN32 */
 
@@ -103,7 +123,6 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		open
 #  define	tstrchr		strchr
 #  define	tstrcmp		strcmp
-#  define	tstrcpy		strcpy
 #  define	tstrlen		strlen
 #  define	tstrrchr	strrchr
 #  define	tstrtoul	strtoul
@@ -111,20 +130,22 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	tunlink		unlink
 #  define	tutimbuf	utimbuf
 #  define	tutime		utime
-#  define	O_BINARY	0
+#  define	tstat		stat
+#  define	tfstat		fstat
+#  define	stat_t		struct stat
 
 #endif /* !_WIN32 */
 
-extern const tchar *program_invocation_name;
+extern const tchar *prog_invocation_name;
+extern bool suppress_warnings;
 
-extern void _printf(1, 2) msg(const char *fmt, ...);
-extern void _printf(1, 2) msg_errno(const char *fmt, ...);
+void _printf(1, 2) msg(const char *fmt, ...);
+void _printf(1, 2) msg_errno(const char *fmt, ...);
+void _printf(1, 2) warn(const char *fmt, ...);
 
-extern void *xmalloc(size_t size);
+void *xmalloc(size_t size);
 
-extern u64 current_time(void);
-
-extern const tchar *get_filename(const tchar *path);
+void begin_program(tchar *argv[]);
 
 struct file_stream {
 	int fd;
@@ -135,27 +156,26 @@ struct file_stream {
 	size_t mmap_size;
 };
 
-extern int xopen_for_read(const tchar *path, struct file_stream *strm);
-extern int xopen_for_write(const tchar *path, bool force,
-			   struct file_stream *strm);
-extern int map_file_contents(struct file_stream *strm, u64 size);
+int xopen_for_read(const tchar *path, bool symlink_ok,
+		   struct file_stream *strm);
+int xopen_for_write(const tchar *path, bool force, struct file_stream *strm);
+int map_file_contents(struct file_stream *strm, u64 size);
 
-extern ssize_t xread(struct file_stream *strm, void *buf, size_t count);
-extern int skip_bytes(struct file_stream *strm, size_t count);
-extern int full_write(struct file_stream *strm, const void *buf, size_t count);
+ssize_t xread(struct file_stream *strm, void *buf, size_t count);
+int full_write(struct file_stream *strm, const void *buf, size_t count);
 
-extern int xclose(struct file_stream *strm);
+int xclose(struct file_stream *strm);
 
-extern int parse_compression_level(tchar opt_char, const tchar *arg);
+int parse_compression_level(tchar opt_char, const tchar *arg);
 
-extern struct deflate_compressor *alloc_compressor(int level);
-extern struct deflate_decompressor *alloc_decompressor(void);
+struct libdeflate_compressor *alloc_compressor(int level);
+struct libdeflate_decompressor *alloc_decompressor(void);
 
 /* tgetopt.c */
 
 extern tchar *toptarg;
 extern int toptind, topterr, toptopt;
 
-extern int tgetopt(int argc, tchar *argv[], const tchar *optstring);
+int tgetopt(int argc, tchar *argv[], const tchar *optstring);
 
 #endif /* PROGRAMS_PROG_UTIL_H */

programs/test_checksums.c

@@ -0,0 +1,196 @@
+/*
+ * test_checksums.c
+ *
+ * Verify that libdeflate's Adler-32 and CRC-32 functions produce the same
+ * results as their zlib equivalents.
+ */
+
+#include <stdlib.h>
+#include <time.h>
+
+#include "test_util.h"
+
+static unsigned int rng_seed;
+
+typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
+
+static u32
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
+}
+
+static u32
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
+{
+	return adler32(adler, buf, len);
+}
+
+static u32
+crc32_zlib(u32 crc, const void *buf, size_t len)
+{
+	return crc32(crc, buf, len);
+}
+
+static u32
+select_initial_crc(void)
+{
+	if (rand() & 1)
+		return 0;
+	return ((u32)rand() << 16) | rand();
+}
+
+static u32
+select_initial_adler(void)
+{
+	u32 lo, hi;
+
+	if (rand() & 1)
+		return 1;
+
+	lo = (rand() % 4 == 0 ? 65520 : rand() % 65521);
+	hi = (rand() % 4 == 0 ? 65520 : rand() % 65521);
+	return (hi << 16) | lo;
+}
+
+static void
+test_initial_values(cksum_fn_t cksum, u32 expected)
+{
+	ASSERT(cksum(0, NULL, 0) == expected);
+	if (cksum != adler32_zlib) /* broken */
+		ASSERT(cksum(0, NULL, 1) == expected);
+	ASSERT(cksum(0, NULL, 1234) == expected);
+	ASSERT(cksum(1234, NULL, 0) == expected);
+	ASSERT(cksum(1234, NULL, 1234) == expected);
+}
+
+static void
+test_multipart(const u8 *buffer, size_t size, const char *name,
+	       cksum_fn_t cksum, u32 v, u32 expected)
+{
+	size_t division = rand() % (size + 1);
+	v = cksum(v, buffer, division);
+	v = cksum(v, buffer + division, size - division);
+	if (v != expected) {
+		fprintf(stderr, "%s checksum failed multipart test\n", name);
+		ASSERT(0);
+	}
+}
+
+static void
+test_checksums(const void *buffer, size_t size, const char *name,
+	       cksum_fn_t cksum1, cksum_fn_t cksum2, u32 initial_value)
+{
+	u32 v1 = cksum1(initial_value, buffer, size);
+	u32 v2 = cksum2(initial_value, buffer, size);
+
+	if (v1 != v2) {
+		fprintf(stderr, "%s checksum mismatch\n", name);
+		fprintf(stderr, "initial_value=0x%08"PRIx32", buffer=%p, "
+			"size=%zu, buffer=", initial_value, buffer, size);
+		for (size_t i = 0; i < MIN(size, 256); i++)
+			fprintf(stderr, "%02x", ((const u8 *)buffer)[i]);
+		if (size > 256)
+			fprintf(stderr, "...");
+		fprintf(stderr, "\n");
+		ASSERT(0);
+	}
+
+	if ((rand() & 15) == 0) {
+		test_multipart(buffer, size, name, cksum1, initial_value, v1);
+		test_multipart(buffer, size, name, cksum2, initial_value, v1);
+	}
+}
+
+static void
+test_crc32(const void *buffer, size_t size, u32 initial_value)
+{
+	test_checksums(buffer, size, "CRC-32",
+		       crc32_libdeflate, crc32_zlib, initial_value);
+}
+
+static void
+test_adler32(const void *buffer, size_t size, u32 initial_value)
+{
+	test_checksums(buffer, size, "Adler-32",
+		       adler32_libdeflate, adler32_zlib, initial_value);
+}
+
+static void test_random_buffers(u8 *buffer, u8 *guarded_buf_end,
+				size_t limit, u32 num_iter)
+{
+	for (u32 i = 0; i < num_iter; i++) {
+		size_t start = rand() % limit;
+		size_t len = rand() % (limit - start);
+		u32 a0 = select_initial_adler();
+		u32 c0 = select_initial_crc();
+
+		for (size_t j = start; j < start + len; j++)
+			buffer[j] = rand();
+
+		/* Test with chosen size and alignment */
+		test_adler32(&buffer[start], len, a0);
+		test_crc32(&buffer[start], len, c0);
+
+		/* Test with chosen size, with guard page after input buffer */
+		memcpy(guarded_buf_end - len, &buffer[start], len);
+		test_adler32(guarded_buf_end - len, len, a0);
+		test_crc32(guarded_buf_end - len, len, c0);
+	}
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 *buffer = xmalloc(32768);
+	u8 *guarded_buf_start, *guarded_buf_end;
+
+	begin_program(argv);
+
+	alloc_guarded_buffer(32768, &guarded_buf_start, &guarded_buf_end);
+
+	rng_seed = time(NULL);
+	srand(rng_seed);
+
+	test_initial_values(adler32_libdeflate, 1);
+	test_initial_values(adler32_zlib, 1);
+	test_initial_values(crc32_libdeflate, 0);
+	test_initial_values(crc32_zlib, 0);
+
+	/* Test different buffer sizes and alignments */
+	test_random_buffers(buffer, guarded_buf_end, 256, 5000);
+	test_random_buffers(buffer, guarded_buf_end, 1024, 500);
+	test_random_buffers(buffer, guarded_buf_end, 32768, 50);
+
+	/*
+	 * Test Adler-32 overflow cases.  For example, given all 0xFF bytes and
+	 * the highest possible initial (s1, s2) of (65520, 65520), then s2 if
+	 * stored as a 32-bit unsigned integer will overflow if > 5552 bytes are
+	 * processed.  Implementations must make sure to reduce s2 modulo 65521
+	 * before that point.  Also, some implementations make use of 16-bit
+	 * counters which can overflow earlier.
+	 */
+	memset(buffer, 0xFF, 32768);
+	for (u32 i = 0; i < 20; i++) {
+		u32 initial_value;
+
+		if (i == 0)
+			initial_value = ((u32)65520 << 16) | 65520;
+		else
+			initial_value = select_initial_adler();
+
+		test_adler32(buffer, 5553, initial_value);
+		test_adler32(buffer, rand() % 32769, initial_value);
+		buffer[rand() % 32768] = 0xFE;
+	}
+
+	free(buffer);
+	free_guarded_buffer(guarded_buf_start, guarded_buf_end);
+	return 0;
+}

programs/test_custom_malloc.c

@@ -0,0 +1,85 @@
+/*
+ * test_custom_malloc.c
+ *
+ * Test libdeflate_set_memory_allocator().
+ * Also test injecting allocation failures.
+ */
+
+#include "test_util.h"
+
+static int malloc_count = 0;
+static int free_count = 0;
+
+static void *do_malloc(size_t size)
+{
+	malloc_count++;
+	return malloc(size);
+}
+
+static void *do_fail_malloc(size_t size)
+{
+	malloc_count++;
+	return NULL;
+}
+
+static void do_free(void *ptr)
+{
+	free_count++;
+	free(ptr);
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	int level;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+
+	begin_program(argv);
+
+	/* Test that the custom allocator is actually used when requested. */
+
+	libdeflate_set_memory_allocator(do_malloc, do_free);
+	ASSERT(malloc_count == 0);
+	ASSERT(free_count == 0);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c != NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+		libdeflate_free_compressor(c);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 1);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+	libdeflate_free_decompressor(d);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 1);
+
+	/* As long as we're here, also test injecting allocation failures. */
+
+	libdeflate_set_memory_allocator(do_fail_malloc, do_free);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c == NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d == NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+
+	return 0;
+}

programs/test_incomplete_codes.c

@@ -0,0 +1,385 @@
+/*
+ * test_incomplete_codes.c
+ *
+ * Test that the decompressor accepts incomplete Huffman codes in certain
+ * specific cases.
+ */
+
+#include "test_util.h"
+
+static void
+verify_decompression_libdeflate(const u8 *in, size_t in_nbytes,
+				u8 *out, size_t out_nbytes_avail,
+				const u8 *expected_out,
+				size_t expected_out_nbytes)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	size_t actual_out_nbytes;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	res = libdeflate_deflate_decompress(d, in, in_nbytes,
+					    out, out_nbytes_avail,
+					    &actual_out_nbytes);
+	ASSERT(res == LIBDEFLATE_SUCCESS);
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	libdeflate_free_decompressor(d);
+}
+
+static void
+verify_decompression_zlib(const u8 *in, size_t in_nbytes,
+			  u8 *out, size_t out_nbytes_avail,
+			  const u8 *expected_out, size_t expected_out_nbytes)
+{
+	z_stream z;
+	int res;
+	size_t actual_out_nbytes;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	z.next_in = (void *)in;
+	z.avail_in = in_nbytes;
+	z.next_out = (void *)out;
+	z.avail_out = out_nbytes_avail;
+	res = inflate(&z, Z_FINISH);
+	ASSERT(res == Z_STREAM_END);
+	actual_out_nbytes = out_nbytes_avail - z.avail_out;
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	inflateEnd(&z);
+}
+
+static void
+verify_decompression(const u8 *in, size_t in_nbytes,
+		     u8 *out, size_t out_nbytes_avail,
+		     const u8 *expected_out, size_t expected_out_nbytes)
+{
+	verify_decompression_libdeflate(in, in_nbytes, out, out_nbytes_avail,
+					expected_out, expected_out_nbytes);
+	verify_decompression_zlib(in, in_nbytes, out, out_nbytes_avail,
+				  expected_out, expected_out_nbytes);
+
+}
+
+/* Test that an empty offset code is accepted. */
+static void
+test_empty_offset_code(void)
+{
+	static const u8 expected_out[] = { 'A', 'B', 'A', 'A' };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Generate a DEFLATE stream containing a "dynamic Huffman" block
+	 * containing literals, but no offsets; and having an empty offset code
+	 * (all codeword lengths set to 0).
+	 *
+	 * Litlen code:
+	 *	litlensym_A			freq=3 len=1 codeword= 0
+	 *	litlensym_B			freq=1 len=2 codeword=01
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..'A'-1]	= 0	presym_18
+	 *	['A']		= 1	presym_1
+	 *	['B']		= 2	presym_2
+	 *	['B'+1..255]	= 0	presym_18 presym_18
+	 *	[256]		= 2	presym_2
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=3 codeword=011
+	 *	presym_1	freq=1 len=3 codeword=111
+	 *	presym_2	freq=2 len=2 codeword= 01
+	 *	presym_18	freq=3 len=1 codeword=  0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_0: len=3 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_1: len=3 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 54, 7));	/* presym_18, 65 zeroes */
+	ASSERT(put_bits(&os, 0x7, 3));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 89, 7));	/* presym_18, 100 zeroes */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 78, 7));	/* presym_18, 89 zeroes */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 3));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_B */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that a litrunlen code containing only one symbol is accepted. */
+static void
+test_singleton_litrunlen_code(void)
+{
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..256]	= 0	presym_18 presym_18
+	 *	[256]		= 1	presym_1
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=2 codeword=01
+	 *	presym_1	freq=1 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 13; i++)	/* presym_{8,...,14}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		ASSERT(put_bits(&os, 0, 1) &&	/* presym_18, 128 zeroes */
+		       put_bits(&os, 117, 7));
+	}
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out), in, 0);
+}
+
+/* Test that an offset code containing only one symbol is accepted. */
+static void
+test_singleton_offset_code(void)
+{
+	static const u8 expected_out[] = { 255, 255, 255, 255 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_255			freq=1 len=1 codeword= 0
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=01
+	 *	litlensym_257 (len 3)		freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_0 (offset 0)		freq=1 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..254] = 0	presym_{18,18}
+	 *	[255]	 = 1	presym_1
+	 *	[256]	 = 1	presym_2
+	 *	[257]	 = 1	presym_2
+	 *	[258]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=2 codeword=01
+	 *	presym_2	freq=2 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	for (i = 0; i < 12; i++)	/* presym_{0,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 127 zeroes */
+	       put_bits(&os, 116, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+
+	/* Literal */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_0 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that an offset code containing only one symbol is accepted, even if that
+ * symbol is not symbol 0.  The codeword should be '0' in either case. */
+static void
+test_singleton_offset_code_notsymzero(void)
+{
+	static const u8 expected_out[] = { 254, 255, 254, 255, 254 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_254			len=2 codeword=00
+	 *	litlensym_255			len=2 codeword=10
+	 *	litlensym_256 (end-of-block)	len=2 codeword=01
+	 *	litlensym_257 (len 3)		len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_1 (offset 2)		len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..253] = 0	presym_{18,18}
+	 *	[254]	 = 2	presym_2
+	 *	[255]	 = 2	presym_2
+	 *	[256]	 = 2	presym_2
+	 *	[257]	 = 2	presym_2
+	 *	[258]	 = 0	presym_0
+	 *	[259]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_0	len=2 codeword=00
+	 *	presym_1	len=2 codeword=10
+	 *	presym_2	len=2 codeword=01
+	 *	presym_18	len=2 codeword=11
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 1, 5));	/* num_offset_syms: 1 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_18: len=2 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 126 zeroes */
+	       put_bits(&os, 115, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 2));	/* presym_0 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* presym_1 */
+
+	/* Literals */
+	ASSERT(put_bits(&os, 0x0, 2));	/* litlensym_254 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_1 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	begin_program(argv);
+
+	test_empty_offset_code();
+	test_singleton_litrunlen_code();
+	test_singleton_offset_code();
+	test_singleton_offset_code_notsymzero();
+
+	return 0;
+}

programs/test_litrunlen_overflow.c

@@ -0,0 +1,72 @@
+/*
+ * test_litrunlen_overflow.c
+ *
+ * Regression test for commit f2f0df727444 ("deflate_compress: fix corruption
+ * with long literal run").  Try to compress a file longer than 65535 bytes
+ * where no 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes
+ * apart, and the distribution of bytes remains constant throughout, and yet not
+ * all bytes are used so the data is still slightly compressible.  There will be
+ * no matches in this data, but the compressor should still output a compressed
+ * block, and this block should contain more than 65535 consecutive literals,
+ * which triggered the bug.
+ *
+ * Note: on random data, this situation is extremely unlikely if the compressor
+ * uses all matches it finds, since random data will on average have a 3-byte
+ * match every (256**3)/32768 = 512 bytes.
+ */
+
+#include "test_util.h"
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t data_size = 2 * 250 * 251;
+	u8 *orig_data, *compressed_data, *decompressed_data;
+	int i, stride, multiple, j = 0;
+	struct libdeflate_decompressor *d;
+	static const int levels[] = { 3, 6, 12 };
+
+	begin_program(argv);
+
+	orig_data = xmalloc(data_size);
+	compressed_data = xmalloc(data_size);
+	decompressed_data = xmalloc(data_size);
+
+	for (i = 0; i < 2; i++) {
+		for (stride = 1; stride < 251; stride++) {
+			for (multiple = 0; multiple < 251; multiple++)
+				orig_data[j++] = (stride * multiple) % 251;
+		}
+	}
+	ASSERT(j == data_size);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(levels); i++) {
+		struct libdeflate_compressor *c;
+		size_t csize;
+		enum libdeflate_result res;
+
+		c = libdeflate_alloc_compressor(levels[i]);
+		ASSERT(c != NULL);
+
+		csize = libdeflate_deflate_compress(c, orig_data, data_size,
+						    compressed_data, data_size);
+		ASSERT(csize > 0 && csize < data_size);
+
+		res = libdeflate_deflate_decompress(d, compressed_data, csize,
+						    decompressed_data,
+						    data_size, NULL);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(memcmp(orig_data, decompressed_data, data_size) == 0);
+
+		libdeflate_free_compressor(c);
+	}
+
+	libdeflate_free_decompressor(d);
+	free(orig_data);
+	free(compressed_data);
+	free(decompressed_data);
+	return 0;
+}

programs/test_overread.c

@@ -0,0 +1,95 @@
+/*
+ * test_overread.c
+ *
+ * Test that the decompressor doesn't produce an unbounded amount of output if
+ * it runs out of input, even when implicit zeroes appended to the input would
+ * continue producing output (as is the case when the input ends during a
+ * DYNAMIC_HUFFMAN block where a literal has an all-zeroes codeword).
+ *
+ * This is a regression test for commit 3f21ec9d6121 ("deflate_decompress: error
+ * out if overread count gets too large").
+ */
+
+#include "test_util.h"
+
+static void
+generate_test_input(struct output_bitstream *os)
+{
+	int i;
+
+	put_bits(os, 0, 1);	/* BFINAL: 0 */
+	put_bits(os, 2, 2);	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Write the Huffman codes.
+	 *
+	 * Litlen code:
+	 *	litlensym_0   (0)		len=1 codeword=0
+	 *	litlensym_256 (end-of-block)	len=1 codeword=1
+	 * Offset code:
+	 *	offsetsym_0 (unused)		len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0]	 = 1	presym_1
+	 *	[1..255] = 0	presym_{18,18}
+	 *	[256]	 = 1	presym_1
+	 *	[257]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	len=1 codeword=0
+	 *	presym_18	len=1 codeword=1
+	 */
+	put_bits(os, 0, 5);	/* num_litlen_syms: 0 + 257 */
+	put_bits(os, 0, 5);	/* num_offset_syms: 0 + 1 */
+	put_bits(os, 14, 4);	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	put_bits(os, 0, 3);		/* presym_16: len=0 */
+	put_bits(os, 0, 3);		/* presym_17: len=0 */
+	put_bits(os, 1, 3);		/* presym_18: len=1 */
+	for (i = 0; i < 14; i++)	/* presym_{0,...,14}: len=0 */
+		put_bits(os, 0, 3);
+	put_bits(os, 1, 3);		/* presym_1: len=1 */
+
+	/* Litlen and offset codeword lengths */
+	put_bits(os, 0, 1);		/* presym_1 */
+	put_bits(os, 1, 1);		/* presym_18 ... */
+	put_bits(os, 117, 7);		/* ... 11 + 117 zeroes */
+	put_bits(os, 1, 1);		/* presym_18 ... */
+	put_bits(os, 116, 7);		/* ... 11 + 116 zeroes */
+	put_bits(os, 0, 1);		/* presym_1 */
+	put_bits(os, 0, 1);		/* presym_1 */
+
+	/* Implicit zeroes would generate endless literals from here. */
+
+	ASSERT(flush_bits(os));
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 cdata[16];
+	u8 udata[256];
+	struct output_bitstream os =
+		{ .next = cdata, .end = cdata + sizeof(cdata) };
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	size_t actual_out_nbytes;
+
+	begin_program(argv);
+
+	generate_test_input(&os);
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	res = libdeflate_deflate_decompress(d, cdata, os.next - cdata,
+					    udata, sizeof(udata),
+					    &actual_out_nbytes);
+	/* Before the fix, the result was LIBDEFLATE_INSUFFICIENT_SPACE here. */
+	ASSERT(res == LIBDEFLATE_BAD_DATA);
+
+	libdeflate_free_decompressor(d);
+	return 0;
+}

programs/test_slow_decompression.c

@@ -0,0 +1,472 @@
+/*
+ * test_slow_decompression.c
+ *
+ * Test how quickly libdeflate decompresses degenerate/malicious compressed data
+ * streams that start new Huffman blocks extremely frequently.
+ */
+
+#include "test_util.h"
+
+/*
+ * Generate a DEFLATE stream containing all empty "static Huffman" blocks.
+ *
+ * libdeflate used to decompress this very slowly (~1000x slower than typical
+ * data), but now it's much faster (only ~2x slower than typical data) because
+ * now it skips rebuilding the decode tables for the static Huffman codes when
+ * they're already loaded into the decompressor.
+ */
+static void
+generate_empty_static_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (put_bits(&os, 0, 1) &&	/* BFINAL: 0 */
+	       put_bits(&os, 1, 2) &&	/* BTYPE: STATIC_HUFFMAN */
+	       put_bits(&os, 0, 7))	/* litlensym_256 (end-of-block) */
+		;
+}
+
+static bool
+generate_empty_dynamic_huffman_block(struct output_bitstream *os)
+{
+	int i;
+
+	if (!put_bits(os, 0, 1))	/* BFINAL: 0 */
+		return false;
+	if (!put_bits(os, 2, 2))	/* BTYPE: DYNAMIC_HUFFMAN */
+		return false;
+
+	/*
+	 * Write a minimal Huffman code, then the end-of-block symbol.
+	 *
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	offsetsym_0 (unused)		freq=0 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..255] = 0	presym_{18,18}
+	 *	[256]	 = 1	presym_1
+	 *	[257]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=1 codeword=0
+	 *	presym_18	freq=2 len=1 codeword=1
+	 */
+
+	if (!put_bits(os, 0, 5))	/* num_litlen_syms: 0 + 257 */
+		return false;
+	if (!put_bits(os, 0, 5))	/* num_offset_syms: 0 + 1 */
+		return false;
+	if (!put_bits(os, 14, 4))	/* num_explicit_precode_lens: 14 + 4 */
+		return false;
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++) {	/* presym_{16,17}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_18: len=1 */
+		return false;
+	for (i = 0; i < 14; i++) {	/* presym_{0,...,14}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_1: len=1 */
+		return false;
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		if (!put_bits(os, 1, 1) || /* presym_18, 128 zeroes */
+		    !put_bits(os, 117, 7))
+			return false;
+	}
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	/* Done writing the Huffman codes */
+
+	return put_bits(os, 0, 1);	/* litlensym_256 (end-of-block) */
+}
+
+/*
+ * Generate a DEFLATE stream containing all empty "dynamic Huffman" blocks.
+ *
+ * This is the worst known case currently, being ~100x slower to decompress than
+ * typical data.
+ */
+static void
+generate_empty_dynamic_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (generate_empty_dynamic_huffman_block(&os))
+		;
+}
+
+#define NUM_ITERATIONS	100
+
+static u64
+do_test_libdeflate(const char *input_type, const u8 *in, size_t in_nbytes,
+		   u8 *out, size_t out_nbytes_avail)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	u64 t;
+	int i;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		res = libdeflate_deflate_decompress(d, in, in_nbytes, out,
+						    out_nbytes_avail, NULL);
+		ASSERT(res == LIBDEFLATE_BAD_DATA ||
+		       res == LIBDEFLATE_INSUFFICIENT_SPACE);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, libdeflate]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	libdeflate_free_decompressor(d);
+	return t;
+}
+
+static u64
+do_test_zlib(const char *input_type, const u8 *in, size_t in_nbytes,
+	     u8 *out, size_t out_nbytes_avail)
+{
+	z_stream z;
+	int res;
+	u64 t;
+	int i;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		inflateReset(&z);
+		z.next_in = (void *)in;
+		z.avail_in = in_nbytes;
+		z.next_out = out;
+		z.avail_out = out_nbytes_avail;
+		res = inflate(&z, Z_FINISH);
+		ASSERT(res == Z_BUF_ERROR || res == Z_DATA_ERROR);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, zlib      ]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	inflateEnd(&z);
+	return t;
+}
+
+/*
+ * Test case from https://github.com/ebiggers/libdeflate/issues/33
+ * with the gzip header and footer removed to leave just the DEFLATE stream
+ */
+static const u8 orig_repro[3962] =
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\x04\xea\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x00\x01\x04\x00\x3f\x00\x00\x00\x00"
+	"\x28\xf7\xff\x00\xff\xff\xff\xff\x00\x00";
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 in[4096];
+	u8 out[10000];
+	u64 t, tz;
+
+	begin_program(argv);
+
+	begin_performance_test();
+
+	/* static huffman case */
+	generate_empty_static_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("static huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("static huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is faster than zlib in this case, e.g.
+	 *	[static huffman, libdeflate]: 215861 KB/s
+	 *	[static huffman, zlib      ]: 73651 KB/s
+	 */
+	putchar('\n');
+	ASSERT(t < tz);
+
+	/* dynamic huffman case */
+	generate_empty_dynamic_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("dynamic huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("dynamic huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is slower than zlib in this case, though not super bad.
+	 *	[dynamic huffman, libdeflate]: 6277 KB/s
+	 *	[dynamic huffman, zlib      ]: 10419 KB/s
+	 * FIXME: make it faster.
+	 */
+	putchar('\n');
+	ASSERT(t < 4 * tz);
+
+	/* original reproducer */
+	t = do_test_libdeflate("original repro", orig_repro, sizeof(orig_repro),
+			       out, sizeof(out));
+	tz = do_test_zlib("original repro", orig_repro, sizeof(orig_repro),
+			  out, sizeof(out));
+	ASSERT(t < tz);
+
+	return 0;
+}

programs/test_trailing_bytes.c

@@ -0,0 +1,152 @@
+/*
+ * test_trailing_bytes.c
+ *
+ * Test that decompression correctly stops at the end of the first DEFLATE,
+ * zlib, or gzip stream, and doesn't process any additional trailing bytes.
+ */
+
+#include "test_util.h"
+
+static const struct {
+	size_t (LIBDEFLATEAPI *compress)(
+			struct libdeflate_compressor *compressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_out_nbytes_ret);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress_ex)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_in_nbytes_ret,
+			size_t *actual_out_nbytes_ret);
+} codecs[] = {
+	{
+		.compress = libdeflate_deflate_compress,
+		.decompress = libdeflate_deflate_decompress,
+		.decompress_ex = libdeflate_deflate_decompress_ex,
+	}, {
+		.compress = libdeflate_zlib_compress,
+		.decompress = libdeflate_zlib_decompress,
+		.decompress_ex = libdeflate_zlib_decompress_ex,
+	}, {
+		.compress = libdeflate_gzip_compress,
+		.decompress = libdeflate_gzip_decompress,
+		.decompress_ex = libdeflate_gzip_decompress_ex,
+	}
+};
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t original_nbytes = 32768;
+	const size_t compressed_nbytes_total = 32768;
+	/*
+	 * Don't use the full buffer for compressed data, because we want to
+	 * test whether decompression can deal with additional trailing bytes.
+	 *
+	 * Note: we can't use a guarded buffer (i.e. a buffer where the byte
+	 * after compressed_nbytes is unmapped) because the decompressor may
+	 * read a few bytes beyond the end of the stream (but ultimately not
+	 * actually use those bytes) as long as they are within the buffer.
+	 */
+	const size_t compressed_nbytes_avail = 30000;
+	size_t i;
+	u8 *original;
+	u8 *compressed;
+	u8 *decompressed;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	size_t compressed_nbytes;
+	enum libdeflate_result res;
+	size_t actual_compressed_nbytes;
+	size_t actual_decompressed_nbytes;
+
+	begin_program(argv);
+
+	ASSERT(compressed_nbytes_avail < compressed_nbytes_total);
+
+	/* Prepare some dummy data to compress */
+	original = xmalloc(original_nbytes);
+	ASSERT(original != NULL);
+	for (i = 0; i < original_nbytes; i++)
+		original[i] = (i % 123) + (i % 1023);
+
+	compressed = xmalloc(compressed_nbytes_total);
+	ASSERT(compressed != NULL);
+	memset(compressed, 0, compressed_nbytes_total);
+
+	decompressed = xmalloc(original_nbytes);
+	ASSERT(decompressed != NULL);
+
+	c = libdeflate_alloc_compressor(6);
+	ASSERT(c != NULL);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(codecs); i++) {
+		compressed_nbytes = codecs[i].compress(c, original,
+						       original_nbytes,
+						       compressed,
+						       compressed_nbytes_avail);
+		ASSERT(compressed_nbytes > 0);
+		ASSERT(compressed_nbytes <= compressed_nbytes_avail);
+
+		/* Test decompress() of stream that fills the whole buffer */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed, compressed_nbytes,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream that fills the whole buffer */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed, compressed_nbytes,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress() of stream with trailing bytes */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed,
+					   compressed_nbytes_total,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream with trailing bytes */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed,
+					      compressed_nbytes_total,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+	}
+
+	free(original);
+	free(compressed);
+	free(decompressed);
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

programs/test_util.c

@@ -0,0 +1,243 @@
+/*
+ * test_util.c - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _WIN32
+/* for MAP_ANONYMOUS or MAP_ANON, which unfortunately aren't part of POSIX... */
+#  undef _POSIX_C_SOURCE
+#  ifdef __APPLE__
+#    define _DARWIN_C_SOURCE
+#  elif defined(__linux__)
+#    define _GNU_SOURCE
+#  endif
+#endif
+
+#include "test_util.h"
+
+#include <fcntl.h>
+#include <time.h>
+#ifdef _WIN32
+#  include <windows.h>
+#else
+#  include <unistd.h>
+#  include <sys/mman.h>
+#  include <sys/time.h>
+#endif
+
+#ifndef MAP_ANONYMOUS
+#  define MAP_ANONYMOUS MAP_ANON
+#endif
+
+/* Abort with an error message */
+_noreturn void
+assertion_failed(const char *expr, const char *file, int line)
+{
+	msg("Assertion failed: %s at %s:%d", expr, file, line);
+	abort();
+}
+
+void
+begin_performance_test(void)
+{
+	/* Skip performance tests by default, since they can be flaky. */
+	if (getenv("INCLUDE_PERF_TESTS") == NULL)
+		exit(0);
+}
+
+static size_t
+get_page_size(void)
+{
+#ifdef _WIN32
+	SYSTEM_INFO info;
+
+	GetSystemInfo(&info);
+	return info.dwPageSize;
+#else
+	return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+/* Allocate a buffer with guard pages */
+void
+alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret)
+{
+	const size_t pagesize = get_page_size();
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	u8 *base_addr;
+	u8 *start, *end;
+#ifdef _WIN32
+	DWORD oldProtect;
+#endif
+
+	*start_ret = NULL;
+	*end_ret = NULL;
+
+#ifdef _WIN32
+	/* Allocate buffer and guard pages with no access. */
+	base_addr = VirtualAlloc(NULL, (nr_pages + 2) * pagesize,
+				 MEM_COMMIT | MEM_RESERVE, PAGE_NOACCESS);
+	if (!base_addr) {
+		msg("Unable to allocate memory (VirtualAlloc): Windows error %u",
+		    (unsigned int)GetLastError());
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Grant read+write access to just the buffer. */
+	if (!VirtualProtect(start, end - start, PAGE_READWRITE, &oldProtect)) {
+		msg("Unable to protect memory (VirtualProtect): Windows error %u",
+		    (unsigned int)GetLastError());
+		VirtualFree(base_addr, 0, MEM_RELEASE);
+		ASSERT(0);
+	}
+#else
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	if (base_addr == (u8 *)MAP_FAILED) {
+		msg_errno("Unable to allocate memory (anonymous mmap)");
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+#endif
+	*start_ret = start;
+	*end_ret = end;
+}
+
+/* Free a buffer that was allocated by alloc_guarded_buffer() */
+void
+free_guarded_buffer(u8 *start, u8 *end)
+{
+	if (!start)
+		return;
+#ifdef _WIN32
+	VirtualFree(start - get_page_size(), 0, MEM_RELEASE);
+#else
+	munmap(start, end - start);
+#endif
+}
+
+/*
+ * Return the number of timer ticks that have elapsed since some unspecified
+ * point fixed at the start of program execution
+ */
+u64
+timer_ticks(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER count;
+
+	QueryPerformanceCounter(&count);
+	return count.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
+#else
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (1000000 * (u64)tv.tv_sec) + tv.tv_usec;
+#endif
+}
+
+/*
+ * Return the number of timer ticks per second
+ */
+static u64
+timer_frequency(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return freq.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	return 1000000000;
+#else
+	return 1000000;
+#endif
+}
+
+/*
+ * Convert a number of elapsed timer ticks to milliseconds
+ */
+u64 timer_ticks_to_ms(u64 ticks)
+{
+	return ticks * 1000 / timer_frequency();
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to MB/s
+ */
+u64 timer_MB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000000;
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to KB/s
+ */
+u64 timer_KB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000;
+}
+
+bool
+put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits)
+{
+	os->bitbuf |= bits << os->bitcount;
+	os->bitcount += num_bits;
+	while (os->bitcount >= 8) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	return true;
+}
+
+bool
+flush_bits(struct output_bitstream *os)
+{
+	while (os->bitcount > 0) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	os->bitcount = 0;
+	return true;
+}

programs/test_util.h

@@ -0,0 +1,67 @@
+/*
+ * test_util.h - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PROGRAMS_TEST_UTIL_H
+#define PROGRAMS_TEST_UTIL_H
+
+#include "prog_util.h"
+
+#include <zlib.h> /* for comparison purposes */
+
+#ifdef __GNUC__
+# define _noreturn __attribute__((noreturn))
+#else
+# define _noreturn
+#endif
+
+void _noreturn
+assertion_failed(const char *expr, const char *file, int line);
+
+#define ASSERT(expr) { if (unlikely(!(expr))) \
+	assertion_failed(#expr, __FILE__, __LINE__); }
+
+void begin_performance_test(void);
+
+void alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret);
+void free_guarded_buffer(u8 *start, u8 *end);
+
+u64 timer_ticks(void);
+u64 timer_ticks_to_ms(u64 ticks);
+u64 timer_MB_per_s(u64 bytes, u64 ticks);
+u64 timer_KB_per_s(u64 bytes, u64 ticks);
+
+struct output_bitstream {
+	machine_word_t bitbuf;
+	int bitcount;
+	u8 *next;
+	u8 *end;
+};
+
+bool put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits);
+bool flush_bits(struct output_bitstream *os);
+
+#endif /* PROGRAMS_TEST_UTIL_H */

scripts/afl-fuzz/.gitignore

@@ -0,0 +1 @@
+*/fuzz

scripts/afl-fuzz/Makefile

@@ -1,7 +1,6 @@
 SRC := $(wildcard */*.c)
 EXE := $(SRC:.c=)
 
-CFLAGS := -O2 -s
 LDLIBS := -ldeflate
 LDFLAGS := -L../..
 CPPFLAGS := -I../..

scripts/afl-fuzz/deflate_compress/fuzz.c

@@ -0,0 +1,47 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+int main(int argc, char **argv)
+{
+	struct libdeflate_decompressor *d;
+	struct libdeflate_compressor *c;
+	int ret;
+	int fd = open(argv[1], O_RDONLY);
+	struct stat stbuf;
+	unsigned char level;
+	assert(fd >= 0);
+	ret = fstat(fd, &stbuf);
+	assert(!ret);
+
+	if (stbuf.st_size == 0)
+		return 0;
+	ret = read(fd, &level, 1);
+	assert(ret == 1);
+	level %= 13;
+
+	char in[stbuf.st_size - 1];
+	ret = read(fd, in, sizeof in);
+	assert(ret == sizeof in);
+
+	c = libdeflate_alloc_compressor(level);
+	d = libdeflate_alloc_decompressor();
+
+	char out[sizeof(in)];
+	char checkarray[sizeof(in)];
+
+	size_t csize = libdeflate_deflate_compress(c, in,sizeof in, out, sizeof out);
+	if (csize) {
+		enum libdeflate_result res;
+		res = libdeflate_deflate_decompress(d, out, csize, checkarray, sizeof in, NULL);
+		assert(!res);
+		assert(!memcmp(in, checkarray, sizeof in));
+	}
+
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

scripts/afl-fuzz/deflate_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/afl-fuzz/fuzz.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+set -e -u -o pipefail
+
+cd "$(dirname "$0")"
+
+read -r -a AVAILABLE_TARGETS < <(echo */fuzz.c | sed 's@/fuzz.c@@g')
+
+usage()
+{
+	cat << EOF
+Usage: $0 [OPTION]... [TARGET]...
+
+Fuzz libdeflate with afl-fuzz.
+
+Options:
+   --asan          Enable AddressSanitizer
+   --no-resume     Don't resume existing afl-fuzz session; start a new one
+   --ubsan         Enable UndefinedBehaviorSanitizer
+
+Available targets: ${AVAILABLE_TARGETS[*]}
+EOF
+}
+
+die()
+{
+	echo "$*" 1>&2
+	exit 1
+}
+
+asan=false
+ubsan=false
+may_resume=true
+
+longopts_array=(
+asan
+help
+no-resume
+ubsan
+)
+longopts=$(echo "${longopts_array[@]}" | tr ' ' ',')
+
+if ! options=$(getopt -o "" -l "$longopts" -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+eval set -- "$options"
+while (( $# >= 0 )); do
+	case "$1" in
+	--asan)
+		asan=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--no-resume)
+		may_resume=false
+		;;
+	--ubsan)
+		ubsan=true
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Invalid option: \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+if $asan && $ubsan; then
+	die "--asan and --ubsan are mutually exclusive"
+fi
+
+if ! type -P afl-fuzz > /dev/null; then
+	die "afl-fuzz is not installed"
+fi
+
+if (( $# == 0 )); then
+	targets=("${AVAILABLE_TARGETS[@]}")
+else
+	for target; do
+		found=false
+		for t in "${AVAILABLE_TARGETS[@]}"; do
+			if [ "$target" = "$t" ]; then
+				found=true
+			fi
+		done
+		if ! $found; then
+			echo 1>&2 "Unknown target '$target'"
+			echo 1>&2 "Available targets: ${AVAILABLE_TARGETS[*]}"
+			exit 1
+		fi
+	done
+	targets=("$@")
+fi
+if (( ${#targets[@]} > 1 )) && ! type -P urxvt > /dev/null; then
+	die "urxvt is not installed"
+fi
+
+afl_opts=""
+if $asan; then
+	export AFL_USE_ASAN=1
+	export CFLAGS="-O2 -m32"
+	export CC=afl-clang
+	afl_opts+=" -m 800"
+elif $ubsan; then
+	export CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined"
+	export CC=afl-gcc
+else
+	export AFL_HARDEN=1
+	export CFLAGS="-O2"
+	export CC=afl-clang-fast
+fi
+
+sudo sh -c "echo core > /proc/sys/kernel/core_pattern"
+sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
+
+NPROC=$(getconf _NPROCESSORS_ONLN)
+
+make "-j$NPROC" -C ../../ libdeflate.a
+make "-j$NPROC" -B
+
+for dir in "${targets[@]}"; do
+	cp -vaT "$dir" "/tmp/$dir"
+	indir=/tmp/$dir/inputs
+	outdir=/tmp/$dir/outputs
+	if [ -e "$outdir" ]; then
+		if $may_resume; then
+			indir="-"
+		else
+			rm -rf "${outdir:?}"/*
+		fi
+	else
+		mkdir "$outdir"
+	fi
+	cmd="afl-fuzz -i $indir -o $outdir -T $dir $afl_opts -- /tmp/$dir/fuzz @@"
+	if (( ${#targets[@]} > 1 )); then
+		urxvt -e bash -c "$cmd" &
+	else
+		$cmd
+	fi
+done
+wait

scripts/afl-fuzz/gzip_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/afl-fuzz/zlib_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/android_build.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+API_LEVEL=28
+ARCH=arm64
+CFLAGS=
+ENABLE_CRC=false
+ENABLE_CRYPTO=false
+NDKDIR=$HOME/android-ndk-r21d
+
+usage() {
+	cat << EOF
+Usage: $0 [OPTION]... -- [MAKE_TARGET]...
+Build libdeflate for Android.
+
+  --api-level=LEVEL    Android API level to target (default: $API_LEVEL)
+  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
+  --enable-crc         Enable crc instructions
+  --enable-crypto      Enable crypto instructions
+  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
+EOF
+}
+if ! options=$(getopt -o '' \
+	-l 'api-level:,arch:,enable-crc,enable-crypto,help,ndkdir:' -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+
+eval set -- "$options"
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--api-level)
+		API_LEVEL="$2"
+		shift
+		;;
+	--arch)
+		ARCH="$2"
+		shift
+		;;
+	--enable-crc)
+		ENABLE_CRC=true
+		;;
+	--enable-crypto)
+		ENABLE_CRYPTO=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--ndkdir)
+		NDKDIR="$2"
+		shift
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Unknown option \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+BINDIR=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/
+
+case "$ARCH" in
+arm|arm32|aarch32)
+	CC=$BINDIR/armv7a-linux-androideabi$API_LEVEL-clang
+	if $ENABLE_CRC || $ENABLE_CRYPTO; then
+		CFLAGS="-march=armv8-a"
+		if $ENABLE_CRC; then
+			CFLAGS+=" -mcrc"
+		else
+			CFLAGS+=" -mnocrc"
+		fi
+		if $ENABLE_CRYPTO; then
+			CFLAGS+=" -mfpu=crypto-neon-fp-armv8"
+		else
+			CFLAGS+=" -mfpu=neon"
+		fi
+	fi
+	;;
+arm64|aarch64)
+	CC=$BINDIR/aarch64-linux-android$API_LEVEL-clang
+	features=""
+	if $ENABLE_CRC; then
+		features+="+crc"
+	fi
+	if $ENABLE_CRYPTO; then
+		features+="+crypto"
+	fi
+	if [ -n "$features" ]; then
+		CFLAGS="-march=armv8-a$features"
+	fi
+	;;
+*)
+	echo 1>&2 "Unknown architecture: \"$ARCH\""
+	usage 1>&2
+	exit 1
+esac
+
+cmd=(make "-j$(grep -c processor /proc/cpuinfo)" "CC=$CC" "CFLAGS=$CFLAGS" "$@")
+echo "${cmd[*]}"
+"${cmd[@]}"

scripts/android_tests.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Test libdeflate on a connected arm64 Android device.
+# Requires the Android NDK (release 19 or later) and adb.
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use NDKDIR if specified in environment, else use default value.
+: "${NDKDIR:=$HOME/android-ndk-r21d}"
+if [ ! -e "$NDKDIR" ]; then
+	cat 1>&2 << EOF
+Android NDK was not found in NDKDIR=$NDKDIR!  Set the
+environmental variable NDKDIR to the location of your Android NDK installation.
+EOF
+	exit 1
+fi
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+android_build_and_test() {
+	echo "Running Android tests with $*"
+
+	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" \
+		all test_programs > /dev/null
+	adb push "$TESTDATA" ./scripts/exec_tests.sh benchmark test_* \
+		/data/local/tmp/ > /dev/null
+
+	# Note: adb shell always returns 0, even if the shell command fails...
+	adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \
+		> "$TMPDIR/adb.out"
+	if ! grep -q "exec_tests finished successfully" "$TMPDIR/adb.out"; then
+		echo 1>&2 "Android test failure!  adb shell output:"
+		cat "$TMPDIR/adb.out"
+		exit 1
+	fi
+}
+
+for arch in arm32 arm64; do
+	android_build_and_test --arch=$arch
+	android_build_and_test --arch=$arch --enable-crc
+	android_build_and_test --arch=$arch --enable-crypto
+	android_build_and_test --arch=$arch --enable-crc --enable-crypto
+done
+echo "Android tests passed"

scripts/checksum_benchmarks.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+have_cpu_feature() {
+	local feature="$1"
+	local tag
+	case $ARCH in
+	arm*|aarch*)
+		tag="Features"
+		;;
+	*)
+		tag="flags"
+		;;
+	esac
+	grep -q "^$tag"$'[ \t]'"*:.*\<$feature\>" /proc/cpuinfo
+}
+
+make_and_test() {
+	# Build the checksum program and tests.  Set the special test support
+	# flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	make "$@" TEST_SUPPORT__DO_NOT_USE=1 checksum test_checksums > /dev/null
+
+	# Run the checksum tests, for good measure.  (This isn't actually part
+	# of the benchmarking.)
+	./test_checksums > /dev/null
+}
+
+__do_benchmark() {
+	local impl="$1" speed
+	shift
+	local flags=("$@")
+
+	speed=$(./checksum "${CKSUM_FLAGS[@]}" "${flags[@]}" -t "$FILE" | \
+		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
+	printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
+}
+
+do_benchmark() {
+	local impl="$1"
+
+	if [ "$impl" = zlib ]; then
+		__do_benchmark "$impl" "-Z"
+	else
+		make_and_test CFLAGS="${EXTRA_CFLAGS[*]}"
+		__do_benchmark "libdeflate, $impl"
+		if [ "$ARCH" = x86_64 ]; then
+			make_and_test CFLAGS="-m32 ${EXTRA_CFLAGS[*]}"
+			__do_benchmark "libdeflate, $impl, 32-bit"
+		fi
+	fi
+}
+
+sort_by_speed() {
+	awk '{print $NF, $0}' | sort -nr | cut -f2- -d' '
+}
+
+disable_cpu_feature() {
+	local name="$1"
+	shift
+	local extra_cflags=("$@")
+
+	LIBDEFLATE_DISABLE_CPU_FEATURES+=",$name"
+	EXTRA_CFLAGS+=("${extra_cflags[@]}")
+}
+
+cleanup() {
+	if $USING_TMPFILE; then
+		rm "$FILE"
+	fi
+}
+
+ARCH="$(uname -m)"
+USING_TMPFILE=false
+
+if (( $# > 1 )); then
+	echo "Usage: $0 [FILE]" 1>&2
+	exit 1
+fi
+
+trap cleanup EXIT
+
+if (( $# == 0 )); then
+	# Generate default test data file.
+	FILE=$(mktemp -t checksum_testdata.XXXXXXXXXX)
+	USING_TMPFILE=true
+	echo "Generating 100 MB test file: $FILE"
+	head -c 100000000 /dev/urandom > "$FILE"
+else
+	FILE="$1"
+fi
+
+cat << EOF
+Method                                       Speed (MB/s)
+------                                       ------------
+EOF
+
+# CRC-32
+CKSUM_NAME="CRC-32"
+CKSUM_FLAGS=()
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature pclmulqdq && have_cpu_feature avx; then
+		do_benchmark "PCLMUL/AVX"
+		disable_cpu_feature "avx" "-mno-avx"
+	fi
+	if have_cpu_feature pclmulqdq; then
+		do_benchmark "PCLMUL"
+		disable_cpu_feature "pclmul" "-mno-pclmul"
+	fi
+	;;
+arm*|aarch*)
+	if have_cpu_feature crc32; then
+		do_benchmark "ARM"
+		disable_cpu_feature "crc32" "-march=armv8-a+nocrc"
+	fi
+	if have_cpu_feature pmull; then
+		do_benchmark "PMULL"
+		disable_cpu_feature "pmull" "-march=armv8-a+nocrc+nocrypto"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed
+
+# Adler-32
+CKSUM_NAME="Adler-32"
+CKSUM_FLAGS=(-A)
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+echo
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature avx512bw; then
+		do_benchmark "AVX-512BW"
+		disable_cpu_feature "avx512bw" "-mno-avx512bw"
+	fi
+	if have_cpu_feature avx2; then
+		do_benchmark "AVX2"
+		disable_cpu_feature "avx2" "-mno-avx2"
+	fi
+	if have_cpu_feature sse2; then
+		do_benchmark "SSE2"
+		disable_cpu_feature "sse2" "-mno-sse2"
+	fi
+	;;
+arm*)
+	if have_cpu_feature neon; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-mfpu=vfpv3"
+	fi
+	;;
+aarch*)
+	if have_cpu_feature asimd; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-march=armv8-a+nosimd"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed

scripts/deflate_benchmarks.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+set -eu -o pipefail
+topdir="$(dirname "$0")/.."
+tmpfile=$(mktemp)
+trap 'rm -f $tmpfile' EXIT
+
+run_benchmark()
+{
+	local best_ctime=1000000000
+	local i
+
+	for i in $(seq "$NUM_ITERATIONS"); do
+		"$@" > "$tmpfile"
+		csize=$(awk '/Compressed/{print $4}' "$tmpfile")
+		ctime=$(awk '/Compression time/{print $3}' "$tmpfile")
+		if (( ctime <  best_ctime )); then
+			best_ctime=$ctime
+		fi
+		: "$i" # make shellcheck happy
+	done
+	CSIZE=$csize
+	CTIME=$best_ctime
+}
+
+multifile()
+{
+	local file results cmd best em
+
+	NUM_ITERATIONS=1
+
+	echo "File | zlib -6 | zlib -9 | libdeflate -6 | libdeflate -9 | libdeflate -12"
+	echo "-----|---------|---------|---------------|---------------|---------------"
+
+	for file in "$@"; do
+		echo -n "$(basename "$file")"
+		results=()
+		cmd=("$topdir/benchmark" -s"$(stat -c "%s" "$file")" "$file")
+		run_benchmark "${cmd[@]}" -Y -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -Y -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -6
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -9
+		results+=("$CSIZE")
+		run_benchmark "${cmd[@]}" -12
+		results+=("$CSIZE")
+		best=2000000000
+		for result in "${results[@]}"; do
+			if (( result < best)); then
+				best=$result
+			fi
+		done
+		for result in "${results[@]}"; do
+			if (( result == best )); then
+				em="**"
+			else
+				em=""
+			fi
+			echo -n " | ${em}${result}${em}"
+		done
+		echo
+	done
+}
+
+single_file()
+{
+	local file=$1
+	local usize args
+	local include_old=false
+
+	usize=$(stat -c "%s" "$file")
+	: ${NUM_ITERATIONS:=3}
+
+	if [ -e "$topdir/benchmark-old" ]; then
+		include_old=true
+	fi
+	echo -n "Level | libdeflate (new) "
+	if $include_old; then
+		echo -n "| libdeflate (old) "
+	fi
+	echo "| zlib"
+	echo -n "------|------------------"
+	if $include_old; then
+		echo -n "|------------------"
+	fi
+	echo "|-----"
+	for level in {1..12}; do
+		echo -n "$level"
+		args=("$file" -s "$usize" "-$level")
+
+		run_benchmark "$topdir/benchmark" "${args[@]}"
+		echo -n " | $CSIZE / $CTIME"
+
+		if $include_old; then
+			run_benchmark "$topdir/benchmark-old" "${args[@]}"
+			echo -n " | $CSIZE / $CTIME"
+		fi
+
+		if (( level > 9 )); then
+			echo -n " | N/A"
+		else
+			run_benchmark "$topdir/benchmark" "${args[@]}" -Y
+			echo -n " | $CSIZE / $CTIME"
+		fi
+		echo
+	done
+}
+
+if (( $# > 1 )); then
+	multifile "$@"
+elif (( $# == 1 )); then
+	single_file "$@"
+else
+	echo 1>&2 "Usage: $0 FILE..."
+fi

scripts/detect.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+set -eu
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
+
+echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
+echo "#ifndef CONFIG_H"
+echo "#define CONFIG_H"
+
+program_compiles() {
+	echo "$1" | $CC $CFLAGS -Wno-error -x c - -o /dev/null > /dev/null 2>&1
+}
+
+check_function() {
+	funcname=$1
+	macro="HAVE_$(echo "$funcname" | tr '[:lower:]' '[:upper:]')"
+
+	echo
+	echo "/* Is the $funcname() function available? */"
+	if program_compiles "int main() { $funcname(); }"; then
+		echo "#define $macro 1"
+	else
+		echo "/* $macro is not set */"
+	fi
+}
+
+have_stat_field() {
+	program_compiles "#include <sys/types.h>
+			  #include <sys/stat.h>
+			  int main() { struct stat st; st.$1; }"
+}
+
+check_stat_nanosecond_precision() {
+	echo
+	echo "/* Does stat() provide nanosecond-precision timestamps? */"
+	if have_stat_field st_atim; then
+		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
+	elif have_stat_field st_atimespec; then
+		# Nonstandard field names used by OS X and older BSDs
+		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
+		echo "#define st_atim st_atimespec"
+		echo "#define st_mtim st_mtimespec"
+		echo "#define st_ctim st_ctimespec"
+	else
+		echo "/* HAVE_STAT_NANOSECOND_PRECISION is not set */"
+	fi
+}
+
+check_function clock_gettime
+check_function futimens
+check_function futimes
+check_function posix_fadvise
+check_function posix_madvise
+
+check_stat_nanosecond_precision
+
+echo
+echo "#endif /* CONFIG_H */"

scripts/exec_tests.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# Helper script used by run_tests.sh and android_tests.sh,
+# not intended to be run directly
+#
+
+set -eu
+
+run_cmd() {
+	echo "$WRAPPER $*"
+	$WRAPPER "$@" > /dev/null
+}
+
+for prog in ./test_*; do
+	run_cmd "$prog"
+done
+
+for format in '' '-g' '-z'; do
+	for ref_impl in '' '-Y' '-Z'; do
+		run_cmd ./benchmark $format $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9; do
+	for ref_impl in '' '-Y'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9 12; do
+	for ref_impl in '' '-Z'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+
+echo "exec_tests finished successfully" # Needed for 'adb shell'

scripts/gen_crc32_multipliers.c

@@ -0,0 +1,108 @@
+/*
+ * gen_crc32_multipliers.c
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <inttypes.h>
+#include <stdio.h>
+
+/* generator polynomial G(x) */
+#define CRCPOLY		0xEDB88320 /* G(x) without x^32 term */
+#define CRCPOLY_FULL	(((uint64_t)CRCPOLY << 1) | 1) /* G(x) */
+
+/* Compute x^D mod G(x) */
+static uint32_t
+compute_multiplier(int D)
+{
+	/* Start with x^0 mod G(x) */
+	uint32_t remainder = 0x80000000;
+
+	/* Each iteration, 'remainder' becomes x^i mod G(x) */
+	for (int i = 1; i <= D; i++)
+		remainder = (remainder >> 1) ^ ((remainder & 1) ? CRCPOLY : 0);
+
+	/* Now 'remainder' is x^D mod G(x) */
+	return remainder;
+}
+
+/* Compute floor(x^64 / G(x)) */
+static uint64_t
+compute_barrett_reduction_constant(void)
+{
+	uint64_t quotient = 0;
+	uint64_t dividend = 0x1;
+
+	for (int i = 0; i < 64 - 32 + 1; i++) {
+		if ((dividend >> i) & 1) {
+			quotient |= (uint64_t)1 << i;
+			dividend ^= CRCPOLY_FULL << i;
+		}
+	}
+
+	return quotient;
+}
+
+/*
+ * This program computes the constant multipliers needed for carryless
+ * multiplication accelerated CRC-32.  It assumes 128-bit vectors divided into
+ * two 64-bit halves which are multiplied separately with different 32-bit
+ * multipliers, producing two 95-bit products.  For a given number of 128-bit
+ * vectors per iteration, the program outputs a pair of multipliers, one for
+ * each 64-bit half.
+ *
+ * Careful: all polynomials are "bit-reversed", meaning that the low-order bits
+ * have the highest degree and the high-order bits have the lowest degree!
+ */
+int
+main(void)
+{
+	printf("\t/* Constants precomputed by gen_crc32_multipliers.c.  "
+	       "Do not edit! */\n");
+
+	/* High and low multipliers for each needed vector count */
+	for (int order = 2; order >= 0; order--) {
+		int vecs_per_iteration = 1 << order;
+		int right = (128 * vecs_per_iteration) + 95;
+		printf("\tconst __v2di multipliers_%d = (__v2di)"
+		       "{ 0x%08"PRIX32", 0x%08"PRIX32" };\n",
+		       vecs_per_iteration,
+		       compute_multiplier(right - 64) /* higher degree half */,
+		       compute_multiplier(right - 128) /* lower degree half */);
+	}
+
+	/* Multiplier for final 96 => 64 bit fold */
+	printf("\tconst __v2di final_multiplier = (__v2di){ 0x%08"PRIX32" };\n",
+	       compute_multiplier(63));
+
+	/* 32-bit mask */
+	printf("\tconst __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };\n");
+
+	/* Constants for final 64 => 32 bit reduction */
+	printf("\tconst __v2di barrett_reduction_constants =\n"
+	       "\t\t\t(__v2di){ 0x%016"PRIX64", 0x%016"PRIX64" };\n",
+	       compute_barrett_reduction_constant(), CRCPOLY_FULL);
+
+	return 0;
+}

scripts/gen_crc32_table.c

@@ -1,15 +1,28 @@
 /*
  * gen_crc32_table.c - a program for CRC-32 table generation
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <stdint.h>

scripts/gen_default_litlen_costs.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+#
+# This script computes the default litlen symbol costs for the near-optimal
+# parser.
+
+from math import log2
+
+BIT_COST = 16 # Must match BIT_COST in deflate_compress.c
+NUM_LEN_SLOTS = 29
+
+print("""static const struct {
+	u8 used_lits_to_lit_cost[257];
+	u8 len_sym_cost;
+} default_litlen_costs[] = {""")
+MATCH_PROBS = [0.25, 0.50, 0.75]
+for i, match_prob in enumerate(MATCH_PROBS):
+    len_prob = match_prob / NUM_LEN_SLOTS
+    len_sym_cost = int(-log2(len_prob) * BIT_COST)
+    if i == 0:
+        print('\t{', end='')
+    print(f' /* match_prob = {match_prob} */')
+    print('\t\t.used_lits_to_lit_cost = {')
+
+    j = 0
+    for num_used_literals in range(0, 257):
+        if num_used_literals == 0:
+            num_used_literals = 1
+        lit_prob = (1 - match_prob) / num_used_literals
+        lit_cost = int(-log2(lit_prob) * BIT_COST)
+        if j == 0:
+            print('\t\t\t', end='')
+        if j == 7 or num_used_literals == 256:
+            print(f'{lit_cost},')
+            j = 0
+        else:
+            print(f'{lit_cost}, ', end='')
+            j += 1
+    print('\t\t},')
+    print(f'\t\t.len_sym_cost = {len_sym_cost},')
+    if i < len(MATCH_PROBS) - 1:
+        print('\t}, {', end='')
+    else:
+        print('\t},')
+print('};')

scripts/gen_offset_slot_map.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+# This script generates the deflate_offset_slot[] array, which is a condensed
+# map from offsets to offset slots.
+
+DEFLATE_OFFSET_SLOT_BASE = [
+	1    , 2    , 3    , 4     , 5     , 7     , 9     , 13    ,
+	17   , 25   , 33   , 49    , 65    , 97    , 129   , 193   ,
+	257  , 385  , 513  , 769   , 1025  , 1537  , 2049  , 3073  ,
+	4097 , 6145 , 8193 , 12289 , 16385 , 24577 ,
+]
+
+DEFLATE_EXTRA_OFFSET_BITS = [
+	0    , 0    , 0    , 0     , 1     , 1     , 2     , 2     ,
+	3    , 3    , 4    , 4     , 5     , 5     , 6     , 6     ,
+	7    , 7    , 8    , 8     , 9     , 9     , 10    , 10    ,
+	11   , 11   , 12   , 12    , 13    , 13    ,
+]
+
+offset_slot_map = [0] * 512
+
+for offset_slot, offset_base in enumerate(DEFLATE_OFFSET_SLOT_BASE):
+    num_extra_bits = DEFLATE_EXTRA_OFFSET_BITS[offset_slot]
+    offset_end = offset_base + (1 << num_extra_bits)
+    if offset_base <= 256:
+        for offset in range(offset_base, offset_end):
+            offset_slot_map[offset] = offset_slot
+    else:
+        for offset in range(offset_base, offset_end, 128):
+            offset_slot_map[256 + ((offset - 1) >> 7)] = offset_slot
+
+print('static const u8 deflate_offset_slot_map[512] = {')
+for i in range(0, len(offset_slot_map), 16):
+    print('\t', end='')
+    for j, v in enumerate(offset_slot_map[i:i+16]):
+        print(f'{v},', end='')
+        if j == 15:
+            print('')
+        else:
+            print(' ', end='')
+print('};')

scripts/gzip_tests.sh

@@ -0,0 +1,507 @@
+#!/bin/bash
+#
+# Test script for libdeflate's gzip and gunzip programs.
+#
+# To run, you must set GZIP and GUNZIP in the environment to the absolute paths
+# to the gzip and gunzip programs to test.  All tests should pass regardless of
+# whether the GNU versions or the libdeflate versions, or a combination, of
+# these programs are used.
+#
+# The environmental variable TESTDATA must also be set to a file containing
+# test data.
+#
+
+set -eu -o pipefail
+
+export -n GZIP GUNZIP TESTDATA
+
+TMPDIR="$(mktemp -d)"
+CURRENT_TEST=
+
+cleanup() {
+	if [ -n "$CURRENT_TEST" ]; then
+		echo "TEST FAILED: \"$CURRENT_TEST\""
+	fi
+	rm -rf -- "$TMPDIR"
+}
+
+trap cleanup EXIT
+
+TESTDATA="$(readlink -f "$TESTDATA")"
+cd "$TMPDIR"
+
+begin_test() {
+	CURRENT_TEST="$1"
+	rm -rf -- "${TMPDIR:?}"/*
+	cp "$TESTDATA" file
+	chmod +w file
+}
+
+gzip() {
+	$GZIP "$@"
+}
+
+gunzip() {
+	$GUNZIP "$@"
+}
+
+assert_status() {
+	local expected_status="$1"
+	local expected_msg="$2"
+	shift 2
+	(
+		set +e
+		{ eval "$*" > /dev/null; } 2>&1
+		local actual_status=$?
+		if [ "$actual_status" != "$expected_status" ]; then
+			echo 1>&2 "Command '$*' exited with status" \
+				"$actual_status but expected status" \
+				"$expected_status"
+			exit 1
+		fi
+		exit 0
+	) > command_output
+	if ! grep -E -q "$expected_msg" command_output; then
+		echo 1>&2 "Expected output of command '$*' to match regex" \
+			"'$expected_msg'"
+		echo 1>&2 "Actual output was:"
+		echo 1>&2 "---------------------------------------------------"
+		cat 1>&2 command_output
+		echo 1>&2 "---------------------------------------------------"
+		return 1
+	fi
+}
+
+assert_error() {
+	assert_status 1 "$@"
+}
+
+assert_warning() {
+	assert_status 2 "$@"
+}
+
+assert_skipped() {
+	assert_warning '\<(ignored|skipping|unchanged)\>' "$@"
+}
+
+assert_equals() {
+	local expected="$1"
+	local actual="$2"
+
+	if [ "$expected" != "$actual" ]; then
+		echo 1>&2 "Expected '$expected', but got '$actual'"
+		return 1
+	fi
+}
+
+# Get the filesystem type.
+FSTYPE=$(df -T . | tail -1 | awk '{print $2}')
+
+# If gzip or gunzip is the GNU version, require that it supports the '-k'
+# option.  This option was added in v1.6, released in 2013.
+check_version_prereq() {
+	local prog=$1
+
+	if ! echo | { $prog -k || true; } |& grep -q 'invalid option'; then
+		return 0
+	fi
+	if ! $prog -V |& grep -q 'Free Software Foundation'; then
+		echo 1>&2 "Unexpected case: not GNU $prog, but -k option is invalid"
+		exit 1
+	fi
+	echo "GNU $prog is too old; skipping gzip/gunzip tests"
+	exit 0
+}
+check_version_prereq gzip
+check_version_prereq gunzip
+
+begin_test 'Basic compression and decompression works'
+cp file orig
+gzip file
+[ ! -e file ] && [ -e file.gz ]
+gunzip file.gz
+[ -e file ] && [ ! -e file.gz ]
+cmp file orig
+
+
+begin_test 'gzip -d is gunzip'
+cp file orig
+gzip file
+gzip -d file.gz
+cmp file orig
+
+
+begin_test '-k (keep original file) works'
+cp file orig
+gzip -k file
+cmp file orig
+rm file
+cp file.gz orig.gz
+gunzip -k file.gz
+cmp file.gz orig.gz
+
+
+begin_test '-c (write to stdout) works'
+cp file orig
+gzip -k file
+gzip -c file > 2.gz
+cmp file orig
+cmp file.gz 2.gz
+gunzip -c 2.gz > file
+cmp file.gz 2.gz
+cmp file orig
+
+
+# Note: in some of the commands below, we intentionally use 'cat file | gzip'
+# rather than 'gzip < file', in order to test the use of a pipe.  This produces
+# a shellcheck warning about 'cat' being unnecessary.  Suppress that warning by
+# using { cat file; true; }.
+begin_test 'Reading from stdin works'
+gzip < file > 1.gz
+gzip - < file > 2.gz
+{ cat file; true; } | gzip > 3.gz
+{ cat file; true; } | gzip - > 4.gz
+cmp file <(gunzip < 1.gz)
+cmp file <(gunzip - < 2.gz)
+cmp file <({ cat 3.gz; true; } | gunzip)
+cmp file <({ cat 4.gz; true; } | gunzip -)
+
+
+begin_test '-n option is accepted'
+gzip -n file
+gunzip -n file.gz
+
+
+begin_test 'can specify multiple options'
+gzip -fk1 file
+cmp <(gzip -c -1 file) file.gz
+gunzip -kfd file.gz
+
+
+begin_test 'Compression levels'
+if [ "$GZIP" = /bin/gzip ]; then
+	assert_error '\<invalid option\>' gzip -10
+	max_level=9
+else
+	for level in 13 99999 1a; do
+		assert_error '\<Invalid compression level\>' gzip -$level
+	done
+	max_level=12
+fi
+for level in $(seq 1 $max_level); do
+	gzip -c "-$level" file > "file$level"
+	cmp file <(gunzip -c "file$level")
+done
+rm file command_output
+
+
+begin_test 'Overwriting output file requires -f'
+cp file orig
+echo -n > file.gz
+gzip -c file > 2.gz
+assert_warning 'already exists' gzip file </dev/null
+cmp file.gz /dev/null
+gzip -f file
+cmp 2.gz file.gz
+echo -n > file
+assert_warning 'already exists' gunzip file.gz </dev/null
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Nonexistent input file fails, even with -f'
+for prog in 'gzip' 'gzip -f' 'gunzip' 'gunzip -f'; do
+	assert_error 'No such file or directory' "$prog" NONEXISTENT
+done
+
+
+begin_test 'Compressing already-suffixed file requires -f or -c'
+gzip file
+gzip -c file.gz > c.gz
+gzip file.gz 2>&1 >/dev/null | grep -q 'already has .gz suffix'
+[ -e file.gz ] && [ ! -e file.gz.gz ]
+gzip -f file.gz
+[ ! -e file.gz ] && [ -e file.gz.gz ]
+cmp file.gz.gz c.gz
+
+
+begin_test 'gunzip -f -c passes through non-gzip data'
+echo hello > file
+cp file orig
+gunzip -f -c file > foo
+cmp file foo
+gzip file
+gunzip -f -c file.gz > foo
+cmp foo orig
+
+
+begin_test 'Decompressing unsuffixed file only works with -c'
+gzip file && mv file.gz file
+assert_skipped gunzip file
+assert_skipped gunzip -f file
+gunzip -c file > orig
+mv file file.gz && gunzip file.gz && cmp file orig
+
+
+begin_test '... unless there is a corresponding suffixed file'
+cp file orig
+gzip file
+[ ! -e file ] && [ -e file.gz ]
+gunzip -c file > tmp
+cmp tmp orig
+rm tmp
+ln -s NONEXISTENT file
+gunzip -c file > tmp
+cmp tmp orig
+rm tmp file
+gunzip file
+[ -e file ] && [ ! -e file.gz ]
+cmp file orig
+
+
+begin_test 'Directory is skipped, even with -f'
+mkdir dir
+mkdir dir.gz
+for opt in '' '-f' '-c'; do
+	assert_skipped gzip $opt dir
+done
+#assert_skipped gzip dir.gz  # XXX: GNU gzip warns, libdeflate gzip no-ops
+for opt in '' '-f' '-c'; do
+	for name in dir dir.gz; do
+		assert_skipped gunzip $opt $name
+	done
+done
+
+
+begin_test '(gzip) symlink is rejected without -f or -c'
+ln -s file symlink1
+ln -s file symlink2
+assert_error 'Too many levels of symbolic links' gzip symlink1
+[ -e file ] && [ -e symlink1 ] && [ ! -e symlink1.gz ]
+gzip -f symlink1
+[ -e file ] && [ ! -e symlink1 ] && [ -e symlink1.gz ]
+gzip -c symlink2 > /dev/null
+
+
+begin_test '(gunzip) symlink is rejected without -f or -c'
+gzip file
+ln -s file.gz symlink1.gz
+ln -s file.gz symlink2.gz
+assert_error 'Too many levels of symbolic links' gunzip symlink1
+[ -e file.gz ] && [ -e symlink1.gz ] && [ ! -e symlink1 ]
+gunzip -f symlink1.gz
+[ -e file.gz ] && [ ! -e symlink1.gz ] && [ -e symlink1 ]
+gunzip -c symlink2.gz > /dev/null
+
+
+begin_test 'FIFO is skipped, even with -f'
+mkfifo foo
+mkfifo foo.gz
+assert_skipped gzip foo
+assert_skipped gzip -f foo
+#assert_skipped gzip -c foo # XXX: works with GNU gzip, not libdeflate's
+assert_skipped gunzip foo.gz
+assert_skipped gunzip -f foo.gz
+#assert_skipped gunzip -c foo.gz # XXX: works with GNU gzip, not libdeflate's
+
+
+begin_test '(gzip) overwriting symlink does not follow symlink'
+echo a > a
+echo b > b
+gzip a
+ln -s a.gz b.gz
+gzip -f b
+gunzip a.gz
+cmp <(echo a) a
+
+
+begin_test '(gunzip) overwriting symlink does not follow symlink'
+echo a > a
+echo b > b
+gzip b
+ln -s a b
+gunzip -f b.gz
+cmp <(echo a) a
+cmp <(echo b) b
+
+
+begin_test '(gzip) hard linked file skipped without -f or -c'
+cp file orig
+ln file link
+assert_equals 2 "$(stat -c %h file)"
+assert_skipped gzip file
+gzip -c file > /dev/null
+assert_equals 2 "$(stat -c %h file)"
+gzip -f file
+assert_equals 1 "$(stat -c %h link)"
+assert_equals 1 "$(stat -c %h file.gz)"
+cmp link orig
+# XXX: GNU gzip skips hard linked files with -k, libdeflate's doesn't
+
+
+begin_test '(gunzip) hard linked file skipped without -f or -c'
+gzip file
+ln file.gz link.gz
+cp file.gz orig.gz
+assert_equals 2 "$(stat -c %h file.gz)"
+assert_skipped gunzip file.gz
+gunzip -c file.gz > /dev/null
+assert_equals 2 "$(stat -c %h file.gz)"
+gunzip -f file
+assert_equals 1 "$(stat -c %h link.gz)"
+assert_equals 1 "$(stat -c %h file)"
+cmp link.gz orig.gz
+
+
+begin_test 'Multiple files'
+cp file file2
+gzip file file2
+[ ! -e file ] && [ ! -e file2 ] && [ -e file.gz ] && [ -e file2.gz ]
+gunzip file.gz file2.gz
+[ -e file ] && [ -e file2 ] && [ ! -e file.gz ] && [ ! -e file2.gz ]
+
+
+begin_test 'Multiple files, continue on warning'
+mkdir 1
+cp file 2
+assert_skipped gzip 1 2
+[ ! -e 1.gz ]
+cmp file <(gunzip -c 2.gz)
+rmdir 1
+mkdir 1.gz
+assert_skipped gunzip 1.gz 2.gz
+[ ! -e 1 ]
+cmp 2 file
+
+
+if (( $(id -u) != 0 )); then
+	begin_test 'Multiple files, continue on error'
+	cp file 1
+	cp file 2
+	chmod a-r 1
+	assert_error 'Permission denied' gzip 1 2
+	[ ! -e 1.gz ]
+	cmp file <(gunzip -c 2.gz)
+	rm -f 1
+	cp 2.gz 1.gz
+	chmod a-r 1.gz
+	assert_error 'Permission denied' gunzip 1.gz 2.gz
+	[ ! -e 1 ]
+	cmp 2 file
+fi
+
+
+begin_test 'Compressing empty file'
+echo -n > empty
+gzip empty
+gunzip empty.gz
+cmp /dev/null empty
+
+
+begin_test 'Decompressing malformed file'
+echo -n > foo.gz
+assert_error '\<(not in gzip format|unexpected end of file)\>' \
+	gunzip foo.gz
+echo 1 > foo.gz
+assert_error '\<not in gzip format\>' gunzip foo.gz
+echo abcdefgh > foo.gz
+assert_error '\<not in gzip format\>' gunzip foo.gz
+echo -ne '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4b\x4c\x4a\x4e\x49\x24\x16\x73\x01\x00\x6c\x5b\xa2\x62\x2e\x00\x00\x00' \
+	> foo.gz
+assert_error '\<(not in gzip format|crc error)\>' gunzip foo.gz
+
+
+for suf in .foo foo .blaaaaaaaaaaaaaaaargh; do
+	begin_test "Custom suffix: $suf"
+	gzip -S $suf file
+	[ ! -e file ] && [ ! -e file.gz ] && [ -e file$suf ]
+	assert_skipped gunzip file$suf
+	gunzip -S $suf file$suf
+	[ -e file ] && [ ! -e file.gz ] && [ ! -e file$suf ]
+done
+# DIFFERENCE: GNU gzip lower cases suffix, we don't
+
+
+begin_test 'Empty suffix is rejected'
+assert_error '\<invalid suffix\>' gzip -S '""' file
+assert_error '\<invalid suffix\>' gunzip -S '""' file
+
+
+begin_test 'Timestamps and mode are preserved'
+if [ "$FSTYPE" = shiftfs ]; then
+	# In Travis CI, the filesystem (shiftfs) only supports seconds precision
+	# timestamps.  Nanosecond precision still sometimes seems to work,
+	# probably due to caching, but it is unreliable.
+	format='%a;%X;%Y'
+else
+	format='%a;%x;%y'
+fi
+chmod 777 file
+orig_stat="$(stat -c "$format" file)"
+gzip file
+sleep 1
+gunzip file.gz
+assert_equals "$orig_stat" "$(stat -c "$format" file)"
+
+
+begin_test 'Decompressing multi-member gzip file'
+cat file file > orig
+gzip -c file > file.gz
+gzip -c file >> file.gz
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Decompressing multi-member gzip file (final member smaller)'
+echo 'hello world' > hello
+cat file hello > orig
+gzip -c file > file.gz
+gzip -c hello >> file.gz
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Help option'
+gzip -h 2>&1 | grep -q 'Usage'
+gunzip -h 2>&1 | grep -q 'Usage'
+
+
+begin_test 'Incorrect usage'
+for prog in gzip gunzip; do
+	for opt in '--invalid-option' '-0'; do
+		assert_error '\<(unrecognized|invalid) option\>' $prog $opt
+	done
+done
+
+
+begin_test '-t (test) option works'
+good_files=(
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIoAUrnJFZVKqTkp+txAQBqzFDrLQAAAA==')
+bad_files=(
+'H4sIAO1YYmAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+46ehsAAAA='
+'H4sIAO1YYmAAA3PMSVTITVTIzi85VABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsBAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIogUrnJFZVKqTkp+txAQBqzFDrLQAAAA=='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9L')
+for contents in "${good_files[@]}"; do
+	echo "$contents" | base64 -d | gzip -t
+done
+for contents in "${bad_files[@]}"; do
+	echo "$contents" | base64 -d > file
+	assert_error '\<invalid compressed data|file corrupt|unexpected end of file|Out of memory\>' \
+		gzip -t file
+done
+
+
+begin_test '-q (quiet) option works'
+mkdir dir
+gunzip -q dir &> output || true
+[ ! -s output ]
+
+
+begin_test 'Version information'
+gzip -V | grep -q Copyright
+gunzip -V | grep -q Copyright
+
+CURRENT_TEST=

scripts/make-windows-releases.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+for arch in 'i686' 'x86_64'; do
+	make clean
+	make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all \
+		benchmark.exe checksum.exe
+	dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin
+	rm -rf "$dir" "$dir.zip"
+	mkdir "$dir"
+	cp libdeflate.{dll,lib,def} libdeflatestatic.lib libdeflate.h ./*.exe \
+		"$dir"
+	${arch}-w64-mingw32-strip "$dir/libdeflate.dll" "$dir"/*.exe
+	for file in COPYING NEWS.md README.md; do
+		sed < $file > "$dir/${file}.txt" -e 's/$/\r/g'
+	done
+	(cd "$dir" && zip -r "../${dir}.zip" .)
+done

scripts/pgo_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Try gcc profile-guided optimizations
+
+set -eu
+
+MAKE="make -j$(grep -c processor /proc/cpuinfo)"
+DATAFILE="$HOME/data/silesia"
+
+$MAKE benchmark > /dev/null
+echo "====================="
+echo "Original performance:"
+echo "---------------------"
+./benchmark "$@" "$DATAFILE"
+
+$MAKE CFLAGS=-fprofile-generate LDFLAGS=-fprofile-generate benchmark > /dev/null
+./benchmark "$@" "$DATAFILE" > /dev/null
+$MAKE CFLAGS=-fprofile-use benchmark > /dev/null
+rm -f {lib,programs}/*.gcda
+echo "=========================="
+echo "PGO-optimized performance:"
+echo "--------------------------"
+./benchmark "$@" "$DATAFILE"

scripts/run_tests.sh

@@ -0,0 +1,329 @@
+#!/bin/bash
+#
+# Test script for libdeflate
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+MAKE="make -j$(getconf _NPROCESSORS_ONLN)"
+
+CC_VERSION=$($CC --version | head -1)
+
+ARCH=$(uname -m)
+
+for skip in SKIP_FREESTANDING SKIP_VALGRIND SKIP_UBSAN SKIP_ASAN SKIP_CFI \
+	    SKIP_SHARED_LIB; do
+	if [ "${!skip:-}" = "1" ]; then
+		eval $skip=true
+	else
+		eval $skip=false
+	fi
+done
+
+###############################################################################
+
+INDENT=0
+
+log() {
+	echo -n "[$(date)] "
+	head -c $(( INDENT * 4 )) /dev/zero | tr '\0' ' '
+	echo "$@"
+}
+
+begin() {
+	log "$@"
+	(( INDENT++ )) || true
+}
+
+end() {
+	(( INDENT-- )) || true
+}
+
+run_cmd() {
+	log "$@"
+	"$@" > /dev/null
+}
+
+fail() {
+	echo 1>&2 "$@"
+	exit 1
+}
+
+file_count() {
+	local dir=$1
+
+	find "$dir" -type f -o -type l | wc -l
+}
+
+cflags_supported() {
+	# -Werror is needed here in order for old versions of clang to reject
+	# invalid options.
+	echo 'int main(void){ return 0; }' \
+		| $CC $CFLAGS "$@" -Werror -x c - -o /dev/null 2>/dev/null
+}
+
+valgrind_version_at_least() {
+	local want_vers=$1
+	local vers
+
+	vers=$(valgrind --version | grep -E -o '[0-9\.]+' | head -1)
+
+	[ "$want_vers" = "$(echo -e "$vers\n$want_vers" | sort -V | head -1)" ]
+}
+
+build_and_run_tests() {
+	local quick=false
+	if [ "${1:-}" = "--quick" ]; then
+		quick=true
+		shift
+	fi
+
+	begin "CC=$CC CFLAGS=\"$CFLAGS\" WRAPPER=\"$WRAPPER\" $*"
+
+	# Build libdeflate, including the test programs.  Set the special test
+	# support flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	$MAKE "$@" TEST_SUPPORT__DO_NOT_USE=1 all test_programs > /dev/null
+
+	# When not using -march=native, run the tests multiple times with
+	# different combinations of CPU features disabled.  This is needed to
+	# test all variants of dynamically-dispatched code.
+	#
+	# For now, we aren't super exhausive in which combinations of features
+	# we test disabling.  We just disable the features roughly in order from
+	# newest to oldest for each architecture, cumulatively.  In practice,
+	# that's good enough to cover all the code.
+	local features=('')
+	if ! [[ "$CFLAGS" =~ "-march=native" ]] && ! $quick; then
+		case "$ARCH" in
+		i386|x86_64)
+			features+=(avx512bw avx2 avx bmi2 pclmul sse2)
+			;;
+		arm*|aarch*)
+			features+=(crc32 pmull neon)
+			;;
+		esac
+	fi
+	local disable_str=""
+	local feature
+	for feature in "${features[@]}"; do
+		if [ -n "$feature" ]; then
+			if [ -n "$disable_str" ]; then
+				disable_str+=","
+			fi
+			disable_str+="$feature"
+		fi
+		log "Using LIBDEFLATE_DISABLE_CPU_FEATURES=$disable_str"
+		LIBDEFLATE_DISABLE_CPU_FEATURES="$disable_str" \
+		    sh ./scripts/exec_tests.sh > /dev/null
+	done
+	end
+}
+
+verify_freestanding_build() {
+	# It is expected that sanitizer builds link to external functions.
+	if [[ "$CFLAGS" =~ "-fsanitize" ]]; then
+		return 0
+	fi
+	log "Verifying that freestanding build is really freestanding"
+	if nm libdeflate.so | grep -q ' U '; then
+		echo 1>&2 "Freestanding lib links to external functions!:"
+		nm libdeflate.so | grep ' U '
+		return 1
+	fi
+	if ldd libdeflate.so | grep -q -v '\<statically linked\>'; then
+		echo 1>&2 "Freestanding lib links to external libraries!:"
+		ldd libdeflate.so
+		return 1
+	fi
+}
+
+gzip_tests() {
+	local gzips=("$PWD/gzip")
+	local gunzips=("$PWD/gunzip")
+	if [ "${1:-}" != "--quick" ]; then
+		gzips+=(/bin/gzip)
+		gunzips+=(/bin/gunzip)
+	fi
+	local gzip gunzip
+
+	begin "Running gzip program tests with CC=\"$CC\" CFLAGS=\"$CFLAGS\""
+	$MAKE gzip gunzip > /dev/null
+	for gzip in "${gzips[@]}"; do
+		for gunzip in "${gunzips[@]}"; do
+			log "GZIP=$gzip, GUNZIP=$gunzip"
+			GZIP="$gzip" GUNZIP="$gunzip" TESTDATA="$TESTDATA" \
+				./scripts/gzip_tests.sh
+		done
+	done
+	end
+}
+
+do_run_tests() {
+	build_and_run_tests "$@"
+	if [ "${1:-}" != "--quick" ]; then
+		if $SKIP_FREESTANDING; then
+			log "Skipping freestanding build tests due to SKIP_FREESTANDING=1"
+		else
+			build_and_run_tests FREESTANDING=1
+			verify_freestanding_build
+		fi
+	fi
+	gzip_tests "$@"
+}
+
+check_symbol_prefixes() {
+	log "Checking that all global symbols are prefixed with \"libdeflate_\""
+	$MAKE libdeflate.a > /dev/null
+	if nm libdeflate.a | grep ' T ' | grep -E -v " _?libdeflate_"; then
+		fail "Some global symbols aren't prefixed with \"libdeflate_\""
+	fi
+	log "Checking that all exported symbols are prefixed with \"libdeflate\""
+	$MAKE libdeflate.so > /dev/null
+	if nm libdeflate.so | grep ' T ' \
+			| grep -E -v " (libdeflate_|_init\>|_fini\>)"; then
+		fail "Some exported symbols aren't prefixed with \"libdeflate_\""
+	fi
+}
+
+test_use_shared_lib() {
+	if $SKIP_SHARED_LIB; then
+		log "Skipping USE_SHARED_LIB=1 tests due to SKIP_SHARED_LIB=1"
+		return
+	fi
+	log "Testing USE_SHARED_LIB=1"
+	$MAKE gzip > /dev/null
+	if ldd gzip | grep -q 'libdeflate.so'; then
+		fail "Binary should be statically linked by default"
+	fi
+	$MAKE USE_SHARED_LIB=1 all check > /dev/null
+	ldd gzip > "$TMPDIR/ldd.out"
+	if ! grep -q 'libdeflate.so' "$TMPDIR/ldd.out"; then
+		cat 1>&2 "$TMPDIR/ldd.out"
+		fail "Binary isn't dynamically linked"
+	fi
+	rm "$TMPDIR/ldd.out"
+}
+
+install_uninstall_tests() {
+	local shell
+
+	begin "Testing 'make install' and 'make uninstall'"
+	for shell in '/bin/bash' '/bin/dash'; do
+		log "Trying SHELL=$shell"
+		$MAKE SHELL=$shell clean > /dev/null
+		$MAKE SHELL=$shell DESTDIR="$TMPDIR/inst" install > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" == 0 )); then
+			fail "'make install' didn't install any files"
+		fi
+		make SHELL=$shell DESTDIR="$TMPDIR/inst" uninstall > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" != 0 )); then
+			fail "'make uninstall' didn't uninstall all files"
+		fi
+		rm -r "$TMPDIR/inst"
+	done
+	end
+}
+
+run_tests() {
+	export WRAPPER="" # no wrapper by default; overridden by valgrind tests
+	local cflags
+
+	begin "Running tests"
+	do_run_tests
+	end
+
+	cflags=("-O3")
+	if cflags_supported "${cflags[@]}" "-march=native"; then
+		cflags+=("-march=native")
+	fi
+	begin "Running tests with ${cflags[*]}"
+	CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests
+	end
+
+	# Need valgrind 3.9.0 for '--errors-for-leak-kinds=all'
+	# Need valgrind 3.12.0 for armv8 crypto and crc instructions
+	if $SKIP_VALGRIND; then
+		log "Skipping valgrind tests due to SKIP_VALGRIND=1"
+	elif valgrind_version_at_least 3.12.0; then
+		begin "Running tests with Valgrind"
+		WRAPPER="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all" \
+			do_run_tests --quick
+		end
+	fi
+
+	cflags=("-fsanitize=undefined" "-fno-sanitize-recover=undefined")
+	if $SKIP_UBSAN; then
+		log "Skipping UBSAN tests due to SKIP_UBSAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with UBSAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping UBSAN tests because compiler ($CC_VERSION) doesn't support UBSAN"
+	fi
+
+	cflags=("-fsanitize=address" "-fno-sanitize-recover=address")
+	if $SKIP_ASAN; then
+		log "Skipping ASAN tests due to SKIP_ASAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with ASAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping ASAN tests because compiler ($CC_VERSION) doesn't support ASAN"
+	fi
+
+	cflags=("-fsanitize=cfi" "-fno-sanitize-recover=cfi" "-flto"
+		"-fvisibility=hidden")
+	if $SKIP_CFI; then
+		log "Skipping CFI tests due to SKIP_CFI=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with CFI"
+		CFLAGS="$CFLAGS ${cflags[*]}" AR=llvm-ar do_run_tests --quick
+		end
+	else
+		log "Skipping CFI tests because compiler ($CC_VERSION) doesn't support CFI"
+	fi
+
+	install_uninstall_tests
+	check_symbol_prefixes
+	test_use_shared_lib
+}
+
+###############################################################################
+
+log "Starting libdeflate tests"
+run_tests
+log "All tests passed!"