v1.8

- crc32: On ICC, __v2di is defined in immintrin.h
- adler32.c: __v64qi etc are not available on ICC

.cirrus.yml

@@ -0,0 +1,8 @@
+task:
+  freebsd_instance:
+    matrix:
+      - image_family: freebsd-11-3-snap
+      - image_family: freebsd-12-1-snap
+  install_script: pkg install -y gmake
+  script:
+    - gmake check

.github/workflows/ci.yml

@@ -0,0 +1,123 @@
+name: CI
+on: [pull_request]
+env:
+  CFLAGS: -Werror
+
+jobs:
+  x86_64-build-and-test:
+    name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, ubuntu-18.04, ubuntu-16.04]
+        compiler: [gcc, clang]
+        exclude:
+          # clang 3.8.0-2ubuntu4 crashes with:
+          # "fatal error: error in backend: Cannot select: 0x21025a0: v64i8 = X86ISD::VBROADCAST 0x2101fb0"
+          - os: ubuntu-16.04
+            compiler: clang
+    runs-on: ${{ matrix.os }}
+    env:
+      CC: ${{ matrix.compiler }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang llvm libz-dev valgrind
+    - run: scripts/run_tests.sh
+
+  other-arch-build-and-test:
+    name: Build and test (${{ matrix.arch }}, Debian Buster, ${{ matrix.compiler }})
+    strategy:
+      matrix:
+        arch: [armv6, armv7, aarch64, s390x, ppc64le]
+        compiler: [gcc, clang]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: uraimo/run-on-arch-action@v2.0.5
+        with:
+          arch: ${{ matrix.arch }}
+          distro: buster
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update
+            apt-get install -y build-essential clang llvm libz-dev valgrind
+          run: |
+            # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64
+            # here.  (It's probably something related to the QEMU user-mode
+            # emulation that run-on-arch-action uses.)
+            export SKIP_VALGRIND=1
+            export SKIP_ASAN=1
+
+            case ${{ matrix.arch }} in
+            s390x)
+              # On s390x, in freestanding builds the shared library links to an
+              # external symbol __clzdi2, even when -static-libgcc is used.
+              export SKIP_FREESTANDING=1
+              ;;
+            aarch64)
+              # "ldd: exited with unknown exit code (139)"
+              if [ ${{ matrix.compiler }} = clang ]; then
+                export SKIP_SHARED_LIB=1
+              fi
+              ;;
+            esac
+
+            export CC=${{ matrix.compiler }}
+            scripts/run_tests.sh
+
+  macos-build-and-test:
+    name: Build and test (macOS)
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: make all check
+
+  windows-build-and-test:
+    name: Build and test (Windows)
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - shell: bash
+      run: |
+        PATH="C:\\msys64\\mingw64\\bin:C:\\msys64\\usr\\bin:$PATH" \
+          make CC=gcc all check
+
+  run-clang-static-analyzer:
+    name: Run clang static analyzer
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang-tools
+    - name: Run clang static analyzer
+      run: make scan-build
+
+  run-shellcheck:
+    name: Run shellcheck
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y shellcheck
+    - name: Run shellcheck
+      run: make shellcheck
+
+  cross-compile-for-windows:
+    name: Cross compile for Windows
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev
+    - name: 32-bit build
+      run: make CC=i686-w64-mingw32-gcc all test_programs
+    - name: 64-bit build
+      run: make CC=x86_64-w64-mingw32-gcc all test_programs

.gitignore

@@ -1,16 +1,21 @@
 *.a
+*.def
 *.dll
+*.dllobj
+*.dylib
 *.exe
 *.exp
 *.lib
 *.o
 *.obj
 *.so
-/.lib-cflags
-/.prog-cflags
+*.so.*
+/.build-config
 /programs/config.h
 /benchmark
+/checksum
 /gzip
 /gunzip
+/test_*
 tags
 cscope*

COPYING.LIB

@@ -1,121 +0,0 @@
-Creative Commons Legal Code
-
-CC0 1.0 Universal
-
-    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
-    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
-    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
-    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
-    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
-    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
-    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
-    HEREUNDER.
-
-Statement of Purpose
-
-The laws of most jurisdictions throughout the world automatically confer
-exclusive Copyright and Related Rights (defined below) upon the creator
-and subsequent owner(s) (each and all, an "owner") of an original work of
-authorship and/or a database (each, a "Work").
-
-Certain owners wish to permanently relinquish those rights to a Work for
-the purpose of contributing to a commons of creative, cultural and
-scientific works ("Commons") that the public can reliably and without fear
-of later claims of infringement build upon, modify, incorporate in other
-works, reuse and redistribute as freely as possible in any form whatsoever
-and for any purposes, including without limitation commercial purposes.
-These owners may contribute to the Commons to promote the ideal of a free
-culture and the further production of creative, cultural and scientific
-works, or to gain reputation or greater distribution for their Work in
-part through the use and efforts of others.
-
-For these and/or other purposes and motivations, and without any
-expectation of additional consideration or compensation, the person
-associating CC0 with a Work (the "Affirmer"), to the extent that he or she
-is an owner of Copyright and Related Rights in the Work, voluntarily
-elects to apply CC0 to the Work and publicly distribute the Work under its
-terms, with knowledge of his or her Copyright and Related Rights in the
-Work and the meaning and intended legal effect of CC0 on those rights.
-
-1. Copyright and Related Rights. A Work made available under CC0 may be
-protected by copyright and related or neighboring rights ("Copyright and
-Related Rights"). Copyright and Related Rights include, but are not
-limited to, the following:
-
-  i. the right to reproduce, adapt, distribute, perform, display,
-     communicate, and translate a Work;
- ii. moral rights retained by the original author(s) and/or performer(s);
-iii. publicity and privacy rights pertaining to a person's image or
-     likeness depicted in a Work;
- iv. rights protecting against unfair competition in regards to a Work,
-     subject to the limitations in paragraph 4(a), below;
-  v. rights protecting the extraction, dissemination, use and reuse of data
-     in a Work;
- vi. database rights (such as those arising under Directive 96/9/EC of the
-     European Parliament and of the Council of 11 March 1996 on the legal
-     protection of databases, and under any national implementation
-     thereof, including any amended or successor version of such
-     directive); and
-vii. other similar, equivalent or corresponding rights throughout the
-     world based on applicable law or treaty, and any national
-     implementations thereof.
-
-2. Waiver. To the greatest extent permitted by, but not in contravention
-of, applicable law, Affirmer hereby overtly, fully, permanently,
-irrevocably and unconditionally waives, abandons, and surrenders all of
-Affirmer's Copyright and Related Rights and associated claims and causes
-of action, whether now known or unknown (including existing as well as
-future claims and causes of action), in the Work (i) in all territories
-worldwide, (ii) for the maximum duration provided by applicable law or
-treaty (including future time extensions), (iii) in any current or future
-medium and for any number of copies, and (iv) for any purpose whatsoever,
-including without limitation commercial, advertising or promotional
-purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
-member of the public at large and to the detriment of Affirmer's heirs and
-successors, fully intending that such Waiver shall not be subject to
-revocation, rescission, cancellation, termination, or any other legal or
-equitable action to disrupt the quiet enjoyment of the Work by the public
-as contemplated by Affirmer's express Statement of Purpose.
-
-3. Public License Fallback. Should any part of the Waiver for any reason
-be judged legally invalid or ineffective under applicable law, then the
-Waiver shall be preserved to the maximum extent permitted taking into
-account Affirmer's express Statement of Purpose. In addition, to the
-extent the Waiver is so judged Affirmer hereby grants to each affected
-person a royalty-free, non transferable, non sublicensable, non exclusive,
-irrevocable and unconditional license to exercise Affirmer's Copyright and
-Related Rights in the Work (i) in all territories worldwide, (ii) for the
-maximum duration provided by applicable law or treaty (including future
-time extensions), (iii) in any current or future medium and for any number
-of copies, and (iv) for any purpose whatsoever, including without
-limitation commercial, advertising or promotional purposes (the
-"License"). The License shall be deemed effective as of the date CC0 was
-applied by Affirmer to the Work. Should any part of the License for any
-reason be judged legally invalid or ineffective under applicable law, such
-partial invalidity or ineffectiveness shall not invalidate the remainder
-of the License, and in such case Affirmer hereby affirms that he or she
-will not (i) exercise any of his or her remaining Copyright and Related
-Rights in the Work or (ii) assert any associated claims and causes of
-action with respect to the Work, in either case contrary to Affirmer's
-express Statement of Purpose.
-
-4. Limitations and Disclaimers.
-
- a. No trademark or patent rights held by Affirmer are waived, abandoned,
-    surrendered, licensed or otherwise affected by this document.
- b. Affirmer offers the Work as-is and makes no representations or
-    warranties of any kind concerning the Work, express, implied,
-    statutory or otherwise, including without limitation warranties of
-    title, merchantability, fitness for a particular purpose, non
-    infringement, or the absence of latent or other defects, accuracy, or
-    the present or absence of errors, whether or not discoverable, all to
-    the greatest extent permissible under applicable law.
- c. Affirmer disclaims responsibility for clearing rights of other persons
-    that may apply to the Work or any use thereof, including without
-    limitation any person's Copyright and Related Rights in the Work.
-    Further, Affirmer disclaims responsibility for obtaining any necessary
-    consents, permissions or other rights required for any use of the
-    Work.
- d. Affirmer understands and acknowledges that Creative Commons is not a
-    party to this document and has no duty or obligation with respect to
-    this CC0 or use of the Work.

Makefile

@@ -3,6 +3,9 @@
 #
 # Define V=1 to enable "verbose" mode, showing all executed commands.
 #
+# Define USE_SHARED_LIB to link the binaries to the shared library version of
+# libdeflate rather than to the static library version.
+#
 # Define DECOMPRESSION_ONLY to omit all compression code, building a
 # decompression-only library.  If doing this, you must also build a specific
 # library target such as 'libdeflate.a', as the programs will no longer compile.
@@ -11,45 +14,111 @@
 #
 # Define DISABLE_ZLIB to disable support for the zlib wrapper format.
 #
+# Define PREFIX to override the installation prefix, like './configure --prefix'
+# in autotools-based projects (default: /usr/local)
+#
+# Define BINDIR to override where to install binaries, like './configure
+# --bindir' in autotools-based projects (default: PREFIX/bin)
+#
+# Define INCDIR to override where to install headers, like './configure
+# --includedir' in autotools-based projects (default: PREFIX/include)
+#
+# Define LIBDIR to override where to install libraries, like './configure
+# --libdir' in autotools-based projects (default: PREFIX/lib)
+#
+# Define DESTDIR to override the installation destination directory
+# (default: empty string)
+#
+# Define FREESTANDING to build a freestanding library, i.e. a library that
+# doesn't link to any libc functions like malloc(), free(), and memcpy().
+# All users will need to call libdeflate_set_memory_allocator().
+#
+# You can also specify a custom CC, CFLAGS, CPPFLAGS, and/or LDFLAGS.
+#
 ##############################################################################
 
-#### Common compiler flags.
-#### Flags given here are not intended to be overridden, but you can add more
-#### by defining CFLAGS in the environment or on the 'make' command line.
+#### Common compiler flags.  You can add additional flags by defining CFLAGS
+#### in the environment or on the 'make' command line.
+####
+#### The default optimization flags can be overridden, e.g. via CFLAGS="-O3" or
+#### CFLAGS="-O0 -fno-omit-frame-pointer".  But this usually isn't recommended;
+#### you're unlikely to get significantly better performance even with -O3.
 
 cc-option = $(shell if $(CC) $(1) -c -x c /dev/null -o /dev/null \
 	      1>&2 2>/dev/null; then echo $(1); fi)
 
 override CFLAGS :=							\
-	$(CFLAGS) -O2 -fomit-frame-pointer -std=c99 -I. -Icommon	\
-	-Wall -Wundef							\
+	-O2 -fomit-frame-pointer -std=c99 -I. -Wall -Wundef		\
 	$(call cc-option,-Wpedantic)					\
 	$(call cc-option,-Wdeclaration-after-statement)			\
 	$(call cc-option,-Wmissing-prototypes)				\
 	$(call cc-option,-Wstrict-prototypes)				\
-	$(call cc-option,-Wvla)
+	$(call cc-option,-Wvla)						\
+	$(call cc-option,-Wimplicit-fallthrough)			\
+	$(CFLAGS)
+
+FREESTANDING :=
+ifdef FREESTANDING
+override CPPFLAGS += -DFREESTANDING
+LIB_CFLAGS += -ffreestanding -nostdlib
+endif
+
+# Don't use this option except for testing; it isn't a stable interface.
+TEST_SUPPORT__DO_NOT_USE :=
+ifdef TEST_SUPPORT__DO_NOT_USE
+override CPPFLAGS += -DTEST_SUPPORT__DO_NOT_USE
+endif
 
 ##############################################################################
 
-STATIC_LIB_SUFFIX := .a
-SHARED_LIB_SUFFIX := .so
-PROG_SUFFIX       :=
-PROG_CFLAGS       :=
-PIC_REQUIRED      := 1
-HARD_LINKS        := 1
+PREFIX ?= /usr/local
+BINDIR ?= $(PREFIX)/bin
+INCDIR ?= $(PREFIX)/include
+LIBDIR ?= $(PREFIX)/lib
+
+SOVERSION          := 0
+
+STATIC_LIB_SUFFIX  := .a
+PROG_SUFFIX        :=
+PROG_CFLAGS        :=
+HARD_LINKS         := 1
 
 # Compiling for Windows with MinGW?
-ifneq ($(findstring -mingw,$(CC)),)
+ifneq ($(findstring -mingw,$(shell $(CC) -dumpmachine 2>/dev/null)),)
+    STATIC_LIB_SUFFIX  := static.lib
+    SHARED_LIB         := libdeflate.dll
+    SHARED_LIB_SYMLINK :=
+    SHARED_LIB_CFLAGS  :=
+    SHARED_LIB_LDFLAGS := -Wl,--out-implib,libdeflate.lib \
+                          -Wl,--output-def,libdeflate.def \
+                          -Wl,--add-stdcall-alias
+    PROG_SUFFIX        := .exe
+    PROG_CFLAGS        := -static -municode
+    HARD_LINKS         :=
+    override CFLAGS    := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
+
+    # If AR was not already overridden, then derive it from $(CC).
+    # Note that CC may take different forms, e.g. "cc", "gcc",
+    # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1".
+    # On Windows it may also have a .exe extension.
     ifeq ($(AR),ar)
-        AR := $(patsubst %-gcc,%-ar,$(CC))
+        AR := $(shell echo $(CC) | \
+                sed -E 's/g?cc(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\3/')
     endif
-    STATIC_LIB_SUFFIX := .lib
-    SHARED_LIB_SUFFIX := .dll
-    PROG_SUFFIX       := .exe
-    PROG_CFLAGS       := -static -municode
-    PIC_REQUIRED      :=
-    HARD_LINKS        :=
-    override CFLAGS   := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
+
+# macOS?
+else ifeq ($(shell uname),Darwin)
+   SHARED_LIB         := libdeflate.$(SOVERSION).dylib
+   SHARED_LIB_SYMLINK := libdeflate.dylib
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -install_name $(SHARED_LIB)
+
+# Linux, FreeBSD, etc.
+else
+   SHARED_LIB         := libdeflate.so.$(SOVERSION)
+   SHARED_LIB_SYMLINK := libdeflate.so
+   SHARED_LIB_CFLAGS  := -fPIC
+   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
 endif
 
 ##############################################################################
@@ -69,19 +138,39 @@ endif
 
 ##############################################################################
 
-COMMON_HEADERS := $(wildcard common/*.h)
-ALL_TARGETS :=
+# Rebuild if a user-specified setting that affects the build changed.
+.build-config: FORCE
+	@flags=$$(							\
+		echo 'USE_SHARED_LIB=$(USE_SHARED_LIB)';		\
+		echo 'DECOMPRESSION_ONLY=$(DECOMPRESSION_ONLY)';	\
+		echo 'DISABLE_GZIP=$(DISABLE_GZIP)';			\
+		echo 'DISABLE_ZLIB=$(DISABLE_ZLIB)';			\
+		echo 'FREESTANDING=$(FREESTANDING)';			\
+		echo 'CC=$(CC)';					\
+		echo 'CFLAGS=$(CFLAGS)';				\
+		echo 'CPPFLAGS=$(CPPFLAGS)';				\
+		echo 'LDFLAGS=$(LDFLAGS)';				\
+	);								\
+	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then		\
+		[ -e $@ ] && echo "Rebuilding due to new settings";	\
+		echo "$$flags" > $@;					\
+	fi
+
+##############################################################################
+
+COMMON_HEADERS := $(wildcard common/*.h) libdeflate.h
+DEFAULT_TARGETS :=
 
 #### Library
 
 STATIC_LIB := libdeflate$(STATIC_LIB_SUFFIX)
-SHARED_LIB := libdeflate$(SHARED_LIB_SUFFIX)
 
 LIB_CFLAGS += $(CFLAGS) -fvisibility=hidden -D_ANSI_SOURCE
 
-LIB_HEADERS := $(wildcard lib/*.h)
+LIB_HEADERS := $(wildcard lib/*.h) $(wildcard lib/*/*.h)
 
-LIB_SRC := lib/aligned_malloc.c lib/deflate_decompress.c lib/x86_cpu_features.c
+LIB_SRC := lib/deflate_decompress.c lib/utils.c \
+	   $(wildcard lib/*/cpu_features.c)
 
 DECOMPRESSION_ONLY :=
 ifndef DECOMPRESSION_ONLY
@@ -104,79 +193,96 @@ ifndef DISABLE_GZIP
     endif
 endif
 
-LIB_OBJ := $(LIB_SRC:.c=.o)
-LIB_PIC_OBJ := $(LIB_SRC:.c=.pic.o)
-ifdef PIC_REQUIRED
-    SHLIB_OBJ := $(LIB_PIC_OBJ)
-else
-    SHLIB_OBJ := $(LIB_OBJ)
-endif
+STATIC_LIB_OBJ := $(LIB_SRC:.c=.o)
+SHARED_LIB_OBJ := $(LIB_SRC:.c=.shlib.o)
 
-# Compile position dependent library object files
-$(LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) $<
+# Compile static library object files
+$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) $<
 
-# Compile position independent library object files
-$(LIB_PIC_OBJ): %.pic.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .lib-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(LIB_CFLAGS) -fPIC $<
-
-# Link shared library
-$(SHARED_LIB):$(SHLIB_OBJ)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) -shared $+
-
-ALL_TARGETS += $(SHARED_LIB)
+# Compile shared library object files
+$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_CFLAGS) -DLIBDEFLATE_DLL $<
 
 # Create static library
-$(STATIC_LIB):$(LIB_OBJ)
+$(STATIC_LIB):$(STATIC_LIB_OBJ)
 	$(QUIET_AR) $(AR) cr $@ $+
 
-ALL_TARGETS += $(STATIC_LIB)
+DEFAULT_TARGETS += $(STATIC_LIB)
 
-# Rebuild if CC or LIB_CFLAGS changed
-.lib-cflags: FORCE
-	@flags='$(CC):$(LIB_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding library due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
+# Create shared library
+$(SHARED_LIB):$(SHARED_LIB_OBJ)
+	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) \
+		$(SHARED_LIB_LDFLAGS) -shared $+
+
+DEFAULT_TARGETS += $(SHARED_LIB)
+
+ifdef SHARED_LIB_SYMLINK
+# Create the symlink libdeflate.so => libdeflate.so.$SOVERSION
+$(SHARED_LIB_SYMLINK):$(SHARED_LIB)
+	$(QUIET_LN) ln -sf $+ $@
+DEFAULT_TARGETS += $(SHARED_LIB_SYMLINK)
+endif
 
 ##############################################################################
 
 #### Programs
 
-PROG_CFLAGS += $(CFLAGS)		\
-	       -D_DEFAULT_SOURCE	\
-	       -D_FILE_OFFSET_BITS=64	\
+PROG_CFLAGS += $(CFLAGS)		 \
+	       -D_POSIX_C_SOURCE=200809L \
+	       -D_FILE_OFFSET_BITS=64	 \
 	       -DHAVE_CONFIG_H
 
-PROG_COMMON_HEADERS := programs/prog_util.h programs/config.h
-PROG_COMMON_SRC := programs/prog_util.c programs/tgetopt.c
-PROG_SPECIFIC_SRC := programs/gzip.c programs/benchmark.c
+ALL_PROG_COMMON_HEADERS := programs/config.h \
+			   programs/prog_util.h \
+			   programs/test_util.h
+PROG_COMMON_SRC      := programs/prog_util.c \
+			programs/tgetopt.c
+NONTEST_PROG_SRC     := programs/gzip.c
+TEST_PROG_COMMON_SRC := programs/test_util.c
+TEST_PROG_SRC        := programs/benchmark.c \
+			programs/checksum.c \
+			$(filter-out $(TEST_PROG_COMMON_SRC),$(wildcard programs/test_*.c))
 
-PROG_COMMON_OBJ := $(PROG_COMMON_SRC:.c=.o)
-PROG_SPECIFIC_OBJ := $(PROG_SPECIFIC_SRC:.c=.o)
-PROG_OBJ := $(PROG_COMMON_OBJ) $(PROG_SPECIFIC_OBJ)
+NONTEST_PROGRAMS := $(NONTEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
+DEFAULT_TARGETS  += $(NONTEST_PROGRAMS)
+TEST_PROGRAMS    := $(TEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
+
+PROG_COMMON_OBJ      := $(PROG_COMMON_SRC:%.c=%.o)
+NONTEST_PROG_OBJ     := $(NONTEST_PROG_SRC:%.c=%.o)
+TEST_PROG_COMMON_OBJ := $(TEST_PROG_COMMON_SRC:%.c=%.o)
+TEST_PROG_OBJ        := $(TEST_PROG_SRC:%.c=%.o)
+
+ALL_PROG_OBJ	     := $(PROG_COMMON_OBJ) $(NONTEST_PROG_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(TEST_PROG_OBJ)
 
 # Generate autodetected configuration header
-programs/config.h:programs/detect.sh .prog-cflags
-	$(QUIET_GEN) CC=$(CC) $< > $@
+programs/config.h:scripts/detect.sh .build-config
+	$(QUIET_GEN) CC="$(CC)" CFLAGS="$(PROG_CFLAGS)" $< > $@
 
 # Compile program object files
-$(PROG_OBJ): %.o: %.c $(PROG_COMMON_HEADERS) $(COMMON_HEADERS) .prog-cflags
-	$(QUIET_CC) $(CC) -o $@ -c $(PROG_CFLAGS) $<
+$(ALL_PROG_OBJ): %.o: %.c $(ALL_PROG_COMMON_HEADERS) $(COMMON_HEADERS) \
+			.build-config
+	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(PROG_CFLAGS) $<
 
-# Link benchmark program
-benchmark$(PROG_SUFFIX):programs/benchmark.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
+# Link the programs.
+#
+# Note: the test programs are not compiled by default.  One reason is that the
+# test programs must be linked with zlib for doing comparisons.
 
-# 'benchmark' is not compiled by default because of the zlib link requirement.
-#ALL_TARGETS += benchmark$(PROG_SUFFIX)
+ifdef USE_SHARED_LIB
+LIB := $(SHARED_LIB)
+else
+LIB := $(STATIC_LIB)
+endif
 
-# Link gzip program
-gzip$(PROG_SUFFIX):programs/gzip.o $(PROG_COMMON_OBJ) $(STATIC_LIB)
+$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(LIB)
 	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+
 
-ALL_TARGETS += gzip$(PROG_SUFFIX)
+$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) \
+			$(TEST_PROG_COMMON_OBJ) $(LIB)
+	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
 
 ifdef HARD_LINKS
 # Hard link gunzip to gzip
@@ -188,39 +294,79 @@ gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
 	$(QUIET_CP) cp -f $< $@
 endif
 
-ALL_TARGETS += gunzip$(PROG_SUFFIX)
-
-# Rebuild if CC or PROG_CFLAGS changed
-.prog-cflags: FORCE
-	@flags='$(CC):$(PROG_CFLAGS)'; \
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \
-		[ -e $@ ] && echo "Rebuilding programs due to new compiler flags"; \
-		echo "$$flags" > $@; \
-	fi
+DEFAULT_TARGETS += gunzip$(PROG_SUFFIX)
 
 ##############################################################################
 
-all:$(ALL_TARGETS)
+all:$(DEFAULT_TARGETS)
+
+# Install the files.  Note: not all versions of the 'install' program have the
+# '-D' and '-t' options, so don't use them; use portable commands only.
+install:all
+	install -d $(DESTDIR)$(LIBDIR) $(DESTDIR)$(INCDIR) $(DESTDIR)$(BINDIR)
+	install -m644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)
+	install -m755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR)
+	install -m644 libdeflate.h $(DESTDIR)$(INCDIR)
+	install -m755 gzip$(PROG_SUFFIX) \
+		$(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)
+	ln -f $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)		\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX)
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		ln -sf $(SHARED_LIB)					\
+		       $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
+uninstall:
+	rm -f $(DESTDIR)$(LIBDIR)/$(STATIC_LIB)				\
+	      $(DESTDIR)$(LIBDIR)/$(SHARED_LIB)				\
+	      $(DESTDIR)$(INCDIR)/libdeflate.h				\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)		\
+	      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX)
+	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
+		rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
+	fi
+
+test_programs:$(TEST_PROGRAMS)
+
+# A minimal 'make check' target.  This only runs some quick tests;
+# use scripts/run_tests.sh if you want to run the full tests.
+check:test_programs
+	LD_LIBRARY_PATH=. ./benchmark$(PROG_SUFFIX) < ./benchmark$(PROG_SUFFIX)
+	for prog in test_*; do		\
+		LD_LIBRARY_PATH=. ./$$prog || exit 1;	\
+	done
+
+# Run the clang static analyzer.
+scan-build:
+	scan-build --status-bugs make all test_programs
+
+# Run shellcheck on all shell scripts.
+shellcheck:
+	shellcheck scripts/*.sh
 
 help:
 	@echo "Available targets:"
 	@echo "------------------"
-	@for target in $(ALL_TARGETS) benchmark$(PROG_SUFFIX); do \
+	@for target in $(DEFAULT_TARGETS) $(TEST_PROGRAMS); do \
 		echo -e "$$target";		\
 	done
 
 clean:
-	rm -f *.a *.dll *.exe *.exp *.so \
-		lib/*.o lib/*.obj programs/*.o programs/*.obj \
-		benchmark gzip gunzip programs/config.h \
-		libdeflate.lib libdeflatestatic.lib \
-		.lib-cflags .prog-cflags
+	rm -f *.a *.dll *.exe *.exp *.dylib *.so \
+		lib/*.o lib/*/*.o \
+		lib/*.obj lib/*/*.obj \
+		lib/*.dllobj lib/*/*.dllobj \
+		programs/*.o programs/*.obj \
+		$(DEFAULT_TARGETS) $(TEST_PROGRAMS) programs/config.h \
+		libdeflate.lib libdeflate.def libdeflatestatic.lib \
+		.build-config
 
 realclean: clean
 	rm -f tags cscope*
 
 FORCE:
 
-.PHONY: all help clean realclean
+.PHONY: all install uninstall test_programs check scan-build shellcheck help \
+	clean realclean
 
 .DEFAULT_GOAL = all

Makefile.msc

@@ -5,45 +5,53 @@
 #	nmake /f Makefile.msc
 #
 
+.SUFFIXES: .c .obj .dllobj
+
 CC = cl
 LD = link
 AR = lib
-CFLAGS = /MD /O2 -I. -Icommon
+CFLAGS = /MD /O2 -I.
 LDFLAGS =
 
-STATICLIB = libdeflatestatic.lib
-SHAREDLIB = libdeflate.dll
-IMPLIB    = libdeflate.lib
+STATIC_LIB = libdeflatestatic.lib
+SHARED_LIB = libdeflate.dll
+IMPORT_LIB = libdeflate.lib
 
-LIB_OBJ = lib/aligned_malloc.obj	\
+STATIC_LIB_OBJ =			\
 	  lib/adler32.obj		\
 	  lib/crc32.obj			\
 	  lib/deflate_compress.obj	\
 	  lib/deflate_decompress.obj	\
 	  lib/gzip_compress.obj		\
 	  lib/gzip_decompress.obj	\
-	  lib/x86_cpu_features.obj	\
+	  lib/utils.obj			\
+	  lib/x86/cpu_features.obj	\
 	  lib/zlib_compress.obj		\
 	  lib/zlib_decompress.obj
 
+SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj)
+
 PROG_COMMON_OBJ = programs/prog_util.obj \
 		  programs/tgetopt.obj \
-		  $(STATICLIB)
+		  $(STATIC_LIB)
 
 PROG_CFLAGS = $(CFLAGS) -Iprograms
 
-all: $(STATICLIB) $(SHAREDLIB) $(IMPLIB) gzip.exe gunzip.exe
+all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe
 
 .c.obj:
 	$(CC) -c /Fo$@ $(CFLAGS) $**
 
-$(STATICLIB): $(LIB_OBJ)
-	$(AR) $(ARFLAGS) -out:$@ $(LIB_OBJ)
+.c.dllobj:
+	$(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_DLL $**
 
-$(SHAREDLIB): $(LIB_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPLIB) $(LIB_OBJ)
+$(STATIC_LIB): $(STATIC_LIB_OBJ)
+	$(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ)
 
-$(IMPLIB): $(SHAREDLIB)
+$(SHARED_LIB): $(SHARED_LIB_OBJ)
+	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ)
+
+$(IMPORT_LIB): $(SHARED_LIB)
 
 gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ)
 	$(LD) $(LDFLAGS) -out:$@ $**
@@ -53,4 +61,5 @@ gunzip.exe:gzip.exe
 
 clean:
 	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
-		lib\*.obj programs\*.obj 2>nul
+		lib\*.obj lib\x86\*.obj lib\*.dllobj lib\x86\*.dllobj \
+		programs\*.obj 2>nul

NEWS

@@ -1,19 +0,0 @@
-Version 0.3:
-	Some bug fixes and other minor changes.
-
-Version 0.2:
-	Implemented a new block splitting algorithm which typically improves the
-	compression ratio slightly at all compression levels.
-
-	The compressor now outputs each block using the cheapest type (dynamic
-	Huffman, static Huffman, or uncompressed).
-
-	The gzip program has received an overhaul and now behaves more like the
-	standard version.
-
-	Build system updates, including: some build options were changed and
-	some build options were removed, and the default 'make' target now
-	includes the gzip program as well as the library.
-
-Version 0.1:
-	Initial official release.

NEWS.md

@@ -0,0 +1,230 @@
+# libdeflate release notes
+
+## Version 1.8
+
+* Added `-t` (test) option to `libdeflate-gunzip`.
+
+* Unaligned access optimizations are now enabled on WebAssembly builds.
+
+* Fixed a build error when building with the Intel C Compiler (ICC).
+
+* Fixed a build error when building with uClibc.
+
+* libdeflate's CI system has switched from Travis CI to GitHub Actions.
+
+* Made some improvements to test scripts.
+
+## Version 1.7
+
+* Added support for compression level 0, "no compression".
+
+* Added an ARM CRC32 instruction accelerated implementation of CRC32.
+
+* Added support for linking the programs to the shared library version of
+  libdeflate rather than to the static library version.
+
+* Made the compression level affect the minimum input size at which compression
+  is attempted.
+
+* Fixed undefined behavior in x86 Adler32 implementation.  (No miscompilations
+  were observed in practice.)
+
+* Fixed undefined behavior in x86 CPU feature code.  (No miscompilations were
+  observed in practice.)
+
+* Fixed installing shared lib symlink on macOS.
+
+* Documented third-party bindings.
+
+* Made a lot of improvements to the testing scripts and the CI configuration
+  file.
+
+* Lots of other small improvements and cleanups.
+
+## Version 1.6
+
+* Prevented gcc 10 from miscompiling libdeflate (workaround for
+  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+
+* Removed workaround for gcc 5 and earlier producing slow code on ARM32.  If
+  this affects you, please upgrade your compiler.
+
+* New API function: `libdeflate_zlib_decompress_ex()`.  It provides the actual
+  size of the stream that was decompressed, like the gzip and DEFLATE
+  equivalents.
+
+* `libdeflate_zlib_decompress()` now accepts trailing bytes after the end of the
+  stream, like the gzip and DEFLATE equivalents.
+
+* Added support for custom memory allocators.  (New API function:
+  `libdeflate_set_memory_allocator()`)
+
+* Added support for building the library in freestanding mode.
+
+* Building libdeflate no longer requires `CPPFLAGS=-Icommon`.
+
+## Version 1.5
+
+* Fixed up stdcall support on 32-bit Windows: the functions are now exported
+  using both suffixed and non-suffixed names, and fixed `libdeflate.h` to be
+  MSVC-compatible again.
+
+## Version 1.4
+
+* The 32-bit Windows build of libdeflate now uses the "stdcall" calling
+  convention instead of "cdecl".  If you're calling `libdeflate.dll` directly
+  from C or C++, you'll need to recompile your code.  If you're calling it from
+  another language, or calling it indirectly using `LoadLibrary()`, you'll need
+  to update your code to use the stdcall calling convention.
+
+* The Makefile now supports building libdeflate as a shared
+  library (`.dylib`) on macOS.
+
+* Fixed a bug where support for certain optimizations and optional features
+  (file access hints and more precise timestamps) was incorrectly omitted when
+  libdeflate was compiled with `-Werror`.
+
+* Added `make check` target to the Makefile.
+
+* Added CI configuration files.
+
+## Version 1.3
+
+* `make install` now supports customizing the directories into which binaries,
+  headers, and libraries are installed.
+
+* `make install` now installs into `/usr/local` by default.  To change it, use
+  e.g. `make install PREFIX=/usr`.
+
+* `make install` now works on more platforms.
+
+* The Makefile now supports overriding the optimization flags.
+
+* The compression functions now correctly handle an output data buffer >= 4 GiB
+  in size, and `gzip` and `gunzip` now correctly handle multi-gigabyte files (if
+  enough memory is available).
+
+## Version 1.2
+
+* Slight improvements to decompression speed.
+
+* Added an AVX-512BW implementation of Adler-32.
+
+* The Makefile now supports a user-specified installation `PREFIX`.
+
+* Fixed build error with some Visual Studio versions.
+
+## Version 1.1
+
+* Fixed crash in CRC-32 code when the prebuilt libdeflate for 32-bit Windows was
+  called by a program built with Visual Studio.
+
+* Improved the worst-case decompression speed of malicious data.
+
+* Fixed build error when compiling for an ARM processor without hardware
+  floating point support.
+
+* Improved performance on the PowerPC64 architecture.
+
+* Added soname to `libdeflate.so`, to make packaging easier.
+
+* Added `make install` target to the Makefile.
+
+* The Makefile now supports user-specified `CPPFLAGS`.
+
+* The Windows binary releases now include the import library for
+  `libdeflate.dll`.  `libdeflate.lib` is now the import library, and
+  `libdeflatestatic.lib` is the static library.
+
+## Version 1.0
+
+* Added support for multi-member gzip files.
+
+* Moved architecture-specific code into subdirectories.  If you aren't using the
+  provided Makefile to build libdeflate, you now need to compile `lib/*.c` and
+  `lib/*/*.c` instead of just `lib/*.c`.
+
+* Added an ARM PMULL implementation of CRC-32, which speeds up gzip compression
+  and decompression on 32-bit and 64-bit ARM processors that have the
+  Cryptography Extensions.
+
+* Improved detection of CPU features, resulting in accelerated functions being
+  used in more cases.  This includes:
+
+  * Detect CPU features on 32-bit x86, not just 64-bit as was done previously.
+
+  * Detect CPU features on ARM, both 32 and 64-bit.  (Limited to Linux only
+    currently.)
+
+## Version 0.8
+
+* Build fixes for certain platforms and compilers.
+
+* libdeflate now produces the same output on all CPU architectures.
+
+* Improved documentation for building libdeflate on Windows.
+
+## Version 0.7
+
+* Fixed a very rare bug that caused data to be compressed incorrectly.  The bug
+  affected compression levels 7 and below since libdeflate v0.2.  Although there
+  have been no user reports of the bug, and I believe it would have been highly
+  unlikely to encounter on realistic data, it could occur on data specially
+  crafted to reproduce it.
+
+* Fixed a compilation error when building with clang 3.7.
+
+## Version 0.6
+
+* Various improvements to the gzip program's behavior.
+
+* Faster CRC-32 on AVX-capable processors.
+
+* Other minor changes.
+
+## Version 0.5
+
+* The CRC-32 checksum algorithm has been optimized with carryless multiplication
+  instructions for `x86_64` (PCLMUL).  This speeds up gzip compression and
+  decompression.
+
+* Build fixes for certain platforms and compilers.
+
+* Added more test programs and scripts.
+
+* libdeflate is now entirely MIT-licensed.
+
+## Version 0.4
+
+* The Adler-32 checksum algorithm has been optimized with vector instructions
+  for `x86_64` (SSE2 and AVX2) and ARM (NEON).  This speeds up zlib compression
+  and decompression.
+
+* To avoid naming collisions, functions and definitions in libdeflate's API have
+  been renamed to be prefixed with `libdeflate_` or `LIBDEFLATE_`.  Programs
+  using the old API will need to be updated.
+
+* Various bug fixes and other improvements.
+
+## Version 0.3
+
+* Some bug fixes and other minor changes.
+
+## Version 0.2
+
+* Implemented a new block splitting algorithm which typically improves the
+  compression ratio slightly at all compression levels.
+
+* The compressor now outputs each block using the cheapest type (dynamic
+  Huffman, static Huffman, or uncompressed).
+
+* The gzip program has received an overhaul and now behaves more like the
+  standard version.
+
+* Build system updates, including: some build options were changed and some
+  build options were removed, and the default 'make' target now includes the
+  gzip program as well as the library.
+
+## Version 0.1
+
+* Initial official release.

README.md

@@ -18,39 +18,149 @@ libdeflate itself is a library, but the following command-line programs which
 use this library are also provided:
 
 * gzip (or gunzip), a program which mostly behaves like the standard equivalent,
-  except that it does not yet support reading from standard input and does not
+  except that it does not yet have good streaming support and therefore does not
   yet support very large files
 * benchmark, a program for benchmarking in-memory compression and decompression
 
+For the release notes, see the [NEWS file](NEWS.md).
+
+## Table of Contents
+
+- [Building](#building)
+  - [For UNIX](#for-unix)
+  - [For macOS](#for-macos)
+  - [For Windows](#for-windows)
+    - [Using Cygwin](#using-cygwin)
+    - [Using MSYS2](#using-msys2)
+- [API](#api)
+- [Bindings for other programming languages](#bindings-for-other-programming-languages)
+- [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip)
+- [Compression levels](#compression-levels)
+- [Motivation](#motivation)
+- [License](#license)
+
+
 # Building
 
 ## For UNIX
 
-Just run `make`.  You need GNU Make and either GCC or Clang.  GCC is recommended
-because it builds slightly faster binaries.  There is no `make install` yet;
-just copy the file(s) to where you want.
+Just run `make`, then (if desired) `make install`.  You need GNU Make and either
+GCC or Clang.  GCC is recommended because it builds slightly faster binaries.
 
-By default, all targets are built, including the library and programs, with the
-exception of the `benchmark` program.  `make help` shows the available targets.
-There are also several options which can be set on the `make` command line.  See
-the Makefile for details.
+By default, the following targets are built: the static library `libdeflate.a`,
+the shared library `libdeflate.so`, the `gzip` program, and the `gunzip` program
+(which is actually just a hard link to `gzip`).  Benchmarking and test programs
+such as `benchmark` are not built by default.  You can run `make help` to
+display the available build targets.
+
+There are also many options which can be set on the `make` command line, e.g. to
+omit library features or to customize the directories into which `make install`
+installs files.  See the Makefile for details.
+
+## For macOS
+
+Prebuilt macOS binaries can be installed with [Homebrew](https://brew.sh):
+
+    brew install libdeflate
+
+But if you need to build the binaries yourself, see the section for UNIX above.
 
 ## For Windows
 
-MinGW (GCC) is the recommended compiler to use when building binaries for
-Windows.  MinGW can be used on either Windows or Linux.  Use a command like:
+Prebuilt Windows binaries can be downloaded from
+https://github.com/ebiggers/libdeflate/releases.  But if you need to build the
+binaries yourself, MinGW (gcc) is the recommended compiler to use.  If you're
+performing the build *on* Windows (as opposed to cross-compiling for Windows on
+Linux, for example), you'll need to follow the directions in **one** of the two
+sections below to set up a minimal UNIX-compatible environment using either
+Cygwin or MSYS2, then do the build.  (Other MinGW distributions may not work, as
+they often omit basic UNIX tools such as `sh`.)
 
-    $ make CC=x86_64-w64-mingw32-gcc
+Alternatively, libdeflate may be built using the Visual Studio toolchain by
+running `nmake /f Makefile.msc`.  However, while this is supported in the sense
+that it will produce working binaries, it is not recommended because the
+binaries built with MinGW will be significantly faster.
 
-Windows binaries prebuilt with MinGW may also be downloaded from
-https://github.com/ebiggers/libdeflate/releases.
-
-Alternatively, a separate Makefile, `Makefile.msc`, is provided for the tools
-that come with Visual Studio, for those who strongly prefer that toolchain.
-
-As usual, 64-bit binaries are faster than 32-bit binaries and should be
+Also note that 64-bit binaries are faster than 32-bit binaries and should be
 preferred whenever possible.
 
+### Using Cygwin
+
+Run the Cygwin installer, available from https://cygwin.com/setup-x86_64.exe.
+When you get to the package selection screen, choose the following additional
+packages from category "Devel":
+
+- git
+- make
+- mingw64-i686-binutils
+- mingw64-i686-gcc-g++
+- mingw64-x86_64-binutils
+- mingw64-x86_64-gcc-g++
+
+(You may skip the mingw64-i686 packages if you don't need to build 32-bit
+binaries.)
+
+After the installation finishes, open a Cygwin terminal.  Then download
+libdeflate's source code (if you haven't already) and `cd` into its directory:
+
+    git clone https://github.com/ebiggers/libdeflate
+    cd libdeflate
+
+(Note that it's not required to use `git`; an alternative is to extract a .zip
+or .tar.gz archive of the source code downloaded from the releases page.
+Also, in case you need to find it in the file browser, note that your home
+directory in Cygwin is usually located at `C:\cygwin64\home\<your username>`.)
+
+Then, to build 64-bit binaries:
+
+    make CC=x86_64-w64-mingw32-gcc
+
+or to build 32-bit binaries:
+
+    make CC=i686-w64-mingw32-gcc
+
+### Using MSYS2
+
+Run the MSYS2 installer, available from http://www.msys2.org/.  After
+installing, open an MSYS2 shell and run:
+
+    pacman -Syu
+
+Say `y`, then when it's finished, close the shell window and open a new one.
+Then run the same command again:
+
+    pacman -Syu
+
+Then, install the packages needed to build libdeflate:
+
+    pacman -S git \
+              make \
+              mingw-w64-i686-binutils \
+              mingw-w64-i686-gcc \
+              mingw-w64-x86_64-binutils \
+              mingw-w64-x86_64-gcc
+
+(You may skip the mingw-w64-i686 packages if you don't need to build 32-bit
+binaries.)
+
+Then download libdeflate's source code (if you haven't already):
+
+    git clone https://github.com/ebiggers/libdeflate
+
+(Note that it's not required to use `git`; an alternative is to extract a .zip
+or .tar.gz archive of the source code downloaded from the releases page.
+Also, in case you need to find it in the file browser, note that your home
+directory in MSYS2 is usually located at `C:\msys64\home\<your username>`.)
+
+Then, to build 64-bit binaries, open "MSYS2 MinGW 64-bit" from the Start menu
+and run the following commands:
+
+    cd libdeflate
+    make clean
+    make
+
+Or to build 32-bit binaries, do the same but use "MSYS2 MinGW 32-bit" instead.
+
 # API
 
 libdeflate has a simple API that is not zlib-compatible.  You can create
@@ -72,6 +182,30 @@ This enables you to allocate an output buffer of the correct size without
 guessing.  However, libdeflate's decompression routines do optionally provide
 the actual number of output bytes in case you need it.
 
+Windows developers: note that the calling convention of libdeflate.dll is
+"stdcall" -- the same as the Win32 API.  If you call into libdeflate.dll using a
+non-C/C++ language, or dynamically using LoadLibrary(), make sure to use the
+stdcall convention.  Using the wrong convention may crash your application.
+(Note: older versions of libdeflate used the "cdecl" convention instead.)
+
+# Bindings for other programming languages
+
+The libdeflate project itself only provides a C library.  If you need to use
+libdeflate from a programming language other than C or C++, consider using the
+following bindings:
+
+* C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET)
+* Go: [go-libdeflate](https://github.com/4kills/go-libdeflate)
+* Java: [libdeflate-java](https://github.com/astei/libdeflate-java)
+* Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl)
+* Python: [deflate](https://github.com/dcwatson/deflate)
+* Ruby: [libdeflate-ruby](https://github.com/kaorimatz/libdeflate-ruby)
+* Rust: [libdeflater](https://github.com/adamkewley/libdeflater)
+
+Note: these are third-party projects which haven't necessarily been vetted by
+the authors of libdeflate.  Please direct all questions, bugs, and improvements
+for these bindings to their authors.
+
 # DEFLATE vs. zlib vs. gzip
 
 The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib
@@ -144,16 +278,8 @@ processors.  libdeflate addresses this by providing an optimized DEFLATE
 implementation which can be used for benchmarking purposes.  And, of course,
 real applications can use it as well.
 
-That being said, I have also started [a separate
-project](https://github.com/ebiggers/xpack) for an experimental, more modern
-compression format.
-
 # License
 
-The library portion of libdeflate has been released into the public domain.
-There is NO WARRANTY, to the extent permitted by law.  See the CC0 Public Domain
-Dedication in the COPYING.LIB file for details.
+libdeflate is [MIT-licensed](COPYING).
 
-The non-library portions of the project, such as the gzip program, may be
-modified and/or redistributed under the terms of the MIT license.  There is NO
-WARRANTY, to the extent permitted by law.  See the COPYING file for details.
+I am not aware of any patents or patent applications relevant to libdeflate.

common/common_defs.h

@@ -47,9 +47,7 @@
 #endif
 
 /* Fixed-width integer types */
-#ifndef PRIu32
-#  include <inttypes.h>
-#endif
+#include <stdint.h>
 typedef uint8_t u8;
 typedef uint16_t u16;
 typedef uint32_t u32;
@@ -76,7 +74,7 @@ typedef size_t machine_word_t;
 /*                         Optional compiler features                         */
 /* ========================================================================== */
 
-/* LIBEXPORT - annotate a function that is part of the library API */
+/* LIBEXPORT - export a function from a shared library */
 #ifndef LIBEXPORT
 #  define LIBEXPORT
 #endif
@@ -121,10 +119,42 @@ typedef size_t machine_word_t;
 #  define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
 #endif
 
-/* Does the compiler support __attribute__((target("bmi2")))? */
+/* Which targets are supported with the 'target' function attribute? */
 #ifndef COMPILER_SUPPORTS_BMI2_TARGET
 #  define COMPILER_SUPPORTS_BMI2_TARGET 0
 #endif
+#ifndef COMPILER_SUPPORTS_AVX_TARGET
+#  define COMPILER_SUPPORTS_AVX_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET 0
+#endif
+
+/*
+ * Which targets are supported with the 'target' function attribute and have
+ * intrinsics that work within 'target'-ed functions?
+ */
+#ifndef COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0
+#endif
 
 /* _aligned_attribute(n) - declare that the annotated variable, or variables of
  * the annotated type, are to be aligned on n-byte boundaries */
@@ -140,6 +170,7 @@ typedef size_t machine_word_t;
 #define MAX(a, b)		((a) >= (b) ? (a) : (b))
 #define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
 #define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
 
 /* ========================================================================== */
 /*                           Endianness handling                              */
@@ -216,39 +247,6 @@ static forceinline u64 bswap64(u64 n)
 #  define UNALIGNED_ACCESS_IS_FAST 0
 #endif
 
-/*
- * DEFINE_UNALIGNED_TYPE(type) - a macro that, given an integer type 'type',
- * defines load_type_unaligned(addr) and store_type_unaligned(v, addr) functions
- * which load and store variables of type 'type' from/to unaligned memory
- * addresses.  If not defined, a fallback is used.
- */
-#ifndef DEFINE_UNALIGNED_TYPE
-
-/*
- * Although memcpy() may seem inefficient, it *usually* gets optimized
- * appropriately by modern compilers.  It's portable and may be the best we can
- * do for a fallback...
- */
-#include <string.h>
-
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	type v;							\
-	memcpy(&v, p, sizeof(v));				\
-	return v;						\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	memcpy(p, &v, sizeof(v));				\
-}
-
-#endif /* !DEFINE_UNALIGNED_TYPE */
-
 /* ========================================================================== */
 /*                             Bit scan functions                             */
 /* ========================================================================== */

common/compiler_gcc.h

@@ -1,12 +1,33 @@
 /*
- * compiler_gcc.h - definitions for the GNU C Compiler.  Currently this also
- * handles clang and the Intel C Compiler.
+ * compiler_gcc.h - definitions for the GNU C Compiler.  This also handles clang
+ * and the Intel C Compiler (icc).
+ *
+ * TODO: icc is not well tested, so some things are currently disabled even
+ * though they maybe can be enabled on some icc versions.
  */
 
-#define GCC_PREREQ(major, minor)					\
-	(!defined(__clang__) && !defined(__INTEL_COMPILER) &&		\
-	 (__GNUC__ > (major) ||						\
-	  (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
+#if !defined(__clang__) && !defined(__INTEL_COMPILER)
+#  define GCC_PREREQ(major, minor)		\
+	(__GNUC__ > (major) ||			\
+	 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#  define GCC_PREREQ(major, minor)	0
+#endif
+
+/* Note: only check the clang version when absolutely necessary!
+ * "Vendors" such as Apple can use different version numbers. */
+#ifdef __clang__
+#  ifdef __apple_build_version__
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__apple_build_version__ >= (apple_version))
+#  else
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__clang_major__ > (major) ||			\
+	 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+#  endif
+#else
+#  define CLANG_PREREQ(major, minor, apple_version)	0
+#endif
 
 #ifndef __has_attribute
 #  define __has_attribute(attribute)	0
@@ -33,12 +54,116 @@
 #define prefetchw(addr)		__builtin_prefetch((addr), 1)
 #define _aligned_attribute(n)	__attribute__((aligned(n)))
 
-#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE		\
+#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE	\
 	(GCC_PREREQ(4, 4) || __has_attribute(target))
 
-#define COMPILER_SUPPORTS_BMI2_TARGET				\
-	(COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE &&		\
-	 (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di)))
+#if COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+
+#  if defined(__i386__) || defined(__x86_64__)
+
+#    define COMPILER_SUPPORTS_PCLMUL_TARGET	\
+	(GCC_PREREQ(4, 4) || __has_builtin(__builtin_ia32_pclmulqdq128))
+
+#    define COMPILER_SUPPORTS_AVX_TARGET	\
+	(GCC_PREREQ(4, 6) || __has_builtin(__builtin_ia32_maxps256))
+
+#    define COMPILER_SUPPORTS_BMI2_TARGET	\
+	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di))
+
+#    define COMPILER_SUPPORTS_AVX2_TARGET	\
+	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256))
+
+#    define COMPILER_SUPPORTS_AVX512BW_TARGET	\
+	(GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512))
+
+	/*
+	 * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics
+	 * not available in the main target could not be used in 'target'
+	 * attribute functions.  Unfortunately clang has no feature test macro
+	 * for this so we have to check its version.
+	 */
+#    if GCC_PREREQ(4, 9) || CLANG_PREREQ(3, 8, 7030000)
+#      define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS	1
+#      define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_PCLMUL_TARGET
+#      define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_AVX2_TARGET
+#      define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS	\
+		COMPILER_SUPPORTS_AVX512BW_TARGET
+#    endif
+
+#  elif defined(__arm__) || defined(__aarch64__)
+
+    /*
+     * Determine whether NEON and crypto intrinsics are supported.
+     *
+     * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither
+     * was available unless enabled in the main target.
+     *
+     * But even after that, to include <arm_neon.h> (which contains both the
+     * basic NEON intrinsics and the crypto intrinsics) the main target still
+     * needs to have:
+     *   - gcc: hardware floating point support
+     *   - clang: NEON support (but not necessarily crypto support)
+     */
+#    if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \
+        (defined(__clang__) && defined(__ARM_NEON))
+#      define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1
+       /*
+        * The crypto intrinsics are broken on arm32 with clang, even when using
+        * -mfpu=crypto-neon-fp-armv8, because clang's <arm_neon.h> puts them
+        * behind __aarch64__.  Undefine __ARM_FEATURE_CRYPTO in that case...
+        */
+#      if defined(__clang__) && defined(__arm__)
+#        undef __ARM_FEATURE_CRYPTO
+#      elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__)
+#        define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1
+#      endif
+#    endif
+
+     /*
+      * Determine whether CRC32 intrinsics are supported.
+      *
+      * With gcc r274827 or later (gcc 10.1+, 9.3+, or 8.4+), or with clang,
+      * they work as expected.  (Well, not quite.  There's still a bug, but we
+      * have to work around it later when including arm_acle.h.)
+      */
+#    if GCC_PREREQ(10, 1) || \
+        (GCC_PREREQ(9, 3) && !GCC_PREREQ(10, 0)) || \
+        (GCC_PREREQ(8, 4) && !GCC_PREREQ(9, 0)) || \
+        (defined(__clang__) && __has_builtin(__builtin_arm_crc32b))
+#      define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1
+#    endif
+
+#  endif /* __arm__ || __aarch64__ */
+
+#endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */
+
+/*
+ * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed
+ * integers (e.g. __v4si), not vectors of unsigned integers (e.g.  __v4su).  But
+ * we need the unsigned ones in order to avoid signed integer overflow, which is
+ * undefined behavior.  Add the missing definitions for the unsigned ones if
+ * needed.
+ */
+#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \
+    (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) || \
+    defined(__INTEL_COMPILER)
+typedef unsigned long long  __v2du __attribute__((__vector_size__(16)));
+typedef unsigned int        __v4su __attribute__((__vector_size__(16)));
+typedef unsigned short      __v8hu __attribute__((__vector_size__(16)));
+typedef unsigned char      __v16qu __attribute__((__vector_size__(16)));
+typedef unsigned long long  __v4du __attribute__((__vector_size__(32)));
+typedef unsigned int        __v8su __attribute__((__vector_size__(32)));
+typedef unsigned short     __v16hu __attribute__((__vector_size__(32)));
+typedef unsigned char      __v32qu __attribute__((__vector_size__(32)));
+#endif
+
+#ifdef __INTEL_COMPILER
+typedef int   __v16si __attribute__((__vector_size__(64)));
+typedef short __v32hi __attribute__((__vector_size__(64)));
+typedef char  __v64qi __attribute__((__vector_size__(64)));
+#endif
 
 /* Newer gcc supports __BYTE_ORDER__.  Older gcc doesn't. */
 #ifdef __BYTE_ORDER__
@@ -57,29 +182,19 @@
 #  define bswap64	__builtin_bswap64
 #endif
 
-#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED)
+#if defined(__x86_64__) || defined(__i386__) || \
+    defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
+    /*
+     * For all compilation purposes, WebAssembly behaves like any other CPU
+     * instruction set. Even though WebAssembly engine might be running on top
+     * of different actual CPU architectures, the WebAssembly spec itself
+     * permits unaligned access and it will be fast on most of those platforms,
+     * and simulated at the engine level on others, so it's worth treating it
+     * as a CPU architecture with fast unaligned access.
+    */ defined(__wasm__)
 #  define UNALIGNED_ACCESS_IS_FAST 1
 #endif
 
-/* With gcc, we can access unaligned memory through 'packed' structures. */
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-struct type##unaligned {					\
-	type v;							\
-} __attribute__((packed));					\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	return ((const struct type##unaligned *)p)->v;		\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	((struct type##unaligned *)p)->v = v;			\
-}
-
 #define bsr32(n)	(31 - __builtin_clz(n))
 #define bsr64(n)	(63 - __builtin_clzll(n))
 #define bsf32(n)	__builtin_ctz(n)

common/compiler_msc.h

@@ -2,7 +2,8 @@
  * compiler_msc.h - definitions for the Microsoft C Compiler
  */
 
-#define BUILDING_LIBDEFLATE
+#include <stdint.h>
+#include <stdlib.h> /* for _byteswap_*() */
 
 #define LIBEXPORT	__declspec(dllexport)
 
@@ -24,24 +25,6 @@ typedef long long ssize_t;
 typedef int ssize_t;
 #endif
 
-/*
- * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
- * inttypes.h.  Work around this by defining the PRI* macros ourselves.
- */
-#include <stdint.h>
-#define PRIu8  "hhu"
-#define PRIu16 "hu"
-#define PRIu32 "u"
-#define PRIu64 "llu"
-#define PRIi8  "hhi"
-#define PRIi16 "hi"
-#define PRIi32 "i"
-#define PRIi64 "lli"
-#define PRIx8  "hhx"
-#define PRIx16 "hx"
-#define PRIx32 "x"
-#define PRIx64 "llx"
-
 /* Assume a little endian architecture with fast unaligned access */
 #define CPU_IS_LITTLE_ENDIAN()		1
 #define UNALIGNED_ACCESS_IS_FAST	1

lib/adler32.c

@@ -1,26 +1,38 @@
 /*
  * adler32.c - Adler-32 checksum algorithm
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
+#include "lib_common.h"
+#include "libdeflate.h"
 
-/*
- * The Adler-32 divisor, or "base", value.
- */
+/* The Adler-32 divisor, or "base", value. */
 #define DIVISOR 65521
 
 /*
- * MAX_BYTES_PER_CHUNK is the most bytes that can be processed without the
+ * MAX_CHUNK_SIZE is the most bytes that can be processed without the
  * possibility of s2 overflowing when it is represented as an unsigned 32-bit
  * integer.  This value was computed using the following Python script:
  *
@@ -40,58 +52,43 @@
  * has value 0xFF and that s1 and s2 started with the highest possible values
  * modulo the divisor.
  */
-#define MAX_BYTES_PER_CHUNK	5552
+#define MAX_CHUNK_SIZE	5552
 
-/* Number of bytes to process per loop iteration  */
-#define UNROLL_FACTOR	4
+typedef u32 (*adler32_func_t)(u32, const u8 *, size_t);
 
-u32
-adler32(const void *buffer, size_t size)
+/* Include architecture-specific implementations if available */
+#undef DEFAULT_IMPL
+#undef DISPATCH
+#if defined(__arm__) || defined(__aarch64__)
+#  include "arm/adler32_impl.h"
+#elif defined(__i386__) || defined(__x86_64__)
+#  include "x86/adler32_impl.h"
+#endif
+
+/* Define a generic implementation if needed */
+#ifndef DEFAULT_IMPL
+#define DEFAULT_IMPL adler32_generic
+static u32 adler32_generic(u32 adler, const u8 *p, size_t size)
 {
-	u32 s1 = 1;
-	u32 s2 = 0;
-	const u8 *p = buffer;
-	const u8 *end = p + size;
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+	const u8 * const end = p + size;
+
 	while (p != end) {
-		size_t chunk_size = MIN(end - p, MAX_BYTES_PER_CHUNK);
+		size_t chunk_size = MIN(end - p, MAX_CHUNK_SIZE);
 		const u8 *chunk_end = p + chunk_size;
+		size_t num_unrolled_iterations = chunk_size / 4;
 
-	#if UNROLL_FACTOR > 1
-		size_t num_unrolled_iterations = chunk_size / UNROLL_FACTOR;
 		while (num_unrolled_iterations--) {
 			s1 += *p++;
 			s2 += s1;
-		#if UNROLL_FACTOR >= 2
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 3
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 4
 			s1 += *p++;
 			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 5
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 6
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 7
-			s1 += *p++;
-			s2 += s1;
-		#endif
-		#if UNROLL_FACTOR >= 8
-			s1 += *p++;
-			s2 += s1;
-		#endif
-			STATIC_ASSERT(UNROLL_FACTOR <= 8);
 		}
-	#endif /* UNROLL_FACTOR > 1 */
 		while (p != chunk_end) {
 			s1 += *p++;
 			s2 += s1;
@@ -99,5 +96,35 @@ adler32(const void *buffer, size_t size)
 		s1 %= DIVISOR;
 		s2 %= DIVISOR;
 	}
+
 	return (s2 << 16) | s1;
 }
+#endif /* !DEFAULT_IMPL */
+
+#ifdef DISPATCH
+static u32 dispatch(u32, const u8 *, size_t);
+
+static volatile adler32_func_t adler32_impl = dispatch;
+
+/* Choose the fastest implementation at runtime */
+static u32 dispatch(u32 adler, const u8 *buffer, size_t size)
+{
+	adler32_func_t f = arch_select_adler32_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
+	adler32_impl = f;
+	return adler32_impl(adler, buffer, size);
+}
+#else
+#  define adler32_impl DEFAULT_IMPL /* only one implementation, use it */
+#endif
+
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
+libdeflate_adler32(u32 adler, const void *buffer, size_t size)
+{
+	if (buffer == NULL) /* return initial value */
+		return 1;
+	return adler32_impl(adler, buffer, size);
+}

lib/adler32.h

@@ -1,12 +0,0 @@
-/*
- * adler32.h - Adler-32 checksum algorithm
- */
-
-#ifndef _LIB_ADLER32_H
-#define _LIB_ADLER32_H
-
-#include "common_defs.h"
-
-extern u32 adler32(const void *buffer, size_t size);
-
-#endif /* _LIB_ADLER32_H */

lib/adler32_vec_template.h

@@ -0,0 +1,124 @@
+/*
+ * adler32_vec_template.h - template for vectorized Adler-32 implementations
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file contains a template for vectorized Adler-32 implementations.
+ *
+ * The inner loop between reductions modulo 65521 of an unvectorized Adler-32
+ * implementation looks something like this:
+ *
+ *	do {
+ *		s1 += *p;
+ *		s2 += s1;
+ *	} while (++p != chunk_end);
+ *
+ * For vectorized calculation of s1, we only need to sum the input bytes.  They
+ * can be accumulated into multiple counters which are eventually summed
+ * together.
+ *
+ * For vectorized calculation of s2, the basic idea is that for each iteration
+ * that processes N bytes, we can perform the following vectorizable
+ * calculation:
+ *
+ *	s2 += N*byte_1 + (N-1)*byte_2 + (N-2)*byte_3 + ... + 1*byte_N
+ *
+ * Or, equivalently, we can sum the byte_1...byte_N for each iteration into N
+ * separate counters, then do the multiplications by N...1 just once at the end
+ * rather than once per iteration.
+ *
+ * Also, we must account for how previous bytes will affect s2 by doing the
+ * following at beginning of each iteration:
+ *
+ *	s2 += s1 * N
+ *
+ * Furthermore, like s1, "s2" can actually be multiple counters which are
+ * eventually summed together.
+ */
+
+static u32 ATTRIBUTES
+FUNCNAME(u32 adler, const u8 *p, size_t size)
+{
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+	const u8 * const end = p + size;
+	const u8 *vend;
+	const size_t max_chunk_size =
+		MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) -
+		(MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) %
+		 IMPL_SEGMENT_SIZE);
+
+	/* Process a byte at a time until the needed alignment is reached */
+	if (p != end && (uintptr_t)p % IMPL_ALIGNMENT) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+		} while (p != end && (uintptr_t)p % IMPL_ALIGNMENT);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	/*
+	 * Process "chunks" of bytes using vector instructions.  Chunk sizes are
+	 * limited to MAX_CHUNK_SIZE, which guarantees that s1 and s2 never
+	 * overflow before being reduced modulo DIVISOR.  For vector processing,
+	 * chunk sizes are also made evenly divisible by IMPL_SEGMENT_SIZE and
+	 * may be further limited to IMPL_MAX_CHUNK_SIZE.
+	 */
+	STATIC_ASSERT(IMPL_SEGMENT_SIZE % IMPL_ALIGNMENT == 0);
+	vend = end - ((size_t)(end - p) % IMPL_SEGMENT_SIZE);
+	while (p != vend) {
+		size_t chunk_size = MIN((size_t)(vend - p), max_chunk_size);
+
+		s2 += s1 * chunk_size;
+
+		FUNCNAME_CHUNK((const void *)p, (const void *)(p + chunk_size),
+			       &s1, &s2);
+
+		p += chunk_size;
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	/* Process any remaining bytes */
+	if (p != end) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+		} while (p != end);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	return (s2 << 16) | s1;
+}
+
+#undef FUNCNAME
+#undef FUNCNAME_CHUNK
+#undef ATTRIBUTES
+#undef IMPL_ALIGNMENT
+#undef IMPL_SEGMENT_SIZE
+#undef IMPL_MAX_CHUNK_SIZE

lib/aligned_malloc.c

@@ -1,46 +0,0 @@
-/*
- * aligned_malloc.c - aligned memory allocation
- *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
- *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
- *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
- */
-
-/*
- * This file provides portable aligned memory allocation functions that only
- * use malloc() and free().  This avoids portability problems with
- * posix_memalign(), aligned_alloc(), etc.
- */
-
-#include <stdlib.h>
-
-#include "aligned_malloc.h"
-
-void *
-aligned_malloc(size_t alignment, size_t size)
-{
-	const uintptr_t mask = alignment - 1;
-	char *ptr = NULL;
-	char *raw_ptr;
-
-	raw_ptr = malloc(mask + sizeof(size_t) + size);
-	if (raw_ptr) {
-		ptr = (char *)raw_ptr + sizeof(size_t);
-		ptr = (void *)(((uintptr_t)ptr + mask) & ~mask);
-		*((size_t *)ptr - 1) = ptr - raw_ptr;
-	}
-	return ptr;
-}
-
-void
-aligned_free(void *ptr)
-{
-	if (ptr)
-		free((char *)ptr - *((size_t *)ptr - 1));
-}

lib/aligned_malloc.h

@@ -1,13 +0,0 @@
-/*
- * aligned_malloc.c - aligned memory allocation
- */
-
-#ifndef _LIB_ALIGNED_MALLOC_H
-#define _LIB_ALIGNED_MALLOC_H
-
-#include "common_defs.h"
-
-extern void *aligned_malloc(size_t alignment, size_t size);
-extern void aligned_free(void *ptr);
-
-#endif /* _LIB_ALIGNED_MALLOC_H */

lib/arm/adler32_impl.h

@@ -0,0 +1,125 @@
+/*
+ * arm/adler32_impl.h - ARM implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_ADLER32_IMPL_H
+#define LIB_ARM_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/* NEON implementation */
+#undef DISPATCH_NEON
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__ARM_NEON) || (ARM_CPU_FEATURES_ENABLED &&	\
+				 COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_neon
+#  define FUNCNAME_CHUNK	adler32_neon_chunk
+#  define IMPL_ALIGNMENT	16
+#  define IMPL_SEGMENT_SIZE	32
+/* Prevent unsigned overflow of the 16-bit precision byte counters */
+#  define IMPL_MAX_CHUNK_SIZE	(32 * (0xFFFF / 0xFF))
+#  ifdef __ARM_NEON
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_neon
+#  else
+#    ifdef __arm__
+#      define ATTRIBUTES	__attribute__((target("fpu=neon")))
+#    else
+#      define ATTRIBUTES	__attribute__((target("+simd")))
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_NEON	1
+#  endif
+#  include <arm_neon.h>
+static forceinline ATTRIBUTES void
+adler32_neon_chunk(const uint8x16_t *p, const uint8x16_t * const end,
+		   u32 *s1, u32 *s2)
+{
+	uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 };
+	uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_a = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_b = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_c = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	uint16x8_t v_byte_sums_d = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+	do {
+		const uint8x16_t bytes1 = *p++;
+		const uint8x16_t bytes2 = *p++;
+		uint16x8_t tmp;
+
+		v_s2 += v_s1;
+
+		/* Vector Pairwise Add Long (u8 => u16) */
+		tmp = vpaddlq_u8(bytes1);
+
+		/* Vector Pairwise Add and Accumulate Long (u8 => u16) */
+		tmp = vpadalq_u8(tmp, bytes2);
+
+		/* Vector Pairwise Add and Accumulate Long (u16 => u32) */
+		v_s1 = vpadalq_u16(v_s1, tmp);
+
+		/* Vector Add Wide (u8 => u16) */
+		v_byte_sums_a = vaddw_u8(v_byte_sums_a, vget_low_u8(bytes1));
+		v_byte_sums_b = vaddw_u8(v_byte_sums_b, vget_high_u8(bytes1));
+		v_byte_sums_c = vaddw_u8(v_byte_sums_c, vget_low_u8(bytes2));
+		v_byte_sums_d = vaddw_u8(v_byte_sums_d, vget_high_u8(bytes2));
+
+	} while (p != end);
+
+	/* Vector Shift Left (u32) */
+	v_s2 = vqshlq_n_u32(v_s2, 5);
+
+	/* Vector Multiply Accumulate Long (u16 => u32) */
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),  (uint16x4_t) { 32, 31, 30, 29 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_a), (uint16x4_t) { 28, 27, 26, 25 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),  (uint16x4_t) { 24, 23, 22, 21 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_b), (uint16x4_t) { 20, 19, 18, 17 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),  (uint16x4_t) { 16, 15, 14, 13 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_c), (uint16x4_t) { 12, 11, 10,  9 });
+	v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_byte_sums_d), (uint16x4_t) {  8,  7,  6,  5 });
+	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_d), (uint16x4_t) {  4,  3,  2,  1 });
+
+	*s1 += v_s1[0] + v_s1[1] + v_s1[2] + v_s1[3];
+	*s2 += v_s2[0] + v_s2[1] + v_s2[2] + v_s2[3];
+}
+#  include "../adler32_vec_template.h"
+#endif /* NEON implementation */
+
+#ifdef DISPATCH
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_NEON
+	if (features & ARM_CPU_FEATURE_NEON)
+		return adler32_neon;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_ARM_ADLER32_IMPL_H */

lib/arm/cpu_features.c

@@ -0,0 +1,133 @@
+/*
+ * arm/cpu_features.c - feature detection for ARM processors
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * ARM processors don't have a standard way for unprivileged programs to detect
+ * processor features.  But, on Linux we can read the AT_HWCAP and AT_HWCAP2
+ * values from /proc/self/auxv.
+ *
+ * Ideally we'd use the C library function getauxval(), but it's not guaranteed
+ * to be available: it was only added to glibc in 2.16, and in Android it was
+ * added to API level 18 for ARM and level 21 for AArch64.
+ */
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#if ARM_CPU_FEATURES_ENABLED
+
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+
+#define AT_HWCAP	16
+#define AT_HWCAP2	26
+
+volatile u32 _cpu_features = 0;
+
+static void scan_auxv(unsigned long *hwcap, unsigned long *hwcap2)
+{
+	int fd;
+	unsigned long auxbuf[32];
+	int filled = 0;
+	int i;
+
+	fd = open("/proc/self/auxv", O_RDONLY);
+	if (fd < 0)
+		return;
+
+	for (;;) {
+		do {
+			int ret = read(fd, &((char *)auxbuf)[filled],
+				       sizeof(auxbuf) - filled);
+			if (ret <= 0) {
+				if (ret < 0 && errno == EINTR)
+					continue;
+				goto out;
+			}
+			filled += ret;
+		} while (filled < 2 * sizeof(long));
+
+		i = 0;
+		do {
+			unsigned long type = auxbuf[i];
+			unsigned long value = auxbuf[i + 1];
+
+			if (type == AT_HWCAP)
+				*hwcap = value;
+			else if (type == AT_HWCAP2)
+				*hwcap2 = value;
+			i += 2;
+			filled -= 2 * sizeof(long);
+		} while (filled >= 2 * sizeof(long));
+
+		memmove(auxbuf, &auxbuf[i], filled);
+	}
+out:
+	close(fd);
+}
+
+static const struct cpu_feature arm_cpu_feature_table[] = {
+	{ARM_CPU_FEATURE_NEON,		"neon"},
+	{ARM_CPU_FEATURE_PMULL,		"pmull"},
+	{ARM_CPU_FEATURE_CRC32,		"crc32"},
+};
+
+void setup_cpu_features(void)
+{
+	u32 features = 0;
+	unsigned long hwcap = 0;
+	unsigned long hwcap2 = 0;
+
+	scan_auxv(&hwcap, &hwcap2);
+
+#ifdef __arm__
+	STATIC_ASSERT(sizeof(long) == 4);
+	if (hwcap & (1 << 12))	/* HWCAP_NEON */
+		features |= ARM_CPU_FEATURE_NEON;
+	if (hwcap2 & (1 << 1))	/* HWCAP2_PMULL */
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap2 & (1 << 4))	/* HWCAP2_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
+#else
+	STATIC_ASSERT(sizeof(long) == 8);
+	if (hwcap & (1 << 1))	/* HWCAP_ASIMD */
+		features |= ARM_CPU_FEATURE_NEON;
+	if (hwcap & (1 << 4))	/* HWCAP_PMULL */
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap & (1 << 7))	/* HWCAP_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
+#endif
+
+	disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
+					 ARRAY_LEN(arm_cpu_feature_table));
+
+	_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
+}
+
+#endif /* ARM_CPU_FEATURES_ENABLED */

lib/arm/cpu_features.h

@@ -0,0 +1,40 @@
+/*
+ * arm/cpu_features.h - feature detection for ARM processors
+ */
+
+#ifndef LIB_ARM_CPU_FEATURES_H
+#define LIB_ARM_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if (defined(__arm__) || defined(__aarch64__)) && \
+	defined(__linux__) && \
+	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE && \
+	!defined(FREESTANDING)
+#  define ARM_CPU_FEATURES_ENABLED 1
+#else
+#  define ARM_CPU_FEATURES_ENABLED 0
+#endif
+
+#if ARM_CPU_FEATURES_ENABLED
+
+#define ARM_CPU_FEATURE_NEON		0x00000001
+#define ARM_CPU_FEATURE_PMULL		0x00000002
+#define ARM_CPU_FEATURE_CRC32		0x00000004
+
+#define ARM_CPU_FEATURES_KNOWN		0x80000000
+
+extern volatile u32 _cpu_features;
+
+void setup_cpu_features(void);
+
+static inline u32 get_cpu_features(void)
+{
+	if (_cpu_features == 0)
+		setup_cpu_features();
+	return _cpu_features;
+}
+
+#endif /* ARM_CPU_FEATURES_ENABLED */
+
+#endif /* LIB_ARM_CPU_FEATURES_H */

lib/arm/crc32_impl.h

@@ -0,0 +1,247 @@
+/*
+ * arm/crc32_impl.h
+ *
+ * Copyright 2017 Jun He <jun.he@linaro.org>
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_CRC32_IMPL_H
+#define LIB_ARM_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/* Implementation using ARM CRC32 instructions */
+#undef DISPATCH_ARM
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRC32) || \
+     (ARM_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS))
+#  ifdef __ARM_FEATURE_CRC32
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_arm
+#  else
+#    ifdef __arm__
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("armv8-a,crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("arch=armv8-a+crc")))
+#      endif
+#    else
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("crc")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("+crc")))
+#      endif
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_ARM	1
+#  endif
+
+/*
+ * gcc's (as of 10.1) version of arm_acle.h for arm32, and clang's (as of
+ * 10.0.1) version of arm_acle.h for both arm32 and arm64, have a bug where they
+ * only define the CRC32 functions like __crc32b() when __ARM_FEATURE_CRC32 is
+ * defined.  That prevents them from being used via __attribute__((target)) when
+ * the main target doesn't have CRC32 support enabled.  The actual built-ins
+ * like __builtin_arm_crc32b() are available and work, however; it's just the
+ * wrappers in arm_acle.h like __crc32b() that erroneously don't get defined.
+ * Work around this by manually defining __ARM_FEATURE_CRC32.
+ */
+#ifndef __ARM_FEATURE_CRC32
+#  define __ARM_FEATURE_CRC32	1
+#endif
+#include <arm_acle.h>
+
+static u32 ATTRIBUTES
+crc32_arm(u32 remainder, const u8 *p, size_t size)
+{
+	while (size != 0 && (uintptr_t)p & 7) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	while (size >= 32) {
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 0)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 1)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 2)));
+		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 3)));
+		p += 32;
+		size -= 32;
+	}
+
+	while (size >= 8) {
+		remainder = __crc32d(remainder, le64_bswap(*(u64 *)p));
+		p += 8;
+		size -= 8;
+	}
+
+	while (size != 0) {
+		remainder = __crc32b(remainder, *p++);
+		size--;
+	}
+
+	return remainder;
+}
+#undef ATTRIBUTES
+#endif /* Implementation using ARM CRC32 instructions */
+
+/*
+ * CRC-32 folding with ARM Crypto extension-PMULL
+ *
+ * This works the same way as the x86 PCLMUL version.
+ * See x86/crc32_pclmul_template.h for an explanation.
+ */
+#undef DISPATCH_PMULL
+#if !defined(DEFAULT_IMPL) && \
+    (defined(__ARM_FEATURE_CRYPTO) ||	\
+     (ARM_CPU_FEATURES_ENABLED &&	\
+      COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS)) && \
+      /* not yet tested on big endian, probably needs changes to work there */ \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#  define FUNCNAME		crc32_pmull
+#  define FUNCNAME_ALIGNED	crc32_pmull_aligned
+#  ifdef __ARM_FEATURE_CRYPTO
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_pmull
+#  else
+#    ifdef __arm__
+#      define ATTRIBUTES	__attribute__((target("fpu=crypto-neon-fp-armv8")))
+#    else
+#      ifdef __clang__
+#        define ATTRIBUTES	__attribute__((target("crypto")))
+#      else
+#        define ATTRIBUTES	__attribute__((target("+crypto")))
+#      endif
+#    endif
+#    define DISPATCH		1
+#    define DISPATCH_PMULL	1
+#  endif
+
+#include <arm_neon.h>
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_00(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
+				     (poly64_t)vget_low_u8(b));
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_10(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
+				     (poly64_t)vget_high_u8(b));
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+clmul_11(uint8x16_t a, uint8x16_t b)
+{
+	return (uint8x16_t)vmull_high_p64((poly64x2_t)a, (poly64x2_t)b);
+}
+
+static forceinline ATTRIBUTES uint8x16_t
+fold_128b(uint8x16_t dst, uint8x16_t src, uint8x16_t multipliers)
+{
+	return dst ^ clmul_00(src, multipliers) ^ clmul_11(src, multipliers);
+}
+
+static forceinline ATTRIBUTES u32
+crc32_pmull_aligned(u32 remainder, const uint8x16_t *p, size_t nr_segs)
+{
+	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
+	const uint8x16_t multipliers_4 =
+		(uint8x16_t)(uint64x2_t){ 0x8F352D95, 0x1D9513D7 };
+	const uint8x16_t multipliers_1 =
+		(uint8x16_t)(uint64x2_t){ 0xAE689191, 0xCCAA009E };
+	const uint8x16_t final_multiplier =
+		(uint8x16_t)(uint64x2_t){ 0xB8BC6765 };
+	const uint8x16_t mask32 = (uint8x16_t)(uint32x4_t){ 0xFFFFFFFF };
+	const uint8x16_t barrett_reduction_constants =
+			(uint8x16_t)(uint64x2_t){ 0x00000001F7011641,
+						  0x00000001DB710641 };
+	const uint8x16_t zeroes = (uint8x16_t){ 0 };
+
+	const uint8x16_t * const end = p + nr_segs;
+	const uint8x16_t * const end512 = p + (nr_segs & ~3);
+	uint8x16_t x0, x1, x2, x3;
+
+	x0 = *p++ ^ (uint8x16_t)(uint32x4_t){ remainder };
+	if (nr_segs >= 4) {
+		x1 = *p++;
+		x2 = *p++;
+		x3 = *p++;
+
+		/* Fold 512 bits at a time */
+		while (p != end512) {
+			x0 = fold_128b(*p++, x0, multipliers_4);
+			x1 = fold_128b(*p++, x1, multipliers_4);
+			x2 = fold_128b(*p++, x2, multipliers_4);
+			x3 = fold_128b(*p++, x3, multipliers_4);
+		}
+
+		/* Fold 512 bits => 128 bits */
+		x1 = fold_128b(x1, x0, multipliers_1);
+		x2 = fold_128b(x2, x1, multipliers_1);
+		x0 = fold_128b(x3, x2, multipliers_1);
+	}
+
+	/* Fold 128 bits at a time */
+	while (p != end)
+		x0 = fold_128b(*p++, x0, multipliers_1);
+
+	/* Fold 128 => 96 bits, implicitly appending 32 zeroes */
+	x0 = vextq_u8(x0, zeroes, 8) ^ clmul_10(x0, multipliers_1);
+
+	/* Fold 96 => 64 bits */
+	x0 = vextq_u8(x0, zeroes, 4) ^ clmul_00(x0 & mask32, final_multiplier);
+
+	/* Reduce 64 => 32 bits using Barrett reduction */
+	x1 = x0;
+	x0 = clmul_00(x0 & mask32, barrett_reduction_constants);
+	x0 = clmul_10(x0 & mask32, barrett_reduction_constants);
+	return vgetq_lane_u32((uint32x4_t)(x0 ^ x1), 1);
+}
+#define IMPL_ALIGNMENT		16
+#define IMPL_SEGMENT_SIZE	16
+#include "../crc32_vec_template.h"
+#endif /* PMULL implementation */
+
+#ifdef DISPATCH
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_ARM
+	if (features & ARM_CPU_FEATURE_CRC32)
+		return crc32_arm;
+#endif
+#ifdef DISPATCH_PMULL
+	if (features & ARM_CPU_FEATURE_PMULL)
+		return crc32_pmull;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_ARM_CRC32_IMPL_H */

lib/arm/matchfinder_impl.h

@@ -0,0 +1,86 @@
+/*
+ * arm/matchfinder_impl.h - ARM implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_MATCHFINDER_IMPL_H
+#define LIB_ARM_MATCHFINDER_IMPL_H
+
+#ifdef __ARM_NEON
+#  include <arm_neon.h>
+static forceinline void
+matchfinder_init_neon(mf_pos_t *data, size_t size)
+{
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+	};
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_neon
+
+static forceinline void
+matchfinder_rebase_neon(mf_pos_t *data, size_t size)
+{
+	int16x8_t *p = (int16x8_t *)data;
+	int16x8_t v = (int16x8_t) {
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+	};
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = vqaddq_s16(p[0], v);
+		p[1] = vqaddq_s16(p[1], v);
+		p[2] = vqaddq_s16(p[2], v);
+		p[3] = vqaddq_s16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_neon
+
+#endif /* __ARM_NEON */
+
+#endif /* LIB_ARM_MATCHFINDER_IMPL_H */

lib/bt_matchfinder.h

@@ -1,15 +1,30 @@
 /*
  * bt_matchfinder.h - Lempel-Ziv matchfinding with a hash table of binary trees
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
  * ----------------------------------------------------------------------------
  *
@@ -49,6 +64,8 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_BT_MATCHFINDER_H
+#define LIB_BT_MATCHFINDER_H
 
 #include "matchfinder_common.h"
 
@@ -56,9 +73,9 @@
 #define BT_MATCHFINDER_HASH3_WAYS  2
 #define BT_MATCHFINDER_HASH4_ORDER 16
 
-#define BT_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
-	 (1UL << BT_MATCHFINDER_HASH4_ORDER))
+#define BT_MATCHFINDER_TOTAL_HASH_SIZE		\
+	(((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
+	  (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 /* Representation of a match found by the bt_matchfinder  */
 struct lz_match {
@@ -86,7 +103,7 @@ struct bt_matchfinder {
 
 }
 #ifdef _aligned_attribute
-_aligned_attribute(MATCHFINDER_ALIGNMENT)
+_aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
 #endif
 ;
 
@@ -94,14 +111,18 @@ _aligned_attribute(MATCHFINDER_ALIGNMENT)
 static forceinline void
 bt_matchfinder_init(struct bt_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 bt_matchfinder_slide_window(struct bt_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct bt_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 static forceinline mf_pos_t *
@@ -138,8 +159,7 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 	const u8 *in_next = in_base + cur_pos;
 	u32 depth_remaining = max_search_depth;
 	const s32 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
-	u32 next_seq4;
-	u32 next_seq3;
+	u32 next_hashseq;
 	u32 hash3;
 	u32 hash4;
 	s32 cur_node;
@@ -155,14 +175,13 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 	STATIC_ASSERT(BT_MATCHFINDER_HASH3_WAYS >= 1 &&
 		      BT_MATCHFINDER_HASH3_WAYS <= 2);
 
-	next_seq4 = load_u32_unaligned(in_next + 1);
-	next_seq3 = loaded_u32_to_u24(next_seq4);
+	next_hashseq = get_unaligned_le32(in_next + 1);
 
 	hash3 = next_hashes[0];
 	hash4 = next_hashes[1];
 
-	next_hashes[0] = lz_hash(next_seq3, BT_MATCHFINDER_HASH3_ORDER);
-	next_hashes[1] = lz_hash(next_seq4, BT_MATCHFINDER_HASH4_ORDER);
+	next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, BT_MATCHFINDER_HASH3_ORDER);
+	next_hashes[1] = lz_hash(next_hashseq, BT_MATCHFINDER_HASH4_ORDER);
 	prefetchw(&mf->hash3_tab[next_hashes[0]]);
 	prefetchw(&mf->hash4_tab[next_hashes[1]]);
 
@@ -211,8 +230,7 @@ bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
 		matchptr = &in_base[cur_node];
 
 		if (matchptr[len] == in_next[len]) {
-			len = lz_extend(in_next, matchptr, len + 1,
-					(record_matches ? max_len : nice_len));
+			len = lz_extend(in_next, matchptr, len + 1, max_len);
 			if (!record_matches || len > best_len) {
 				if (record_matches) {
 					best_len = len;
@@ -325,7 +343,6 @@ static forceinline void
 bt_matchfinder_skip_position(struct bt_matchfinder *mf,
 			     const u8 *in_base,
 			     ptrdiff_t cur_pos,
-			     u32 max_len,
 			     u32 nice_len,
 			     u32 max_search_depth,
 			     u32 next_hashes[2])
@@ -334,7 +351,7 @@ bt_matchfinder_skip_position(struct bt_matchfinder *mf,
 	bt_matchfinder_advance_one_byte(mf,
 					in_base,
 					cur_pos,
-					max_len,
+					nice_len,
 					nice_len,
 					max_search_depth,
 					next_hashes,
@@ -342,3 +359,5 @@ bt_matchfinder_skip_position(struct bt_matchfinder *mf,
 					NULL,
 					false);
 }
+
+#endif /* LIB_BT_MATCHFINDER_H */

lib/cpu_features_common.h

@@ -0,0 +1,88 @@
+/*
+ * cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
+ *
+ * Copyright 2020 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_CPU_FEATURES_COMMON_H
+#define LIB_CPU_FEATURES_COMMON_H
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+#  define _GNU_SOURCE 1 /* for strdup() and strtok_r() */
+#  include <stdio.h>
+#  include <stdlib.h>
+#  include <string.h>
+#endif
+
+#include "lib_common.h"
+
+struct cpu_feature {
+	u32 bit;
+	const char *name;
+};
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+	char *env_value, *strbuf, *p, *saveptr = NULL;
+	size_t i;
+
+	env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
+	if (!env_value)
+		return;
+	strbuf = strdup(env_value);
+	if (!strbuf)
+		abort();
+	p = strtok_r(strbuf, ",", &saveptr);
+	while (p) {
+		for (i = 0; i < feature_table_length; i++) {
+			if (strcmp(p, feature_table[i].name) == 0) {
+				*features &= ~feature_table[i].bit;
+				break;
+			}
+		}
+		if (i == feature_table_length) {
+			fprintf(stderr,
+				"unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
+				p);
+			abort();
+		}
+		p = strtok_r(NULL, ",", &saveptr);
+	}
+	free(strbuf);
+}
+#else /* TEST_SUPPORT__DO_NOT_USE */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+				 const struct cpu_feature *feature_table,
+				 size_t feature_table_length)
+{
+}
+#endif /* !TEST_SUPPORT__DO_NOT_USE */
+
+#endif /* LIB_CPU_FEATURES_COMMON_H */

lib/crc32.c

@@ -1,22 +1,35 @@
 /*
  * crc32.c - CRC-32 checksum algorithm for the gzip format
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /*
  * High-level description of CRC
  * =============================
  *
- * Consider a bit sequence 'bits[1...len]'.  Interpet 'bits' as the "message"
+ * Consider a bit sequence 'bits[1...len]'.  Interpret 'bits' as the "message"
  * polynomial M(x) with coefficients in GF(2) (the field of integers modulo 2),
  * where the coefficient of 'x^i' is 'bits[len - i]'.  Then, compute:
  *
@@ -24,7 +37,7 @@
  *
  * where G(x) is a selected "generator" polynomial of degree 'n'.  The remainder
  * R(x) is a polynomial of max degree 'n - 1'.  The CRC of 'bits' is R(x)
- * interpeted as a bitstring of length 'n'.
+ * interpreted as a bitstring of length 'n'.
  *
  * CRC used in gzip
  * ================
@@ -57,16 +70,16 @@
  * subtraction can be implemented as bitwise exclusive OR (since we are working
  * in GF(2)).  Here is an unoptimized implementation:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t nbytes)
+ *	static u32 crc32_gzip(const u8 *buffer, size_t size)
  *	{
  *		u32 remainder = 0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < nbytes * 8 + 32; i++) {
+ *		for (size_t i = 0; i < size * 8 + 32; i++) {
  *			int bit;
  *			u32 multiple;
  *
- *			if (i < nbytes * 8)
+ *			if (i < size * 8)
  *				bit = (buffer[i / 8] >> (i % 8)) & 1;
  *			else
  *				bit = 0; // one of the 32 appended 0 bits
@@ -101,12 +114,12 @@
  * of 'multiple' until 32 bits later, we need not actually add each message bit
  * until that point:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t nbytes)
+ *	static u32 crc32_gzip(const u8 *buffer, size_t size)
  *	{
  *		u32 remainder = ~0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < nbytes * 8; i++) {
+ *		for (size_t i = 0; i < size * 8; i++) {
  *			int bit;
  *			u32 multiple;
  *
@@ -151,39 +164,67 @@
  *
  * In crc32_slice8(), this method is extended to 8 bytes at a time.  The
  * intermediate remainder (which we never actually store explicitly) is 96 bits.
+ *
+ * On CPUs that support fast carryless multiplication, CRCs can be computed even
+ * more quickly via "folding".  See e.g. the x86 PCLMUL implementation.
  */
 
-#define CRC32_SLICE8
+#include "lib_common.h"
+#include "libdeflate.h"
 
-#include "crc32.h"
+typedef u32 (*crc32_func_t)(u32, const u8 *, size_t);
+
+/* Include architecture-specific implementations if available */
+#undef CRC32_SLICE1
+#undef CRC32_SLICE4
+#undef CRC32_SLICE8
+#undef DEFAULT_IMPL
+#undef DISPATCH
+#if defined(__arm__) || defined(__aarch64__)
+#  include "arm/crc32_impl.h"
+#elif defined(__i386__) || defined(__x86_64__)
+#  include "x86/crc32_impl.h"
+#endif
+
+/*
+ * Define a generic implementation (crc32_slice8()) if needed.  crc32_slice1()
+ * may also be needed as a fallback for architecture-specific implementations.
+ */
+
+#ifndef DEFAULT_IMPL
+#  define CRC32_SLICE8	1
+#  define DEFAULT_IMPL	crc32_slice8
+#endif
+
+#if defined(CRC32_SLICE1) || defined(CRC32_SLICE4) || defined(CRC32_SLICE8)
 #include "crc32_table.h"
-
 static forceinline u32
 crc32_update_byte(u32 remainder, u8 next_byte)
 {
 	return (remainder >> 8) ^ crc32_table[(u8)remainder ^ next_byte];
 }
+#endif
 
 #ifdef CRC32_SLICE1
 static u32
-crc32_slice1(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice1(u32 remainder, const u8 *buffer, size_t size)
 {
 	size_t i;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x100);
 
-	for (i = 0; i < nbytes; i++)
+	for (i = 0; i < size; i++)
 		remainder = crc32_update_byte(remainder, buffer[i]);
 	return remainder;
 }
-#endif
+#endif /* CRC32_SLICE1 */
 
 #ifdef CRC32_SLICE4
 static u32
-crc32_slice4(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice4(u32 remainder, const u8 *buffer, size_t size)
 {
 	const u8 *p = buffer;
-	const u8 *end = buffer + nbytes;
+	const u8 *end = buffer + size;
 	const u8 *end32;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x400);
@@ -206,14 +247,14 @@ crc32_slice4(u32 remainder, const u8 *buffer, size_t nbytes)
 
 	return remainder;
 }
-#endif
+#endif /* CRC32_SLICE4 */
 
 #ifdef CRC32_SLICE8
 static u32
-crc32_slice8(u32 remainder, const u8 *buffer, size_t nbytes)
+crc32_slice8(u32 remainder, const u8 *buffer, size_t size)
 {
 	const u8 *p = buffer;
-	const u8 *end = buffer + nbytes;
+	const u8 *end = buffer + size;
 	const u8 *end64;
 
 	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x800);
@@ -241,20 +282,32 @@ crc32_slice8(u32 remainder, const u8 *buffer, size_t nbytes)
 
 	return remainder;
 }
+#endif /* CRC32_SLICE8 */
+
+#ifdef DISPATCH
+static u32 dispatch(u32, const u8 *, size_t);
+
+static volatile crc32_func_t crc32_impl = dispatch;
+
+/* Choose the fastest implementation at runtime */
+static u32 dispatch(u32 remainder, const u8 *buffer, size_t size)
+{
+	crc32_func_t f = arch_select_crc32_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
+	crc32_impl = f;
+	return crc32_impl(remainder, buffer, size);
+}
+#else
+#  define crc32_impl DEFAULT_IMPL /* only one implementation, use it */
 #endif
 
-u32
-crc32_gzip(const void *buffer, size_t nbytes)
+LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
+libdeflate_crc32(u32 remainder, const void *buffer, size_t size)
 {
-	u32 remainder = ~0;
-#if defined(CRC32_SLICE1)
-	remainder = crc32_slice1(remainder, buffer, nbytes);
-#elif defined(CRC32_SLICE4)
-	remainder = crc32_slice4(remainder, buffer, nbytes);
-#elif defined(CRC32_SLICE8)
-	remainder = crc32_slice8(remainder, buffer, nbytes);
-#else
-#  error "don't know which CRC-32 implementation to use!"
-#endif
-	return ~remainder;
+	if (buffer == NULL) /* return initial value */
+		return 0;
+	return ~crc32_impl(~remainder, buffer, size);
 }

lib/crc32.h

@@ -1,12 +0,0 @@
-/*
- * crc32.h - CRC-32 checksum algorithm for the gzip format
- */
-
-#ifndef _LIB_CRC32_H
-#define _LIB_CRC32_H
-
-#include "common_defs.h"
-
-extern u32 crc32_gzip(const void *buffer, size_t size);
-
-#endif /* _LIB_CRC32_H */

lib/crc32_vec_template.h

@@ -0,0 +1,61 @@
+/*
+ * crc32_vec_template.h - template for vectorized CRC-32 implementations
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#define CRC32_SLICE1	1
+static u32 crc32_slice1(u32, const u8 *, size_t);
+
+/*
+ * Template for vectorized CRC-32 implementations.
+ *
+ * Note: on unaligned ends of the buffer, we fall back to crc32_slice1() instead
+ * of crc32_slice8() because only a few bytes need to be processed, so a smaller
+ * table is preferable.
+ */
+static u32 ATTRIBUTES
+FUNCNAME(u32 remainder, const u8 *p, size_t size)
+{
+	if ((uintptr_t)p % IMPL_ALIGNMENT) {
+		size_t n = MIN(size, -(uintptr_t)p % IMPL_ALIGNMENT);
+
+		remainder = crc32_slice1(remainder, p, n);
+		p += n;
+		size -= n;
+	}
+	if (size >= IMPL_SEGMENT_SIZE) {
+		remainder = FUNCNAME_ALIGNED(remainder, (const void *)p,
+					     size / IMPL_SEGMENT_SIZE);
+		p += size - (size % IMPL_SEGMENT_SIZE);
+		size %= IMPL_SEGMENT_SIZE;
+	}
+	return crc32_slice1(remainder, p, size);
+}
+
+#undef FUNCNAME
+#undef FUNCNAME_ALIGNED
+#undef ATTRIBUTES
+#undef IMPL_ALIGNMENT
+#undef IMPL_SEGMENT_SIZE

lib/decompress_template.h

@@ -1,15 +1,28 @@
 /*
- * decompress_impl.h
+ * decompress_template.h
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /*
@@ -18,11 +31,11 @@
  * target instruction sets.
  */
 
-static enum decompress_result ATTRIBUTES
-FUNCNAME(struct deflate_decompressor * restrict d,
+static enum libdeflate_result ATTRIBUTES
+FUNCNAME(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
 	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_out_nbytes_ret)
+	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
 {
 	u8 *out_next = out;
 	u8 * const out_end = out_next + out_nbytes_avail;
@@ -76,19 +89,13 @@ next_block:
 		STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == ((1 << 4) - 1) + 4);
 		num_explicit_precode_lens = POP_BITS(4) + 4;
 
+		d->static_codes_loaded = false;
+
 		/* Read the precode codeword lengths.  */
 		STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
-		if (CAN_ENSURE(DEFLATE_NUM_PRECODE_SYMS * 3)) {
-
-			ENSURE_BITS(DEFLATE_NUM_PRECODE_SYMS * 3);
-
-			for (i = 0; i < num_explicit_precode_lens; i++)
-				d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-		} else {
-			for (i = 0; i < num_explicit_precode_lens; i++) {
-				ENSURE_BITS(3);
-				d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
-			}
+		for (i = 0; i < num_explicit_precode_lens; i++) {
+			ENSURE_BITS(3);
+			d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
 		}
 
 		for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
@@ -192,7 +199,7 @@ next_block:
 
 		SAFETY_CHECK(len == (u16)~nlen);
 		if (unlikely(len > out_end - out_next))
-			return DECOMPRESS_INSUFFICIENT_SPACE;
+			return LIBDEFLATE_INSUFFICIENT_SPACE;
 		SAFETY_CHECK(len <= in_end - in_next);
 
 		memcpy(out_next, in_next, len);
@@ -204,9 +211,20 @@ next_block:
 	} else {
 		SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
 
-		/* Static Huffman block: set the static Huffman codeword
-		 * lengths.  Then the remainder is the same as decompressing a
-		 * dynamic Huffman block.  */
+		/*
+		 * Static Huffman block: build the decode tables for the static
+		 * codes.  Skip doing so if the tables are already set up from
+		 * an earlier static block; this speeds up decompression of
+		 * degenerate input of many empty or very short static blocks.
+		 *
+		 * Afterwards, the remainder is the same as decompressing a
+		 * dynamic Huffman block.
+		 */
+
+		if (d->static_codes_loaded)
+			goto have_decode_tables;
+
+		d->static_codes_loaded = true;
 
 		STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
 		STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
@@ -225,19 +243,21 @@ next_block:
 
 		num_litlen_syms = 288;
 		num_offset_syms = 32;
-
 	}
 
 	/* Decompressing a Huffman block (either dynamic or static)  */
 
 	SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
 	SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
+have_decode_tables:
 
 	/* The main DEFLATE decode loop  */
 	for (;;) {
 		u32 entry;
 		u32 length;
 		u32 offset;
+		const u8 *src;
+		u8 *dst;
 
 		/* Decode a litlen symbol.  */
 		ENSURE_BITS(DEFLATE_MAX_LITLEN_CODEWORD_LEN);
@@ -253,7 +273,7 @@ next_block:
 		if (entry & HUFFDEC_LITERAL) {
 			/* Literal  */
 			if (unlikely(out_next == out_end))
-				return DECOMPRESS_INSUFFICIENT_SPACE;
+				return LIBDEFLATE_INSUFFICIENT_SPACE;
 			*out_next++ = (u8)(entry >> HUFFDEC_RESULT_SHIFT);
 			continue;
 		}
@@ -276,7 +296,7 @@ next_block:
 		STATIC_ASSERT(HUFFDEC_END_OF_BLOCK_LENGTH == 0);
 		if (unlikely((size_t)length - 1 >= out_end - out_next)) {
 			if (unlikely(length != HUFFDEC_END_OF_BLOCK_LENGTH))
-				return DECOMPRESS_INSUFFICIENT_SPACE;
+				return LIBDEFLATE_INSUFFICIENT_SPACE;
 			goto block_done;
 		}
 
@@ -310,65 +330,66 @@ next_block:
 		 * output buffer.  */
 		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
 
-		/* Copy the match: 'length' bytes at 'out_next - offset' to
-		 * 'out_next'.  */
+		/*
+		 * Copy the match: 'length' bytes at 'out_next - offset' to
+		 * 'out_next', possibly overlapping.  If the match doesn't end
+		 * too close to the end of the buffer and offset >= WORDBYTES ||
+		 * offset == 1, take a fast path which copies a word at a time
+		 * -- potentially more than the length of the match, but that's
+		 * fine as long as we check for enough extra space.
+		 *
+		 * The remaining cases are not performance-critical so are
+		 * handled by a simple byte-by-byte copy.
+		 */
+
+		src = out_next - offset;
+		dst = out_next;
+		out_next += length;
 
 		if (UNALIGNED_ACCESS_IS_FAST &&
-		    length <= (3 * WORDBYTES) &&
-		    offset >= WORDBYTES &&
-		    length + (3 * WORDBYTES) <= out_end - out_next)
-		{
-			/* Fast case: short length, no overlaps if we copy one
-			 * word at a time, and we aren't getting too close to
-			 * the end of the output array.  */
-			copy_word_unaligned(out_next - offset + (0 * WORDBYTES),
-					    out_next + (0 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (1 * WORDBYTES),
-					    out_next + (1 * WORDBYTES));
-			copy_word_unaligned(out_next - offset + (2 * WORDBYTES),
-					    out_next + (2 * WORDBYTES));
-		} else {
-			const u8 *src = out_next - offset;
-			u8 *dst = out_next;
-			u8 *end = out_next + length;
-
-			if (UNALIGNED_ACCESS_IS_FAST &&
-			    likely(out_end - end >= WORDBYTES - 1)) {
-				if (offset >= WORDBYTES) {
+		    /* max overrun is writing 3 words for a min length match */
+		    likely(out_end - out_next >=
+			   3 * WORDBYTES - DEFLATE_MIN_MATCH_LEN)) {
+			if (offset >= WORDBYTES) { /* words don't overlap? */
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				copy_word_unaligned(src, dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				do {
 					copy_word_unaligned(src, dst);
 					src += WORDBYTES;
 					dst += WORDBYTES;
-					if (dst < end) {
-						do {
-							copy_word_unaligned(src, dst);
-							src += WORDBYTES;
-							dst += WORDBYTES;
-						} while (dst < end);
-					}
-				} else if (offset == 1) {
-					machine_word_t v = repeat_byte(*(dst - 1));
-					do {
-						store_word_unaligned(v, dst);
-						src += WORDBYTES;
-						dst += WORDBYTES;
-					} while (dst < end);
-				} else {
-					*dst++ = *src++;
-					*dst++ = *src++;
-					do {
-						*dst++ = *src++;
-					} while (dst < end);
-				}
+				} while (dst < out_next);
+			} else if (offset == 1) {
+				/* RLE encoding of previous byte, common if the
+				 * data contains many repeated bytes */
+				machine_word_t v = repeat_byte(*src);
+
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				do {
+					store_word_unaligned(v, dst);
+					dst += WORDBYTES;
+				} while (dst < out_next);
 			} else {
 				*dst++ = *src++;
 				*dst++ = *src++;
 				do {
 					*dst++ = *src++;
-				} while (dst < end);
+				} while (dst < out_next);
 			}
+		} else {
+			STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
+			*dst++ = *src++;
+			*dst++ = *src++;
+			do {
+				*dst++ = *src++;
+			} while (dst < out_next);
 		}
-
-		out_next += length;
 	}
 
 block_done:
@@ -379,11 +400,22 @@ block_done:
 
 	/* That was the last block.  */
 
+	/* Discard any readahead bits and check for excessive overread */
+	ALIGN_INPUT();
+
+	/* Optionally return the actual number of bytes read */
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	/* Optionally return the actual number of bytes written */
 	if (actual_out_nbytes_ret) {
 		*actual_out_nbytes_ret = out_next - (u8 *)out;
 	} else {
 		if (out_next != out_end)
-			return DECOMPRESS_SHORT_OUTPUT;
+			return LIBDEFLATE_SHORT_OUTPUT;
 	}
-	return DECOMPRESS_SUCCESS;
+	return LIBDEFLATE_SUCCESS;
 }
+
+#undef FUNCNAME
+#undef ATTRIBUTES

lib/deflate_compress.c

@@ -1,21 +1,32 @@
 /*
  * deflate_compress.c - a compressor for DEFLATE
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <string.h>
-
-#include "aligned_malloc.h"
 #include "deflate_compress.h"
 #include "deflate_constants.h"
 #include "unaligned.h"
@@ -51,12 +62,24 @@
 #endif
 
 /*
- * The minimum and maximum block lengths, in bytes of source data, which the
- * parsing algorithms may choose.  Caveat: due to implementation details, the
- * actual maximum will be slightly higher than the number defined below.
+ * The compressor always chooses a block of at least MIN_BLOCK_LENGTH bytes,
+ * except if the last block has to be shorter.
  */
 #define MIN_BLOCK_LENGTH	10000
-#define MAX_BLOCK_LENGTH	300000
+
+/*
+ * The compressor attempts to end blocks after SOFT_MAX_BLOCK_LENGTH bytes, but
+ * the final length might be slightly longer due to matches extending beyond
+ * this limit.
+ */
+#define SOFT_MAX_BLOCK_LENGTH	300000
+
+/*
+ * The number of observed matches or literals that represents sufficient data to
+ * decide whether the current block should be terminated or not.
+ */
+#define NUM_OBSERVATIONS_PER_BLOCK_CHECK       512
+
 
 #if SUPPORT_NEAR_OPTIMAL_PARSING
 /* Constants specific to the near-optimal parsing algorithm */
@@ -77,7 +100,7 @@
  * However, fallback behavior (immediately terminating the block) on cache
  * overflow is still required.
  */
-#  define CACHE_LENGTH      (MAX_BLOCK_LENGTH * 5)
+#  define CACHE_LENGTH      (SOFT_MAX_BLOCK_LENGTH * 5)
 
 #endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
 
@@ -85,7 +108,7 @@
  * These are the compressor-side limits on the codeword lengths for each Huffman
  * code.  To make outputting bits slightly faster, some of these limits are
  * lower than the limits defined by the DEFLATE format.  This does not
- * significantly affect the compression ratio, at least for the block sizes we
+ * significantly affect the compression ratio, at least for the block lengths we
  * use.
  */
 #define MAX_LITLEN_CODEWORD_LEN		14
@@ -219,10 +242,19 @@ struct deflate_costs {
  */
 struct deflate_sequence {
 
-	/* The number of literals in the run.  This may be 0.  The literals are
-	 * not stored explicitly in this structure; instead, they are read
-	 * directly from the uncompressed data.  */
-	u16 litrunlen;
+	/* Bits 0..22: the number of literals in this run.  This may be 0 and
+	 * can be at most about SOFT_MAX_BLOCK_LENGTH.  The literals are not
+	 * stored explicitly in this structure; instead, they are read directly
+	 * from the uncompressed data.
+	 *
+	 * Bits 23..31: the length of the match which follows the literals, or 0
+	 * if this literal run was the last in the block, so there is no match
+	 * which follows it.  */
+	u32 litrunlen_and_length;
+
+	/* If 'length' doesn't indicate end-of-block, then this is the offset of
+	 * the match which follows the literals.  */
+	u16 offset;
 
 	/* If 'length' doesn't indicate end-of-block, then this is the offset
 	 * symbol of the match which follows the literals.  */
@@ -231,15 +263,6 @@ struct deflate_sequence {
 	/* If 'length' doesn't indicate end-of-block, then this is the length
 	 * slot of the match which follows the literals.  */
 	u8 length_slot;
-
-	/* The length of the match which follows the literals, or 0 if this this
-	 * sequence's literal run was the last literal run in the block, so
-	 * there is no match that follows it.  */
-	u16 length;
-
-	/* If 'length' doesn't indicate end-of-block, then this is the offset of
-	 * the match which follows the literals.  */
-	u16 offset;
 };
 
 #if SUPPORT_NEAR_OPTIMAL_PARSING
@@ -297,10 +320,10 @@ struct block_split_stats {
 };
 
 /* The main DEFLATE compressor structure  */
-struct deflate_compressor {
+struct libdeflate_compressor {
 
 	/* Pointer to the compress() implementation chosen at allocation time */
-	size_t (*impl)(struct deflate_compressor *,
+	size_t (*impl)(struct libdeflate_compressor *,
 		       const u8 *, size_t, u8 *, size_t);
 
 	/* Frequency counters for the current block  */
@@ -343,6 +366,9 @@ struct deflate_compressor {
 	/* The compression level with which this compressor was created.  */
 	unsigned compression_level;
 
+	/* Anything smaller than this we won't bother trying to compress.  */
+	unsigned min_size_to_compress;
+
 	/* Temporary space for Huffman code output  */
 	u32 precode_freqs[DEFLATE_NUM_PRECODE_SYMS];
 	u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];
@@ -365,7 +391,7 @@ struct deflate_compressor {
 			 * that can ever be chosen for a single block, plus one
 			 * for the special entry at the end.  */
 			struct deflate_sequence sequences[
-				DIV_ROUND_UP(MAX_BLOCK_LENGTH,
+				DIV_ROUND_UP(SOFT_MAX_BLOCK_LENGTH,
 					     DEFLATE_MIN_MATCH_LEN) + 1];
 		} g; /* (g)reedy */
 
@@ -411,11 +437,12 @@ struct deflate_compressor {
 			 * This array must be large enough to accommodate the
 			 * worst-case number of nodes, which occurs if we find a
 			 * match of length DEFLATE_MAX_MATCH_LEN at position
-			 * MAX_BLOCK_LENGTH - 1, producing a block of length
-			 * MAX_BLOCK_LENGTH - 1 + DEFLATE_MAX_MATCH_LEN.  Add
-			 * one for the end-of-block node.
+			 * SOFT_MAX_BLOCK_LENGTH - 1, producing a block of
+			 * length SOFT_MAX_BLOCK_LENGTH - 1 +
+			 * DEFLATE_MAX_MATCH_LEN.  Add one for the end-of-block
+			 * node.
 			 */
-			struct deflate_optimum_node optimum_nodes[MAX_BLOCK_LENGTH - 1 +
+			struct deflate_optimum_node optimum_nodes[SOFT_MAX_BLOCK_LENGTH - 1 +
 								  DEFLATE_MAX_MATCH_LEN + 1];
 
 			/* The current cost model being used.  */
@@ -463,18 +490,28 @@ struct deflate_output_bitstream {
 	u8 *end;
 };
 
-#define MIN_OUTPUT_SIZE	(UNALIGNED_ACCESS_IS_FAST ? sizeof(bitbuf_t) : 1)
+/*
+ * OUTPUT_END_PADDING is the size, in bytes, of the extra space that must be
+ * present following os->end, in order to not overrun the buffer when generating
+ * output.  When UNALIGNED_ACCESS_IS_FAST, we need at least sizeof(bitbuf_t)
+ * bytes for put_unaligned_leword().  Otherwise we need only 1 byte.  However,
+ * to make the compression algorithm produce the same result on all CPU
+ * architectures (which is sometimes desirable), we have to unconditionally use
+ * the maximum for any CPU, which is sizeof(bitbuf_t) == 8.
+ */
+#define OUTPUT_END_PADDING	8
 
 /* Initialize the output bitstream.  'size' is assumed to be at least
- * MIN_OUTPUT_SIZE.  */
+ * OUTPUT_END_PADDING.  */
 static void
-deflate_init_output(struct deflate_output_bitstream *os, void *buffer, size_t size)
+deflate_init_output(struct deflate_output_bitstream *os,
+		    void *buffer, size_t size)
 {
 	os->bitbuf = 0;
 	os->bitcount = 0;
 	os->begin = buffer;
 	os->next = os->begin;
-	os->end = os->begin + size - MIN_OUTPUT_SIZE;
+	os->end = os->begin + size - OUTPUT_END_PADDING;
 }
 
 /* Add some bits to the bitbuffer variable of the output bitstream.  The caller
@@ -521,7 +558,7 @@ deflate_align_bitstream(struct deflate_output_bitstream *os)
  * Flush any remaining bits to the output buffer if needed.  Return the total
  * number of bytes written to the output buffer, or 0 if an overflow occurred.
  */
-static u32
+static size_t
 deflate_flush_output(struct deflate_output_bitstream *os)
 {
 	if (os->next == os->end) /* overflow?  */
@@ -572,9 +609,15 @@ heapify_array(u32 A[], unsigned length)
 		heapify_subtree(A, length, subtree_idx);
 }
 
-/* Sort the array 'A', which contains 'length' unsigned 32-bit integers.  */
+/*
+ * Sort the array 'A', which contains 'length' unsigned 32-bit integers.
+ *
+ * Note: name this function heap_sort() instead of heapsort() to avoid colliding
+ * with heapsort() from stdlib.h on BSD-derived systems --- though this isn't
+ * necessary when compiling with -D_ANSI_SOURCE, which is the better solution.
+ */
 static void
-heapsort(u32 A[], unsigned length)
+heap_sort(u32 A[], unsigned length)
 {
 	A--; /* Use 1-based indices  */
 
@@ -682,8 +725,8 @@ sort_symbols(unsigned num_syms, const u32 freqs[restrict],
 	}
 
 	/* Sort the symbols counted in the last counter.  */
-	heapsort(symout + counters[num_counters - 2],
-		 counters[num_counters - 1] - counters[num_counters - 2]);
+	heap_sort(symout + counters[num_counters - 2],
+		  counters[num_counters - 1] - counters[num_counters - 2]);
 
 	return num_used_syms;
 }
@@ -1113,7 +1156,7 @@ make_canonical_huffman_code(unsigned num_syms, unsigned max_codeword_len,
  * This must be called when starting a new DEFLATE block.
  */
 static void
-deflate_reset_symbol_frequencies(struct deflate_compressor *c)
+deflate_reset_symbol_frequencies(struct libdeflate_compressor *c)
 {
 	memset(&c->freqs, 0, sizeof(c->freqs));
 }
@@ -1185,7 +1228,7 @@ deflate_make_huffman_codes(const struct deflate_freqs *freqs,
 
 /* Initialize c->static_codes.  */
 static void
-deflate_init_static_codes(struct deflate_compressor *c)
+deflate_init_static_codes(struct libdeflate_compressor *c)
 {
 	unsigned i;
 
@@ -1206,7 +1249,7 @@ deflate_init_static_codes(struct deflate_compressor *c)
 
 /* Return the offset slot for the specified match offset.  */
 static forceinline unsigned
-deflate_get_offset_slot(struct deflate_compressor *c, unsigned offset)
+deflate_get_offset_slot(struct libdeflate_compressor *c, unsigned offset)
 {
 #if USE_FULL_OFFSET_SLOT_FAST
 	return c->offset_slot_fast[offset];
@@ -1316,7 +1359,7 @@ deflate_compute_precode_items(const u8 lens[restrict],
 
 /* Precompute the information needed to output Huffman codes. */
 static void
-deflate_precompute_huffman_header(struct deflate_compressor *c)
+deflate_precompute_huffman_header(struct libdeflate_compressor *c)
 {
 	/* Compute how many litlen and offset symbols are needed. */
 
@@ -1379,7 +1422,7 @@ deflate_precompute_huffman_header(struct deflate_compressor *c)
 
 /* Output the Huffman codes. */
 static void
-deflate_write_huffman_header(struct deflate_compressor *c,
+deflate_write_huffman_header(struct libdeflate_compressor *c,
 			     struct deflate_output_bitstream *os)
 {
 	unsigned i;
@@ -1424,8 +1467,8 @@ deflate_write_sequences(struct deflate_output_bitstream * restrict os,
 	const struct deflate_sequence *seq = sequences;
 
 	for (;;) {
-		unsigned litrunlen = seq->litrunlen;
-		unsigned length;
+		u32 litrunlen = seq->litrunlen_and_length & 0x7FFFFF;
+		unsigned length = seq->litrunlen_and_length >> 23;
 		unsigned length_slot;
 		unsigned litlen_symbol;
 		unsigned offset_symbol;
@@ -1492,9 +1535,6 @@ deflate_write_sequences(struct deflate_output_bitstream * restrict os,
 		#endif
 		}
 
-
-		length = seq->length;
-
 		if (length == 0)
 			return;
 
@@ -1551,7 +1591,7 @@ deflate_write_sequences(struct deflate_output_bitstream * restrict os,
 static void
 deflate_write_item_list(struct deflate_output_bitstream *os,
 			const struct deflate_codes *codes,
-			struct deflate_compressor *c,
+			struct libdeflate_compressor *c,
 			u32 block_length)
 {
 	struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0];
@@ -1639,7 +1679,7 @@ deflate_write_uncompressed_block(struct deflate_output_bitstream *os,
 
 static void
 deflate_write_uncompressed_blocks(struct deflate_output_bitstream *os,
-				  const u8 *data, u32 data_length,
+				  const u8 *data, size_t data_length,
 				  bool is_final_block)
 {
 	do {
@@ -1657,7 +1697,7 @@ deflate_write_uncompressed_blocks(struct deflate_output_bitstream *os,
  * uncompressed), then output it.
  */
 static void
-deflate_flush_block(struct deflate_compressor * restrict c,
+deflate_flush_block(struct libdeflate_compressor * restrict c,
 		    struct deflate_output_bitstream * restrict os,
 		    const u8 * restrict block_begin, u32 block_length,
 		    bool is_final_block, bool use_item_list)
@@ -1766,19 +1806,18 @@ deflate_flush_block(struct deflate_compressor * restrict c,
 }
 
 static forceinline void
-deflate_choose_literal(struct deflate_compressor *c, unsigned literal,
-		       unsigned *litrunlen_p)
+deflate_choose_literal(struct libdeflate_compressor *c, unsigned literal,
+		       u32 *litrunlen_p)
 {
 	c->freqs.litlen[literal]++;
 	++*litrunlen_p;
 }
 
 static forceinline void
-deflate_choose_match(struct deflate_compressor *c,
+deflate_choose_match(struct libdeflate_compressor *c,
 		     unsigned length, unsigned offset,
-		     unsigned *litrunlen_p, struct deflate_sequence **next_seq_p)
+		     u32 *litrunlen_p, struct deflate_sequence **next_seq_p)
 {
-	unsigned litrunlen = *litrunlen_p;
 	struct deflate_sequence *seq = *next_seq_p;
 	unsigned length_slot = deflate_length_slot[length];
 	unsigned offset_slot = deflate_get_offset_slot(c, offset);
@@ -1786,8 +1825,7 @@ deflate_choose_match(struct deflate_compressor *c,
 	c->freqs.litlen[257 + length_slot]++;
 	c->freqs.offset[offset_slot]++;
 
-	seq->litrunlen = litrunlen;
-	seq->length = length;
+	seq->litrunlen_and_length = ((u32)length << 23) | *litrunlen_p;
 	seq->offset = offset;
 	seq->length_slot = length_slot;
 	seq->offset_symbol = offset_slot;
@@ -1797,10 +1835,9 @@ deflate_choose_match(struct deflate_compressor *c,
 }
 
 static forceinline void
-deflate_finish_sequence(struct deflate_sequence *seq, unsigned litrunlen)
+deflate_finish_sequence(struct deflate_sequence *seq, u32 litrunlen)
 {
-	seq->litrunlen = litrunlen;
-	seq->length = 0;
+	seq->litrunlen_and_length = litrunlen; /* length = 0 */
 }
 
 /******************************************************************************/
@@ -1819,7 +1856,7 @@ deflate_finish_sequence(struct deflate_sequence *seq, unsigned litrunlen)
  * but rather we combine many symbols into a single "observation type".  For
  * literals we only look at the high bits and low bits, and for matches we only
  * look at whether the match is long or not.  The assumption is that for typical
- * "real" data, places that are good block boundaries will tend to be noticable
+ * "real" data, places that are good block boundaries will tend to be noticeable
  * based only on changes in these aggregate frequencies, without looking for
  * subtle differences in individual symbols.  For example, a change from ASCII
  * bytes to non-ASCII bytes, or from few matches (generally less compressible)
@@ -1829,10 +1866,10 @@ deflate_finish_sequence(struct deflate_sequence *seq, unsigned litrunlen)
  * For determining whether the frequency distributions are "different enough" to
  * start a new block, the simply heuristic of splitting when the sum of absolute
  * differences exceeds a constant seems to be good enough.  We also add a number
- * proportional to the block size so that the algorithm is more likely to end
- * large blocks than small blocks.  This reflects the general expectation that
- * it will become increasingly beneficial to start a new block as the current
- * blocks grows larger.
+ * proportional to the block length so that the algorithm is more likely to end
+ * long blocks than short blocks.  This reflects the general expectation that it
+ * will become increasingly beneficial to start a new block as the current
+ * block grows longer.
  *
  * Finally, for an approximation, it is not strictly necessary that the exact
  * symbols being used are considered.  With "near-optimal parsing", for example,
@@ -1874,7 +1911,7 @@ observe_match(struct block_split_stats *stats, unsigned length)
 }
 
 static bool
-do_end_block_check(struct block_split_stats *stats, u32 block_size)
+do_end_block_check(struct block_split_stats *stats, u32 block_length)
 {
 	int i;
 
@@ -1893,8 +1930,8 @@ do_end_block_check(struct block_split_stats *stats, u32 block_size)
 		}
 
 		/* Ready to end the block? */
-		if (total_delta + (block_size >> 12) * stats->num_observations >=
-		    200 * stats->num_observations)
+		if (total_delta + (block_length / 4096) * stats->num_observations >=
+		    NUM_OBSERVATIONS_PER_BLOCK_CHECK * 200 / 512 * stats->num_observations)
 			return true;
 	}
 
@@ -1912,9 +1949,9 @@ should_end_block(struct block_split_stats *stats,
 		 const u8 *in_block_begin, const u8 *in_next, const u8 *in_end)
 {
 	/* Ready to check block split statistics? */
-	if (stats->num_new_observations < 512 ||
+	if (stats->num_new_observations < NUM_OBSERVATIONS_PER_BLOCK_CHECK ||
 	    in_next - in_block_begin < MIN_BLOCK_LENGTH ||
-	    in_end - in_next < 16384)
+	    in_end - in_next < MIN_BLOCK_LENGTH)
 		return false;
 
 	return do_end_block_check(stats, in_next - in_block_begin);
@@ -1922,11 +1959,28 @@ should_end_block(struct block_split_stats *stats,
 
 /******************************************************************************/
 
+/*
+ * This is the level 0 "compressor".  It always outputs uncompressed blocks.
+ */
+static size_t
+deflate_compress_none(struct libdeflate_compressor * restrict c,
+		      const u8 * restrict in, size_t in_nbytes,
+		      u8 * restrict out, size_t out_nbytes_avail)
+{
+	struct deflate_output_bitstream os;
+
+	deflate_init_output(&os, out, out_nbytes_avail);
+
+	deflate_write_uncompressed_blocks(&os, in, in_nbytes, true);
+
+	return deflate_flush_output(&os);
+}
+
 /*
  * This is the "greedy" DEFLATE compressor. It always chooses the longest match.
  */
 static size_t
-deflate_compress_greedy(struct deflate_compressor * restrict c,
+deflate_compress_greedy(struct libdeflate_compressor * restrict c,
 			const u8 * restrict in, size_t in_nbytes,
 			u8 * restrict out, size_t out_nbytes_avail)
 {
@@ -1945,7 +1999,8 @@ deflate_compress_greedy(struct deflate_compressor * restrict c,
 		/* Starting a new DEFLATE block.  */
 
 		const u8 * const in_block_begin = in_next;
-		const u8 * const in_max_block_end = in_next + MIN(in_end - in_next, MAX_BLOCK_LENGTH);
+		const u8 * const in_max_block_end =
+			in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
 		u32 litrunlen = 0;
 		struct deflate_sequence *next_seq = c->p.g.sequences;
 
@@ -2010,7 +2065,7 @@ deflate_compress_greedy(struct deflate_compressor * restrict c,
  * literal and continues to the next position.  If no, it outputs the match.
  */
 static size_t
-deflate_compress_lazy(struct deflate_compressor * restrict c,
+deflate_compress_lazy(struct libdeflate_compressor * restrict c,
 		      const u8 * restrict in, size_t in_nbytes,
 		      u8 * restrict out, size_t out_nbytes_avail)
 {
@@ -2029,7 +2084,8 @@ deflate_compress_lazy(struct deflate_compressor * restrict c,
 		/* Starting a new DEFLATE block.  */
 
 		const u8 * const in_block_begin = in_next;
-		const u8 * const in_max_block_end = in_next + MIN(in_end - in_next, MAX_BLOCK_LENGTH);
+		const u8 * const in_max_block_end =
+			in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
 		u32 litrunlen = 0;
 		struct deflate_sequence *next_seq = c->p.g.sequences;
 
@@ -2155,13 +2211,13 @@ deflate_compress_lazy(struct deflate_compressor * restrict c,
 /*
  * Follow the minimum-cost path in the graph of possible match/literal choices
  * for the current block and compute the frequencies of the Huffman symbols that
- * are needed to output those matches and literals.
+ * would be needed to output those matches and literals.
  */
 static void
-deflate_tally_item_list(struct deflate_compressor *c,
-			struct deflate_optimum_node *end_node)
+deflate_tally_item_list(struct libdeflate_compressor *c, u32 block_length)
 {
 	struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0];
+	struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length];
 	do {
 		unsigned length = cur_node->item & OPTIMUM_LEN_MASK;
 		unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
@@ -2180,7 +2236,8 @@ deflate_tally_item_list(struct deflate_compressor *c,
 
 /* Set the current cost model from the codeword lengths specified in @lens.  */
 static void
-deflate_set_costs(struct deflate_compressor *c, const struct deflate_lens *lens)
+deflate_set_costs_from_codes(struct libdeflate_compressor *c,
+			     const struct deflate_lens *lens)
 {
 	unsigned i;
 
@@ -2232,17 +2289,17 @@ deflate_default_offset_slot_cost(unsigned offset_slot)
 }
 
 /*
- * Set default Huffman symbol costs for the first optimization pass.
+ * Set default symbol costs for the first block's first optimization pass.
  *
- * It works well to assume that each Huffman symbol is equally probable.  This
- * results in each symbol being assigned a cost of (-log2(1.0/num_syms) * (1 <<
+ * It works well to assume that each symbol is equally probable.  This results
+ * in each symbol being assigned a cost of (-log2(1.0/num_syms) * (1 <<
  * COST_SHIFT)) where 'num_syms' is the number of symbols in the corresponding
  * alphabet.  However, we intentionally bias the parse towards matches rather
  * than literals by using a slightly lower default cost for length symbols than
  * for literals.  This often improves the compression ratio slightly.
  */
 static void
-deflate_set_default_costs(struct deflate_compressor *c)
+deflate_set_default_costs(struct libdeflate_compressor *c)
 {
 	unsigned i;
 
@@ -2276,7 +2333,7 @@ deflate_adjust_cost(u32 *cost_p, u32 default_cost)
  * defaults, but don't simply set them to the defaults.
  */
 static void
-deflate_adjust_costs(struct deflate_compressor *c)
+deflate_adjust_costs(struct libdeflate_compressor *c)
 {
 	unsigned i;
 
@@ -2297,120 +2354,131 @@ deflate_adjust_costs(struct deflate_compressor *c)
 				    deflate_default_offset_slot_cost(i));
 }
 
+/*
+ * Find the minimum-cost path through the graph of possible match/literal
+ * choices for this block.
+ *
+ * We find the minimum cost path from 'c->p.n.optimum_nodes[0]', which
+ * represents the node at the beginning of the block, to
+ * 'c->p.n.optimum_nodes[block_length]', which represents the node at the end of
+ * the block.  Edge costs are evaluated using the cost model 'c->p.n.costs'.
+ *
+ * The algorithm works backwards, starting at the end node and proceeding
+ * backwards one node at a time.  At each node, the minimum cost to reach the
+ * end node is computed and the match/literal choice that begins that path is
+ * saved.
+ */
 static void
-deflate_optimize_and_write_block(struct deflate_compressor *c,
-				 struct deflate_output_bitstream *os,
-				 const u8 * const block_begin,
-				 const u32 block_length,
-				 const struct lz_match * const end_cache_ptr,
-				 const bool is_final_block)
+deflate_find_min_cost_path(struct libdeflate_compressor *c,
+			   const u32 block_length,
+			   const struct lz_match *cache_ptr)
+{
+	struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length];
+	struct deflate_optimum_node *cur_node = end_node;
+
+	cur_node->cost_to_end = 0;
+	do {
+		unsigned num_matches;
+		unsigned literal;
+		u32 best_cost_to_end;
+
+		cur_node--;
+		cache_ptr--;
+
+		num_matches = cache_ptr->length;
+		literal = cache_ptr->offset;
+
+		/* It's always possible to choose a literal.  */
+		best_cost_to_end = c->p.n.costs.literal[literal] +
+				   (cur_node + 1)->cost_to_end;
+		cur_node->item = ((u32)literal << OPTIMUM_OFFSET_SHIFT) | 1;
+
+		/* Also consider matches if there are any.  */
+		if (num_matches) {
+			const struct lz_match *match;
+			unsigned len;
+			unsigned offset;
+			unsigned offset_slot;
+			u32 offset_cost;
+			u32 cost_to_end;
+
+			/*
+			 * Consider each length from the minimum
+			 * (DEFLATE_MIN_MATCH_LEN) to the length of the longest
+			 * match found at this position.  For each length, we
+			 * consider only the smallest offset for which that
+			 * length is available.  Although this is not guaranteed
+			 * to be optimal due to the possibility of a larger
+			 * offset costing less than a smaller offset to code,
+			 * this is a very useful heuristic.
+			 */
+			match = cache_ptr - num_matches;
+			len = DEFLATE_MIN_MATCH_LEN;
+			do {
+				offset = match->offset;
+				offset_slot = deflate_get_offset_slot(c, offset);
+				offset_cost = c->p.n.costs.offset_slot[offset_slot];
+				do {
+					cost_to_end = offset_cost +
+						      c->p.n.costs.length[len] +
+						      (cur_node + len)->cost_to_end;
+					if (cost_to_end < best_cost_to_end) {
+						best_cost_to_end = cost_to_end;
+						cur_node->item = ((u32)offset << OPTIMUM_OFFSET_SHIFT) | len;
+					}
+				} while (++len <= match->length);
+			} while (++match != cache_ptr);
+			cache_ptr -= num_matches;
+		}
+		cur_node->cost_to_end = best_cost_to_end;
+	} while (cur_node != &c->p.n.optimum_nodes[0]);
+}
+
+/*
+ * Choose the literal/match sequence to use for the current block.  The basic
+ * algorithm finds a minimum-cost path through the block's graph of
+ * literal/match choices, given a cost model.  However, the cost of each symbol
+ * is unknown until the Huffman codes have been built, but at the same time the
+ * Huffman codes depend on the frequencies of chosen symbols.  Consequently,
+ * multiple passes must be used to try to approximate an optimal solution.  The
+ * first pass uses default costs, mixed with the costs from the previous block
+ * if any.  Later passes use the Huffman codeword lengths from the previous pass
+ * as the costs.
+ */
+static void
+deflate_optimize_block(struct libdeflate_compressor *c, u32 block_length,
+		       const struct lz_match *cache_ptr, bool is_first_block)
 {
-	struct deflate_optimum_node * const end_node =
-		&c->p.n.optimum_nodes[block_length];
 	unsigned num_passes_remaining = c->p.n.num_optim_passes;
 	u32 i;
 
-	/* Force the block to really end at 'end_node', even if some matches
-	 * extend beyond it.  */
+	/* Force the block to really end at the desired length, even if some
+	 * matches extend beyond it. */
 	for (i = block_length; i <= MIN(block_length - 1 + DEFLATE_MAX_MATCH_LEN,
 					ARRAY_LEN(c->p.n.optimum_nodes) - 1); i++)
 		c->p.n.optimum_nodes[i].cost_to_end = 0x80000000;
 
-	do {
-		/*
-		 * Beginning a new optimization pass and finding a new
-		 * minimum-cost path through the graph of possible match/literal
-		 * choices for this block.
-		 *
-		 * We find the minimum cost path from 'c->optimum_nodes[0]',
-		 * which represents the node at the beginning of the block, to
-		 * 'end_node', which represents the node at the end of the
-		 * block.  Edge costs are evaluated using the cost model
-		 * 'c->costs'.
-		 *
-		 * The algorithm works backward, starting at 'end_node' and
-		 * proceeding backwards one position at a time.  At each
-		 * position, the minimum cost to reach 'end_node' is computed
-		 * and the match/literal choice is saved.
-		 */
-		struct deflate_optimum_node *cur_node = end_node;
-		const struct lz_match *cache_ptr = end_cache_ptr;
+	/* Set the initial costs. */
+	if (is_first_block)
+		deflate_set_default_costs(c);
+	else
+		deflate_adjust_costs(c);
 
-		cur_node->cost_to_end = 0;
-		do {
-			unsigned num_matches;
-			unsigned literal;
-			u32 best_cost_to_end;
-			u32 best_item;
+	for (;;) {
+		/* Find the minimum cost path for this pass. */
+		deflate_find_min_cost_path(c, block_length, cache_ptr);
 
-			cur_node--;
-			cache_ptr--;
+		/* Compute frequencies of the chosen symbols. */
+		deflate_reset_symbol_frequencies(c);
+		deflate_tally_item_list(c, block_length);
 
-			num_matches = cache_ptr->length;
-			literal = cache_ptr->offset;
+		if (--num_passes_remaining == 0)
+			break;
 
-			/* It's always possible to choose a literal.  */
-			best_cost_to_end = c->p.n.costs.literal[literal] +
-					   (cur_node + 1)->cost_to_end;
-			best_item = ((u32)literal << OPTIMUM_OFFSET_SHIFT) | 1;
-
-			/* Also consider matches if there are any.  */
-			if (num_matches) {
-				const struct lz_match *match;
-				unsigned len;
-				unsigned offset;
-				unsigned offset_slot;
-				u32 offset_cost;
-				u32 cost_to_end;
-
-				/*
-				 * Consider each length from the minimum
-				 * (DEFLATE_MIN_MATCH_LEN) to the length of the
-				 * longest match found at this position.  For
-				 * each length, we consider only the smallest
-				 * offset for which that length is available.
-				 * Although this is not guaranteed to be optimal
-				 * due to the possibility of a larger offset
-				 * costing less than a smaller offset to code,
-				 * this is a very useful heuristic.
-				 */
-				match = cache_ptr - num_matches;
-				len = DEFLATE_MIN_MATCH_LEN;
-				do {
-					offset = match->offset;
-					offset_slot = deflate_get_offset_slot(c, offset);
-					offset_cost = c->p.n.costs.offset_slot[offset_slot];
-					do {
-						cost_to_end = offset_cost +
-							      c->p.n.costs.length[len] +
-							      (cur_node + len)->cost_to_end;
-						if (cost_to_end < best_cost_to_end) {
-							best_cost_to_end = cost_to_end;
-							best_item = ((u32)offset << OPTIMUM_OFFSET_SHIFT) | len;
-						}
-					} while (++len <= match->length);
-				} while (++match != cache_ptr);
-				cache_ptr -= num_matches;
-			}
-			cur_node->cost_to_end = best_cost_to_end;
-			cur_node->item = best_item;
-		} while (cur_node != &c->p.n.optimum_nodes[0]);
-
-		/* Tally Huffman symbol frequencies.  */
-		deflate_tally_item_list(c, end_node);
-
-		/* If this wasn't the last pass, update the cost model.  */
-		if (num_passes_remaining > 1) {
-			deflate_make_huffman_codes(&c->freqs, &c->codes);
-			deflate_set_costs(c, &c->codes.lens);
-			deflate_reset_symbol_frequencies(c);
-		}
-	} while (--num_passes_remaining);
-
-	/* All optimization passes are done.  Output a block using the
-	 * minimum-cost path computed on the last optimization pass.  */
-	deflate_flush_block(c, os, block_begin, block_length,
-			    is_final_block, true);
+		/* At least one optimization pass remains; update the costs. */
+		deflate_make_huffman_codes(&c->freqs, &c->codes);
+		deflate_set_costs_from_codes(c, &c->codes.lens);
+	}
 }
 
 /*
@@ -2427,7 +2495,7 @@ deflate_optimize_and_write_block(struct deflate_compressor *c,
  *   (so iterative optimization must be used)
  */
 static size_t
-deflate_compress_near_optimal(struct deflate_compressor * restrict c,
+deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
 			      const u8 * restrict in, size_t in_nbytes,
 			      u8 * restrict out, size_t out_nbytes_avail)
 {
@@ -2448,17 +2516,17 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
 
 		struct lz_match *cache_ptr = c->p.n.match_cache;
 		const u8 * const in_block_begin = in_next;
-		const u8 * const in_max_block_end = in_next + MIN(in_end - in_next, MAX_BLOCK_LENGTH);
+		const u8 * const in_max_block_end =
+			in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
 		const u8 *next_observation = in_next;
 
 		init_block_split_stats(&c->split_stats);
-		deflate_reset_symbol_frequencies(c);
 
 		/*
 		 * Find matches until we decide to end the block.  We end the
 		 * block if any of the following is true:
 		 *
-		 * (1) Maximum block size has been reached
+		 * (1) Maximum block length has been reached
 		 * (2) Match catch may overflow.
 		 * (3) Block split heuristic says to split now.
 		 */
@@ -2556,7 +2624,6 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
 						bt_matchfinder_skip_position(&c->p.n.bt_mf,
 									     in_cur_base,
 									     in_next - in_cur_base,
-									     max_len,
 									     nice_len,
 									     c->max_search_depth,
 									     next_hashes);
@@ -2571,16 +2638,12 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
 			 cache_ptr < &c->p.n.match_cache[CACHE_LENGTH] &&
 			 !should_end_block(&c->split_stats, in_block_begin, in_next, in_end));
 
-		/* All the matches for this block have been cached.  Now compute
-		 * a near-optimal sequence of literals and matches, and output
-		 * the block.  */
-		if (in_block_begin == in)
-			deflate_set_default_costs(c);
-		else
-			deflate_adjust_costs(c);
-		deflate_optimize_and_write_block(c, &os, in_block_begin,
-						 in_next - in_block_begin,
-						 cache_ptr, in_next == in_end);
+		/* All the matches for this block have been cached.  Now choose
+		 * the sequence of items to output and flush the block.  */
+		deflate_optimize_block(c, in_next - in_block_begin, cache_ptr,
+				       in_block_begin == in);
+		deflate_flush_block(c, &os, in_block_begin, in_next - in_block_begin,
+				    in_next == in_end, true);
 	} while (in_next != in_end);
 
 	return deflate_flush_output(&os);
@@ -2590,7 +2653,7 @@ deflate_compress_near_optimal(struct deflate_compressor * restrict c,
 
 /* Initialize c->offset_slot_fast.  */
 static void
-deflate_init_offset_slot_fast(struct deflate_compressor *c)
+deflate_init_offset_slot_fast(struct libdeflate_compressor *c)
 {
 	unsigned offset_slot;
 	unsigned offset;
@@ -2622,26 +2685,41 @@ deflate_init_offset_slot_fast(struct deflate_compressor *c)
 	}
 }
 
-LIBEXPORT struct deflate_compressor *
-deflate_alloc_compressor(unsigned int compression_level)
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+libdeflate_alloc_compressor(int compression_level)
 {
-	struct deflate_compressor *c;
-	size_t size;
+	struct libdeflate_compressor *c;
+	size_t size = offsetof(struct libdeflate_compressor, p);
+
+	if (compression_level < 0 || compression_level > 12)
+		return NULL;
 
 #if SUPPORT_NEAR_OPTIMAL_PARSING
 	if (compression_level >= 8)
-		size = offsetof(struct deflate_compressor, p) + sizeof(c->p.n);
-	else
+		size += sizeof(c->p.n);
+	else if (compression_level >= 1)
+		size += sizeof(c->p.g);
+#else
+	if (compression_level >= 1)
+		size += sizeof(c->p.g);
 #endif
-		size = offsetof(struct deflate_compressor, p) + sizeof(c->p.g);
 
-	c = aligned_malloc(MATCHFINDER_ALIGNMENT, size);
+	c = libdeflate_aligned_malloc(MATCHFINDER_MEM_ALIGNMENT, size);
 	if (!c)
 		return NULL;
 
 	c->compression_level = compression_level;
 
+	/*
+	 * The higher the compression level, the more we should bother trying to
+	 * compress very small inputs.
+	 */
+	c->min_size_to_compress = 56 - (compression_level * 4);
+
 	switch (compression_level) {
+	case 0:
+		c->impl = deflate_compress_none;
+		break;
 	case 1:
 		c->impl = deflate_compress_greedy;
 		c->max_search_depth = 2;
@@ -2702,7 +2780,7 @@ deflate_alloc_compressor(unsigned int compression_level)
 		c->nice_match_length = 80;
 		c->p.n.num_optim_passes = 3;
 		break;
-	case 12:
+	default:
 		c->impl = deflate_compress_near_optimal;
 		c->max_search_depth = 100;
 		c->nice_match_length = 133;
@@ -2714,15 +2792,12 @@ deflate_alloc_compressor(unsigned int compression_level)
 		c->max_search_depth = 150;
 		c->nice_match_length = 200;
 		break;
-	case 9:
+	default:
 		c->impl = deflate_compress_lazy;
 		c->max_search_depth = 200;
 		c->nice_match_length = DEFLATE_MAX_MATCH_LEN;
 		break;
 #endif
-	default:
-		aligned_free(c);
-		return NULL;
 	}
 
 	deflate_init_offset_slot_fast(c);
@@ -2731,18 +2806,20 @@ deflate_alloc_compressor(unsigned int compression_level)
 	return c;
 }
 
-LIBEXPORT size_t
-deflate_compress(struct deflate_compressor *c,
-		 const void *in, size_t in_nbytes,
-		 void *out, size_t out_nbytes_avail)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress(struct libdeflate_compressor *c,
+			    const void *in, size_t in_nbytes,
+			    void *out, size_t out_nbytes_avail)
 {
-	if (unlikely(out_nbytes_avail < MIN_OUTPUT_SIZE))
+	if (unlikely(out_nbytes_avail < OUTPUT_END_PADDING))
 		return 0;
 
 	/* For extremely small inputs just use a single uncompressed block. */
-	if (unlikely(in_nbytes < 16)) {
+	if (unlikely(in_nbytes < c->min_size_to_compress)) {
 		struct deflate_output_bitstream os;
 		deflate_init_output(&os, out, out_nbytes_avail);
+		if (in_nbytes == 0)
+			in = &os; /* Avoid passing NULL to memcpy() */
 		deflate_write_uncompressed_block(&os, in, in_nbytes, true);
 		return deflate_flush_output(&os);
 	}
@@ -2750,20 +2827,21 @@ deflate_compress(struct deflate_compressor *c,
 	return (*c->impl)(c, in, in_nbytes, out, out_nbytes_avail);
 }
 
-LIBEXPORT void
-deflate_free_compressor(struct deflate_compressor *c)
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_compressor(struct libdeflate_compressor *c)
 {
-	aligned_free(c);
+	libdeflate_aligned_free(c);
 }
 
 unsigned int
-deflate_get_compression_level(struct deflate_compressor *c)
+deflate_get_compression_level(struct libdeflate_compressor *c)
 {
 	return c->compression_level;
 }
 
-LIBEXPORT size_t
-deflate_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *c,
+				  size_t in_nbytes)
 {
 	/*
 	 * The worst case is all uncompressed blocks where one block has length
@@ -2772,5 +2850,5 @@ deflate_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
 	 * and alignment to a byte boundary; 2 for LEN; and 2 for NLEN.
 	 */
 	size_t max_num_blocks = MAX(DIV_ROUND_UP(in_nbytes, MIN_BLOCK_LENGTH), 1);
-	return (5 * max_num_blocks) + in_nbytes + 1 + MIN_OUTPUT_SIZE;
+	return (5 * max_num_blocks) + in_nbytes + 1 + OUTPUT_END_PADDING;
 }

lib/deflate_compress.h

@@ -1,11 +1,13 @@
-#ifndef _LIB_DEFLATE_COMPRESS_H
-#define _LIB_DEFLATE_COMPRESS_H
+#ifndef LIB_DEFLATE_COMPRESS_H
+#define LIB_DEFLATE_COMPRESS_H
+
+#include "lib_common.h"
 
 /* DEFLATE compression is private to deflate_compress.c, but we do need to be
  * able to query the compression level for zlib and gzip header generation.  */
 
-struct deflate_compressor;
+struct libdeflate_compressor;
 
-extern unsigned int deflate_get_compression_level(struct deflate_compressor *c);
+unsigned int deflate_get_compression_level(struct libdeflate_compressor *c);
 
-#endif /* _LIB_DEFLATE_COMPRESS_H */
+#endif /* LIB_DEFLATE_COMPRESS_H */

lib/deflate_constants.h

@@ -2,8 +2,8 @@
  * deflate_constants.h - constants for the DEFLATE compression format
  */
 
-#ifndef _LIB_DEFLATE_CONSTANTS_H
-#define _LIB_DEFLATE_CONSTANTS_H
+#ifndef LIB_DEFLATE_CONSTANTS_H
+#define LIB_DEFLATE_CONSTANTS_H
 
 /* Valid block types  */
 #define DEFLATE_BLOCKTYPE_UNCOMPRESSED		0
@@ -63,4 +63,4 @@
 	(DEFLATE_MAX_LITLEN_CODEWORD_LEN + DEFLATE_MAX_EXTRA_LENGTH_BITS + \
 	DEFLATE_MAX_OFFSET_CODEWORD_LEN + DEFLATE_MAX_EXTRA_OFFSET_BITS)
 
-#endif /* _LIB_DEFLATE_CONSTANTS_H */
+#endif /* LIB_DEFLATE_CONSTANTS_H */

lib/deflate_decompress.c

@@ -1,15 +1,28 @@
 /*
  * deflate_decompress.c - a decompressor for DEFLATE
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Copyright 2016 Eric Biggers
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
  * ---------------------------------------------------------------------------
  *
@@ -31,18 +44,16 @@
  *   instructions and use it automatically at runtime when supported.
  */
 
-#include <stdlib.h>
-#include <string.h>
+#include <limits.h>
 
 #include "deflate_constants.h"
 #include "unaligned.h"
-#include "x86_cpu_features.h"
 
 #include "libdeflate.h"
 
 /*
  * If the expression passed to SAFETY_CHECK() evaluates to false, then the
- * decompression routine immediately returns DECOMPRESS_BAD_DATA, indicating the
+ * decompression routine immediately returns LIBDEFLATE_BAD_DATA, indicating the
  * compressed data is invalid.
  *
  * Theoretically, these checks could be disabled for specialized applications
@@ -52,7 +63,7 @@
 #  pragma message("UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!")
 #  define SAFETY_CHECK(expr)	(void)(expr)
 #else
-#  define SAFETY_CHECK(expr)	if (unlikely(!(expr))) return DECOMPRESS_BAD_DATA
+#  define SAFETY_CHECK(expr)	if (unlikely(!(expr))) return LIBDEFLATE_BAD_DATA
 #endif
 
 /*
@@ -75,7 +86,7 @@
  * required for the corresponding Huffman code, including the main table and all
  * subtables.  Each number depends on three parameters:
  *
- *	(1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMBOLS)
+ *	(1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMS)
  *	(2) the number of main table bits (the TABLEBITS numbers defined above)
  *	(3) the maximum allowed codeword length (DEFLATE_MAX_*_CODEWORD_LEN)
  *
@@ -98,7 +109,7 @@ typedef u8 len_t;
  * decompression state, but rather only some arrays that are too large to
  * comfortably allocate on the stack.
  */
-struct deflate_decompressor {
+struct libdeflate_decompressor {
 
 	/*
 	 * The arrays aren't all needed at the same time.  'precode_lens' and
@@ -125,8 +136,10 @@ struct deflate_decompressor {
 
 	u32 offset_decode_table[OFFSET_ENOUGH];
 
-	u16 working_space[2 * (DEFLATE_MAX_CODEWORD_LEN + 1) +
-			  DEFLATE_MAX_NUM_SYMS];
+	/* used only during build_decode_table() */
+	u16 sorted_syms[DEFLATE_MAX_NUM_SYMS];
+
+	bool static_codes_loaded;
 };
 
 /*****************************************************************************
@@ -162,18 +175,18 @@ typedef machine_word_t bitbuf_t;
 
 /*
  * Number of bits the bitbuffer variable can hold.
+ *
+ * This is one less than the obvious value because of the optimized arithmetic
+ * in FILL_BITS_WORDWISE() that leaves 'bitsleft' in the range
+ * [WORDBITS - 8, WORDBITS - 1] rather than [WORDBITS - 7, WORDBITS].
  */
-#define BITBUF_NBITS	(8 * sizeof(bitbuf_t))
+#define BITBUF_NBITS	(8 * sizeof(bitbuf_t) - 1)
 
 /*
- * The maximum number of bits that can be requested to be in the bitbuffer
- * variable.  This is the maximum value of 'n' that can be passed
- * ENSURE_BITS(n).
- *
- * This not equal to BITBUF_NBITS because we never read less than one byte at a
- * time.  If the bitbuffer variable contains more than (BITBUF_NBITS - 8) bits,
- * then we can't read another byte without first consuming some bits.  So the
- * maximum count we can ensure is (BITBUF_NBITS - 7).
+ * The maximum number of bits that can be ensured in the bitbuffer variable,
+ * i.e. the maximum value of 'n' that can be passed ENSURE_BITS(n).  The decoder
+ * only reads whole bytes from memory, so this is the lowest value of 'bitsleft'
+ * at which another byte cannot be read without first consuming some bits.
  */
 #define MAX_ENSURE	(BITBUF_NBITS - 7)
 
@@ -189,17 +202,20 @@ typedef machine_word_t bitbuf_t;
 /*
  * Fill the bitbuffer variable, reading one byte at a time.
  *
- * Note: if we would overrun the input buffer, we just don't read anything,
- * leaving the bits as 0 but marking them as filled.  This makes the
- * implementation simpler because this removes the need to distinguish between
- * "real" overruns and overruns that occur because of our own lookahead during
- * Huffman decoding.  The disadvantage is that a "real" overrun can go
- * undetected, and deflate_decompress() may return a success status rather than
- * the expected failure status if one occurs.  However, this is irrelevant
- * because even if this specific case were to be handled "correctly", one could
- * easily come up with a different case where the compressed data would be
- * corrupted in such a way that fully retains its validity.  Users should run a
- * checksum against the uncompressed data if they wish to detect corruptions.
+ * If we would overread the input buffer, we just don't read anything, leaving
+ * the bits zeroed but marking them filled.  This simplifies the decompressor
+ * because it removes the need to distinguish between real overreads and
+ * overreads that occur only because of the decompressor's own lookahead.
+ *
+ * The disadvantage is that real overreads are not detected immediately.
+ * However, this is safe because the decompressor is still guaranteed to make
+ * forward progress when presented never-ending 0 bits.  In an existing block
+ * output will be getting generated, whereas new blocks can only be uncompressed
+ * (since the type code for uncompressed blocks is 0), for which we check for
+ * previous overread.  But even if we didn't check, uncompressed blocks would
+ * fail to validate because LEN would not equal ~NLEN.  So the decompressor will
+ * eventually either detect that the output buffer is full, or detect invalid
+ * input, or finish the final block.
  */
 #define FILL_BITS_BYTEWISE()					\
 do {								\
@@ -211,18 +227,38 @@ do {								\
 } while (bitsleft <= BITBUF_NBITS - 8)
 
 /*
- * Fill the bitbuffer variable by reading the next word from the input buffer.
- * This can be significantly faster than FILL_BITS_BYTEWISE().  However, for
- * this to work correctly, the word must be interpreted in little-endian format.
- * In addition, the memory access may be unaligned.  Therefore, this method is
- * most efficient on little-endian architectures that support fast unaligned
- * access, such as x86 and x86_64.
+ * Fill the bitbuffer variable by reading the next word from the input buffer
+ * and branchlessly updating 'in_next' and 'bitsleft' based on how many bits
+ * were filled.  This can be significantly faster than FILL_BITS_BYTEWISE().
+ * However, for this to work correctly, the word must be interpreted in
+ * little-endian format.  In addition, the memory access may be unaligned.
+ * Therefore, this method is most efficient on little-endian architectures that
+ * support fast unaligned access, such as x86 and x86_64.
+ *
+ * For faster updating of 'bitsleft', we consider the bitbuffer size in bits to
+ * be 1 less than the word size and therefore be all 1 bits.  Then the number of
+ * bits filled is the value of the 0 bits in position >= 3 when changed to 1.
+ * E.g. if words are 64 bits and bitsleft = 16 = b010000 then we refill b101000
+ * = 40 bits = 5 bytes.  This uses only 4 operations to update 'in_next' and
+ * 'bitsleft': one each of +, ^, >>, and |.  (Not counting operations the
+ * compiler optimizes out.)  In contrast, the alternative of:
+ *
+ *	in_next += (BITBUF_NBITS - bitsleft) >> 3;
+ *	bitsleft += (BITBUF_NBITS - bitsleft) & ~7;
+ *
+ * (where BITBUF_NBITS would be WORDBITS rather than WORDBITS - 1) would on
+ * average refill an extra bit, but uses 5 operations: two +, and one each of
+ * -, >>, and &.  Also the - and & must be completed before 'bitsleft' can be
+ * updated, while the current solution updates 'bitsleft' with no dependencies.
  */
 #define FILL_BITS_WORDWISE()					\
 do {								\
+	/* BITBUF_NBITS must be all 1's in binary, see above */	\
+	STATIC_ASSERT((BITBUF_NBITS & (BITBUF_NBITS + 1)) == 0);\
+								\
 	bitbuf |= get_unaligned_leword(in_next) << bitsleft;	\
-	in_next += (BITBUF_NBITS - bitsleft) >> 3;		\
-	bitsleft += (BITBUF_NBITS - bitsleft) & ~7;		\
+	in_next += (bitsleft ^ BITBUF_NBITS) >> 3;		\
+	bitsleft |= BITBUF_NBITS & ~7;				\
 } while (0)
 
 /*
@@ -261,17 +297,19 @@ if (!HAVE_BITS(n)) {						\
 #define POP_BITS(n) (tmp32 = BITS(n), REMOVE_BITS(n), tmp32)
 
 /*
- * Align the input to the next byte boundary, discarding any remaining bits in
- * the current byte.
+ * Verify that the input buffer hasn't been overread, then align the input to
+ * the next byte boundary, discarding any remaining bits in the current byte.
  *
- * Note that if the bitbuffer variable currently contains more than 8 bits, then
+ * Note that if the bitbuffer variable currently contains more than 7 bits, then
  * we must rewind 'in_next', effectively putting those bits back.  Only the bits
  * in what would be the "current" byte if we were reading one byte at a time can
  * be actually discarded.
  */
 #define ALIGN_INPUT()							\
 do {									\
-	in_next -= (bitsleft >> 3) - MIN(overrun_count, bitsleft >> 3);	\
+	SAFETY_CHECK(overrun_count <= (bitsleft >> 3));			\
+	in_next -= (bitsleft >> 3) - overrun_count;			\
+	overrun_count = 0;						\
 	bitbuf = 0;							\
 	bitsleft = 0;							\
 } while(0)
@@ -296,7 +334,7 @@ do {									\
  * pointer", meaning that the entry references a subtable that must be indexed
  * using more bits of the compressed data to decode the symbol.
  *
- * Each decode table (a main table along with with its subtables, if any) is
+ * Each decode table (a main table along with its subtables, if any) is
  * associated with a Huffman code.  Logically, the result of a decode table
  * lookup is a symbol from the alphabet from which the corresponding Huffman
  * code was constructed.  A symbol with codeword length n <= TABLEBITS is
@@ -345,19 +383,28 @@ do {									\
 /* Shift to extract the decode result from a decode table entry.  */
 #define HUFFDEC_RESULT_SHIFT		8
 
+/* Shift a decode result into its position in the decode table entry.  */
+#define HUFFDEC_RESULT_ENTRY(result)	((u32)(result) << HUFFDEC_RESULT_SHIFT)
+
 /* The decode result for each precode symbol.  There is no special optimization
  * for the precode; the decode result is simply the symbol value.  */
 static const u32 precode_decode_results[DEFLATE_NUM_PRECODE_SYMS] = {
-	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+#define ENTRY(presym)	HUFFDEC_RESULT_ENTRY(presym)
+	ENTRY(0)   , ENTRY(1)   , ENTRY(2)   , ENTRY(3)   ,
+	ENTRY(4)   , ENTRY(5)   , ENTRY(6)   , ENTRY(7)   ,
+	ENTRY(8)   , ENTRY(9)   , ENTRY(10)  , ENTRY(11)  ,
+	ENTRY(12)  , ENTRY(13)  , ENTRY(14)  , ENTRY(15)  ,
+	ENTRY(16)  , ENTRY(17)  , ENTRY(18)  ,
+#undef ENTRY
 };
 
 /* The decode result for each litlen symbol.  For literals, this is the literal
  * value itself and the HUFFDEC_LITERAL flag.  For lengths, this is the length
  * base and the number of extra length bits.  */
 static const u32 litlen_decode_results[DEFLATE_NUM_LITLEN_SYMS] = {
-#define ENTRY(literal)	((HUFFDEC_LITERAL >> HUFFDEC_RESULT_SHIFT) | (literal))
 
 	/* Literals  */
+#define ENTRY(literal)	(HUFFDEC_LITERAL | HUFFDEC_RESULT_ENTRY(literal))
 	ENTRY(0)   , ENTRY(1)   , ENTRY(2)   , ENTRY(3)   ,
 	ENTRY(4)   , ENTRY(5)   , ENTRY(6)   , ENTRY(7)   ,
 	ENTRY(8)   , ENTRY(9)   , ENTRY(10)  , ENTRY(11)  ,
@@ -428,8 +475,8 @@ static const u32 litlen_decode_results[DEFLATE_NUM_LITLEN_SYMS] = {
 #define HUFFDEC_LENGTH_BASE_SHIFT	8
 #define HUFFDEC_END_OF_BLOCK_LENGTH	0
 
-#define ENTRY(length_base, num_extra_bits) \
-	(((u32)(length_base) << HUFFDEC_LENGTH_BASE_SHIFT) | (num_extra_bits))
+#define ENTRY(length_base, num_extra_bits)	HUFFDEC_RESULT_ENTRY(	\
+	((u32)(length_base) << HUFFDEC_LENGTH_BASE_SHIFT) | (num_extra_bits))
 
 	/* End of block  */
 	ENTRY(HUFFDEC_END_OF_BLOCK_LENGTH, 0),
@@ -453,8 +500,9 @@ static const u32 offset_decode_results[DEFLATE_NUM_OFFSET_SYMS] = {
 #define HUFFDEC_EXTRA_OFFSET_BITS_SHIFT 16
 #define HUFFDEC_OFFSET_BASE_MASK (((u32)1 << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT) - 1)
 
-#define ENTRY(offset_base, num_extra_bits) \
-	((offset_base) | ((u32)(num_extra_bits) << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT))
+#define ENTRY(offset_base, num_extra_bits)	HUFFDEC_RESULT_ENTRY(	\
+		((u32)(num_extra_bits) << HUFFDEC_EXTRA_OFFSET_BITS_SHIFT) | \
+		(offset_base))
 	ENTRY(1     , 0)  , ENTRY(2     , 0)  , ENTRY(3     , 0)  , ENTRY(4     , 0)  ,
 	ENTRY(5     , 1)  , ENTRY(7     , 1)  , ENTRY(9     , 2)  , ENTRY(13    , 2) ,
 	ENTRY(17    , 3)  , ENTRY(25    , 3)  , ENTRY(33    , 4)  , ENTRY(49    , 4)  ,
@@ -466,13 +514,6 @@ static const u32 offset_decode_results[DEFLATE_NUM_OFFSET_SYMS] = {
 #undef ENTRY
 };
 
-/* Construct a decode table entry from a decode result and codeword length.  */
-static forceinline u32
-make_decode_table_entry(u32 result, u32 length)
-{
-	return (result << HUFFDEC_RESULT_SHIFT) | length;
-}
-
 /*
  * Build a table for fast decoding of symbols from a Huffman code.  As input,
  * this function takes the codeword length of each symbol which may be used in
@@ -504,8 +545,9 @@ make_decode_table_entry(u32 result, u32 length)
  *	The log base-2 of the number of main table entries to use.
  * @max_codeword_len
  *	The maximum allowed codeword length for this Huffman code.
- * @working_space
- *	A temporary array of length '2 * (@max_codeword_len + 1) + @num_syms'.
+ *	Must be <= DEFLATE_MAX_CODEWORD_LEN.
+ * @sorted_syms
+ *	A temporary array of length @num_syms.
  *
  * Returns %true if successful; %false if the codeword lengths do not form a
  * valid Huffman code.
@@ -517,202 +559,264 @@ build_decode_table(u32 decode_table[],
 		   const u32 decode_results[],
 		   const unsigned table_bits,
 		   const unsigned max_codeword_len,
-		   u16 working_space[])
+		   u16 *sorted_syms)
 {
-	u16 * const len_counts = &working_space[0];
-	u16 * const offsets = &working_space[1 * (max_codeword_len + 1)];
-	u16 * const sorted_syms = &working_space[2 * (max_codeword_len + 1)];
-	unsigned len;
-	unsigned sym;
-	s32 remainder;
-	unsigned sym_idx;
-	unsigned codeword_len;
-	unsigned codeword_reversed = 0;
-	unsigned cur_codeword_prefix = -1;
-	unsigned cur_table_start = 0;
-	unsigned cur_table_bits = table_bits;
-	unsigned num_dropped_bits = 0;
-	const unsigned table_mask = (1U << table_bits) - 1;
+	unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
+	unsigned offsets[DEFLATE_MAX_CODEWORD_LEN + 1];
+	unsigned sym;		/* current symbol */
+	unsigned codeword;	/* current codeword, bit-reversed */
+	unsigned len;		/* current codeword length in bits */
+	unsigned count;		/* num codewords remaining with this length */
+	u32 codespace_used;	/* codespace used out of '2^max_codeword_len' */
+	unsigned cur_table_end; /* end index of current table */
+	unsigned subtable_prefix; /* codeword prefix of current subtable */
+	unsigned subtable_start;  /* start index of current subtable */
+	unsigned subtable_bits;   /* log2 of current subtable length */
 
-	/* Count how many symbols have each codeword length, including 0.  */
+	/* Count how many codewords have each length, including 0. */
 	for (len = 0; len <= max_codeword_len; len++)
 		len_counts[len] = 0;
 	for (sym = 0; sym < num_syms; sym++)
 		len_counts[lens[sym]]++;
 
-	/* Sort the symbols primarily by increasing codeword length and
-	 * secondarily by increasing symbol value.  */
+	/*
+	 * Sort the symbols primarily by increasing codeword length and
+	 * secondarily by increasing symbol value; or equivalently by their
+	 * codewords in lexicographic order, since a canonical code is assumed.
+	 *
+	 * For efficiency, also compute 'codespace_used' in the same pass over
+	 * 'len_counts[]' used to build 'offsets[]' for sorting.
+	 */
+
+	/* Ensure that 'codespace_used' cannot overflow. */
+	STATIC_ASSERT(sizeof(codespace_used) == 4);
+	STATIC_ASSERT(UINT32_MAX / (1U << (DEFLATE_MAX_CODEWORD_LEN - 1)) >=
+		      DEFLATE_MAX_NUM_SYMS);
 
-	/* Initialize 'offsets' so that offsets[len] is the number of codewords
-	 * shorter than 'len' bits, including length 0.  */
 	offsets[0] = 0;
-	for (len = 0; len < max_codeword_len; len++)
+	offsets[1] = len_counts[0];
+	codespace_used = 0;
+	for (len = 1; len < max_codeword_len; len++) {
 		offsets[len + 1] = offsets[len] + len_counts[len];
+		codespace_used = (codespace_used << 1) + len_counts[len];
+	}
+	codespace_used = (codespace_used << 1) + len_counts[len];
 
-	/* Use the 'offsets' array to sort the symbols.  */
 	for (sym = 0; sym < num_syms; sym++)
 		sorted_syms[offsets[lens[sym]]++] = sym;
 
-	/* It is already guaranteed that all lengths are <= max_codeword_len,
-	 * but it cannot be assumed they form a complete prefix code.  A
-	 * codeword of length n should require a proportion of the codespace
-	 * equaling (1/2)^n.  The code is complete if and only if, by this
-	 * measure, the codespace is exactly filled by the lengths.  */
-	remainder = 1;
-	for (len = 1; len <= max_codeword_len; len++) {
-		remainder <<= 1;
-		remainder -= len_counts[len];
-		if (unlikely(remainder < 0)) {
-			/* The lengths overflow the codespace; that is, the code
-			 * is over-subscribed.  */
-			return false;
-		}
-	}
+	sorted_syms += offsets[0]; /* Skip unused symbols */
 
-	if (unlikely(remainder != 0)) {
-		/* The lengths do not fill the codespace; that is, they form an
-		 * incomplete code.  */
+	/* lens[] is done being used, so we can write to decode_table[] now. */
 
-		/* Initialize the table entries to default values.  When
-		 * decompressing a well-formed stream, these default values will
-		 * never be used.  But since a malformed stream might contain
-		 * any bits at all, these entries need to be set anyway.  */
-		u32 entry = make_decode_table_entry(decode_results[0], 1);
-		for (sym = 0; sym < (1U << table_bits); sym++)
-			decode_table[sym] = entry;
+	/*
+	 * Check whether the lengths form a complete code (exactly fills the
+	 * codespace), an incomplete code (doesn't fill the codespace), or an
+	 * overfull code (overflows the codespace).  A codeword of length 'n'
+	 * uses proportion '1/(2^n)' of the codespace.  An overfull code is
+	 * nonsensical, so is considered invalid.  An incomplete code is
+	 * considered valid only in two specific cases; see below.
+	 */
 
-		/* A completely empty code is permitted.  */
-		if (remainder == (1U << max_codeword_len))
-			return true;
+	/* overfull code? */
+	if (unlikely(codespace_used > (1U << max_codeword_len)))
+		return false;
 
-		/* The code is nonempty and incomplete.  Proceed only if there
-		 * is a single used symbol and its codeword has length 1.  The
-		 * DEFLATE RFC is somewhat unclear regarding this case.  What
-		 * zlib's decompressor does is permit this case for
-		 * literal/length and offset codes and assume the codeword is 0
-		 * rather than 1.  We do the same except we allow this case for
-		 * precodes too.  */
-		if (remainder != (1U << (max_codeword_len - 1)) ||
-		    len_counts[1] != 1)
-			return false;
-	}
-
-	/* Generate the decode table entries.  Since we process codewords from
-	 * shortest to longest, the main portion of the decode table is filled
-	 * first; then the subtables are filled.  Note that it's already been
-	 * verified that the code is nonempty and not over-subscribed.  */
-
-	/* Start with the smallest codeword length and the smallest-valued
-	 * symbol which has that codeword length.  */
-	sym_idx = offsets[0];
-	codeword_len = 1;
-	while (len_counts[codeword_len] == 0)
-		codeword_len++;
-
-	for (;;) {  /* For each used symbol and its codeword...  */
-		unsigned sym;
+	/* incomplete code? */
+	if (unlikely(codespace_used < (1U << max_codeword_len))) {
 		u32 entry;
 		unsigned i;
-		unsigned end;
-		unsigned increment;
+
+		if (codespace_used == 0) {
+			/*
+			 * An empty code is allowed.  This can happen for the
+			 * offset code in DEFLATE, since a dynamic Huffman block
+			 * need not contain any matches.
+			 */
+
+			/* sym=0, len=1 (arbitrary) */
+			entry = decode_results[0] | 1;
+		} else {
+			/*
+			 * Allow codes with a single used symbol, with codeword
+			 * length 1.  The DEFLATE RFC is unclear regarding this
+			 * case.  What zlib's decompressor does is permit this
+			 * for the litlen and offset codes and assume the
+			 * codeword is '0' rather than '1'.  We do the same
+			 * except we allow this for precodes too, since there's
+			 * no convincing reason to treat the codes differently.
+			 * We also assign both codewords '0' and '1' to the
+			 * symbol to avoid having to handle '1' specially.
+			 */
+			if (codespace_used != (1U << (max_codeword_len - 1)) ||
+			    len_counts[1] != 1)
+				return false;
+			entry = decode_results[*sorted_syms] | 1;
+		}
+		/*
+		 * Note: the decode table still must be fully initialized, in
+		 * case the stream is malformed and contains bits from the part
+		 * of the codespace the incomplete code doesn't use.
+		 */
+		for (i = 0; i < (1U << table_bits); i++)
+			decode_table[i] = entry;
+		return true;
+	}
+
+	/*
+	 * The lengths form a complete code.  Now, enumerate the codewords in
+	 * lexicographic order and fill the decode table entries for each one.
+	 *
+	 * First, process all codewords with len <= table_bits.  Each one gets
+	 * '2^(table_bits-len)' direct entries in the table.
+	 *
+	 * Since DEFLATE uses bit-reversed codewords, these entries aren't
+	 * consecutive but rather are spaced '2^len' entries apart.  This makes
+	 * filling them naively somewhat awkward and inefficient, since strided
+	 * stores are less cache-friendly and preclude the use of word or
+	 * vector-at-a-time stores to fill multiple entries per instruction.
+	 *
+	 * To optimize this, we incrementally double the table size.  When
+	 * processing codewords with length 'len', the table is treated as
+	 * having only '2^len' entries, so each codeword uses just one entry.
+	 * Then, each time 'len' is incremented, the table size is doubled and
+	 * the first half is copied to the second half.  This significantly
+	 * improves performance over naively doing strided stores.
+	 *
+	 * Note that some entries copied for each table doubling may not have
+	 * been initialized yet, but it doesn't matter since they're guaranteed
+	 * to be initialized later (because the Huffman code is complete).
+	 */
+	codeword = 0;
+	len = 1;
+	while ((count = len_counts[len]) == 0)
+		len++;
+	cur_table_end = 1U << len;
+	while (len <= table_bits) {
+		/* Process all 'count' codewords with length 'len' bits. */
+		do {
+			unsigned bit;
+
+			/* Fill the first entry for the current codeword. */
+			decode_table[codeword] =
+				decode_results[*sorted_syms++] | len;
+
+			if (codeword == cur_table_end - 1) {
+				/* Last codeword (all 1's) */
+				for (; len < table_bits; len++) {
+					memcpy(&decode_table[cur_table_end],
+					       decode_table,
+					       cur_table_end *
+						sizeof(decode_table[0]));
+					cur_table_end <<= 1;
+				}
+				return true;
+			}
+			/*
+			 * To advance to the lexicographically next codeword in
+			 * the canonical code, the codeword must be incremented,
+			 * then 0's must be appended to the codeword as needed
+			 * to match the next codeword's length.
+			 *
+			 * Since the codeword is bit-reversed, appending 0's is
+			 * a no-op.  However, incrementing it is nontrivial.  To
+			 * do so efficiently, use the 'bsr' instruction to find
+			 * the last (highest order) 0 bit in the codeword, set
+			 * it, and clear any later (higher order) 1 bits.  But
+			 * 'bsr' actually finds the highest order 1 bit, so to
+			 * use it first flip all bits in the codeword by XOR'ing
+			 * it with (1U << len) - 1 == cur_table_end - 1.
+			 */
+			bit = 1U << bsr32(codeword ^ (cur_table_end - 1));
+			codeword &= bit - 1;
+			codeword |= bit;
+		} while (--count);
+
+		/* Advance to the next codeword length. */
+		do {
+			if (++len <= table_bits) {
+				memcpy(&decode_table[cur_table_end],
+				       decode_table,
+				       cur_table_end * sizeof(decode_table[0]));
+				cur_table_end <<= 1;
+			}
+		} while ((count = len_counts[len]) == 0);
+	}
+
+	/* Process codewords with len > table_bits.  These require subtables. */
+	cur_table_end = 1U << table_bits;
+	subtable_prefix = -1;
+	subtable_start = 0;
+	for (;;) {
+		u32 entry;
+		unsigned i;
+		unsigned stride;
 		unsigned bit;
 
-		/* Get the next symbol.  */
-		sym = sorted_syms[sym_idx];
-
-		/* Start a new subtable if the codeword is long enough to
-		 * require a subtable, *and* the first 'table_bits' bits of the
-		 * codeword don't match the prefix for the previous subtable if
-		 * any.  */
-		if (codeword_len > table_bits &&
-		    (codeword_reversed & table_mask) != cur_codeword_prefix) {
-
-			cur_codeword_prefix = (codeword_reversed & table_mask);
-
-			cur_table_start += 1U << cur_table_bits;
-
-			/* Calculate the subtable length.  If the codeword
-			 * length exceeds 'table_bits' by n, the subtable needs
-			 * at least 2**n entries.  But it may need more; if
-			 * there are fewer than 2**n codewords of length
-			 * 'table_bits + n' remaining, then n will need to be
-			 * incremented to bring in longer codewords until the
-			 * subtable can be filled completely.  Note that it
-			 * always will, eventually, be possible to fill the
-			 * subtable, since the only case where we may have an
-			 * incomplete code is a single codeword of length 1,
-			 * and that never requires any subtables.  */
-			cur_table_bits = codeword_len - table_bits;
-			remainder = (s32)1 << cur_table_bits;
-			for (;;) {
-				remainder -= len_counts[table_bits +
-							cur_table_bits];
-				if (remainder <= 0)
-					break;
-				cur_table_bits++;
-				remainder <<= 1;
+		/*
+		 * Start a new subtable if the first 'table_bits' bits of the
+		 * codeword don't match the prefix of the current subtable.
+		 */
+		if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) {
+			subtable_prefix = (codeword & ((1U << table_bits) - 1));
+			subtable_start = cur_table_end;
+			/*
+			 * Calculate the subtable length.  If the codeword has
+			 * length 'table_bits + n', then the subtable needs
+			 * '2^n' entries.  But it may need more; if fewer than
+			 * '2^n' codewords of length 'table_bits + n' remain,
+			 * then the length will need to be incremented to bring
+			 * in longer codewords until the subtable can be
+			 * completely filled.  Note that because the Huffman
+			 * code is complete, it will always be possible to fill
+			 * the subtable eventually.
+			 */
+			subtable_bits = len - table_bits;
+			codespace_used = count;
+			while (codespace_used < (1U << subtable_bits)) {
+				subtable_bits++;
+				codespace_used = (codespace_used << 1) +
+					len_counts[table_bits + subtable_bits];
 			}
+			cur_table_end = subtable_start + (1U << subtable_bits);
 
-			/* Create the entry that points from the main table to
+			/*
+			 * Create the entry that points from the main table to
 			 * the subtable.  This entry contains the index of the
 			 * start of the subtable and the number of bits with
 			 * which the subtable is indexed (the log base 2 of the
-			 * number of entries it contains).  */
-			decode_table[cur_codeword_prefix] =
+			 * number of entries it contains).
+			 */
+			decode_table[subtable_prefix] =
 				HUFFDEC_SUBTABLE_POINTER |
-				make_decode_table_entry(cur_table_start,
-							cur_table_bits);
-
-			/* Now that we're filling a subtable, we need to drop
-			 * the first 'table_bits' bits of the codewords.  */
-			num_dropped_bits = table_bits;
+				HUFFDEC_RESULT_ENTRY(subtable_start) |
+				subtable_bits;
 		}
 
-		/* Create the decode table entry, which packs the decode result
-		 * and the codeword length (minus 'table_bits' for subtables)
-		 * together.  */
-		entry = make_decode_table_entry(decode_results[sym],
-						codeword_len - num_dropped_bits);
-
-		/* Fill in as many copies of the decode table entry as are
-		 * needed.  The number of entries to fill is a power of 2 and
-		 * depends on the codeword length; it could be as few as 1 or as
-		 * large as half the size of the table.  Since the codewords are
-		 * bit-reversed, the indices to fill are those with the codeword
-		 * in its low bits; it's the high bits that vary.  */
-		i = cur_table_start + (codeword_reversed >> num_dropped_bits);
-		end = cur_table_start + (1U << cur_table_bits);
-		increment = 1U << (codeword_len - num_dropped_bits);
+		/* Fill the subtable entries for the current codeword. */
+		entry = decode_results[*sorted_syms++] | (len - table_bits);
+		i = subtable_start + (codeword >> table_bits);
+		stride = 1U << (len - table_bits);
 		do {
 			decode_table[i] = entry;
-			i += increment;
-		} while (i < end);
+			i += stride;
+		} while (i < cur_table_end);
 
-		/* Advance to the next codeword by incrementing it.  But since
-		 * our codewords are bit-reversed, we must manipulate the bits
-		 * ourselves rather than simply adding 1.  */
-		bit = 1U << (codeword_len - 1);
-		while (codeword_reversed & bit)
-			bit >>= 1;
-		codeword_reversed &= bit - 1;
-		codeword_reversed |= bit;
-
-		/* Advance to the next symbol.  This will either increase the
-		 * codeword length, or keep the same codeword length but
-		 * increase the symbol value.  Note: since we are using
-		 * bit-reversed codewords, we don't need to explicitly append
-		 * zeroes to the codeword when the codeword length increases. */
-		if (++sym_idx == num_syms)
+		/* Advance to the next codeword. */
+		if (codeword == (1U << len) - 1) /* last codeword (all 1's)? */
 			return true;
-		len_counts[codeword_len]--;
-		while (len_counts[codeword_len] == 0)
-			codeword_len++;
+		bit = 1U << bsr32(codeword ^ ((1U << len) - 1));
+		codeword &= bit - 1;
+		codeword |= bit;
+		count--;
+		while (count == 0)
+			count = len_counts[++len];
 	}
 }
 
 /* Build the decode table for the precode.  */
 static bool
-build_precode_decode_table(struct deflate_decompressor *d)
+build_precode_decode_table(struct libdeflate_decompressor *d)
 {
 	/* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
 	STATIC_ASSERT(PRECODE_TABLEBITS == 7 && PRECODE_ENOUGH == 128);
@@ -723,12 +827,12 @@ build_precode_decode_table(struct deflate_decompressor *d)
 				  precode_decode_results,
 				  PRECODE_TABLEBITS,
 				  DEFLATE_MAX_PRE_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 /* Build the decode table for the literal/length code.  */
 static bool
-build_litlen_decode_table(struct deflate_decompressor *d,
+build_litlen_decode_table(struct libdeflate_decompressor *d,
 			  unsigned num_litlen_syms, unsigned num_offset_syms)
 {
 	/* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
@@ -740,12 +844,12 @@ build_litlen_decode_table(struct deflate_decompressor *d,
 				  litlen_decode_results,
 				  LITLEN_TABLEBITS,
 				  DEFLATE_MAX_LITLEN_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 /* Build the decode table for the offset code.  */
 static bool
-build_offset_decode_table(struct deflate_decompressor *d,
+build_offset_decode_table(struct libdeflate_decompressor *d,
 			  unsigned num_litlen_syms, unsigned num_offset_syms)
 {
 	/* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
@@ -757,7 +861,7 @@ build_offset_decode_table(struct deflate_decompressor *d,
 				  offset_decode_results,
 				  OFFSET_TABLEBITS,
 				  DEFLATE_MAX_OFFSET_CODEWORD_LEN,
-				  d->working_space);
+				  d->sorted_syms);
 }
 
 static forceinline machine_word_t
@@ -784,90 +888,113 @@ copy_word_unaligned(const void *src, void *dst)
  *                         Main decompression routine
  *****************************************************************************/
 
-#define FUNCNAME deflate_decompress_default
-#define ATTRIBUTES
-#include "decompress_impl.h"
-#undef FUNCNAME
-#undef ATTRIBUTES
+typedef enum libdeflate_result (*decompress_func_t)
+	(struct libdeflate_decompressor * restrict d,
+	 const void * restrict in, size_t in_nbytes,
+	 void * restrict out, size_t out_nbytes_avail,
+	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
 
-#if X86_CPU_FEATURES_ENABLED && \
-	COMPILER_SUPPORTS_BMI2_TARGET && !defined(__BMI2__)
-#  define FUNCNAME deflate_decompress_bmi2
-#  define ATTRIBUTES __attribute__((target("bmi2")))
-#  include "decompress_impl.h"
-#  undef FUNCNAME
-#  undef ATTRIBUTES
-#  define DISPATCH_ENABLED 1
-#else
-#  define DISPATCH_ENABLED 0
+#undef DEFAULT_IMPL
+#undef DISPATCH
+#if defined(__i386__) || defined(__x86_64__)
+#  include "x86/decompress_impl.h"
 #endif
 
-#if DISPATCH_ENABLED
+#ifndef DEFAULT_IMPL
+#  define FUNCNAME deflate_decompress_default
+#  define ATTRIBUTES
+#  include "decompress_template.h"
+#  define DEFAULT_IMPL deflate_decompress_default
+#endif
 
-static enum decompress_result
-dispatch(struct deflate_decompressor * restrict d,
+#ifdef DISPATCH
+static enum libdeflate_result
+dispatch(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
 	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_out_nbytes_ret);
+	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
 
-typedef enum decompress_result (*decompress_func_t)
-	(struct deflate_decompressor * restrict d,
+static volatile decompress_func_t decompress_impl = dispatch;
+
+/* Choose the fastest implementation at runtime */
+static enum libdeflate_result
+dispatch(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
 	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_out_nbytes_ret);
-
-static decompress_func_t decompress_impl = dispatch;
-
-static enum decompress_result
-dispatch(struct deflate_decompressor * restrict d,
-	 const void * restrict in, size_t in_nbytes,
-	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_out_nbytes_ret)
+	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
 {
-	decompress_func_t f = deflate_decompress_default;
-#if X86_CPU_FEATURES_ENABLED
-	if (x86_have_cpu_feature(X86_CPU_FEATURE_BMI2))
-		f = deflate_decompress_bmi2;
-#endif
+	decompress_func_t f = arch_select_decompress_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
 	decompress_impl = f;
 	return (*f)(d, in, in_nbytes, out, out_nbytes_avail,
-		    actual_out_nbytes_ret);
+		    actual_in_nbytes_ret, actual_out_nbytes_ret);
 }
-#endif /* DISPATCH_ENABLED */
+#else
+#  define decompress_impl DEFAULT_IMPL /* only one implementation, use it */
+#endif
 
 
 /*
  * This is the main DEFLATE decompression routine.  See libdeflate.h for the
  * documentation.
  *
- * Note that the real code is in decompress_impl.h.  The part here just handles
- * calling the appropriate implementation depending on the CPU features at
- * runtime.
+ * Note that the real code is in decompress_template.h.  The part here just
+ * handles calling the appropriate implementation depending on the CPU features
+ * at runtime.
  */
-LIBEXPORT enum decompress_result
-deflate_decompress(struct deflate_decompressor * restrict d,
-		   const void * restrict in, size_t in_nbytes,
-		   void * restrict out, size_t out_nbytes_avail,
-		   size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor * restrict d,
+				 const void * restrict in, size_t in_nbytes,
+				 void * restrict out, size_t out_nbytes_avail,
+				 size_t *actual_in_nbytes_ret,
+				 size_t *actual_out_nbytes_ret)
 {
-#if DISPATCH_ENABLED
-	return (*decompress_impl)(d, in, in_nbytes, out, out_nbytes_avail,
-				  actual_out_nbytes_ret);
-#else
-	return deflate_decompress_default(d, in, in_nbytes, out,
-					  out_nbytes_avail,
-					  actual_out_nbytes_ret);
-#endif
+	return decompress_impl(d, in, in_nbytes, out, out_nbytes_avail,
+			       actual_in_nbytes_ret, actual_out_nbytes_ret);
 }
 
-LIBEXPORT struct deflate_decompressor *
-deflate_alloc_decompressor(void)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress(struct libdeflate_decompressor * restrict d,
+			      const void * restrict in, size_t in_nbytes,
+			      void * restrict out, size_t out_nbytes_avail,
+			      size_t *actual_out_nbytes_ret)
 {
-	return malloc(sizeof(struct deflate_decompressor));
+	return libdeflate_deflate_decompress_ex(d, in, in_nbytes,
+						out, out_nbytes_avail,
+						NULL, actual_out_nbytes_ret);
 }
 
-LIBEXPORT void
-deflate_free_decompressor(struct deflate_decompressor *d)
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
+libdeflate_alloc_decompressor(void)
 {
-	free(d);
+	/*
+	 * Note that only certain parts of the decompressor actually must be
+	 * initialized here:
+	 *
+	 * - 'static_codes_loaded' must be initialized to false.
+	 *
+	 * - The first half of the main portion of each decode table must be
+	 *   initialized to any value, to avoid reading from uninitialized
+	 *   memory during table expansion in build_decode_table().  (Although,
+	 *   this is really just to avoid warnings with dynamic tools like
+	 *   valgrind, since build_decode_table() is guaranteed to initialize
+	 *   all entries eventually anyway.)
+	 *
+	 * But for simplicity, we currently just zero the whole decompressor.
+	 */
+	struct libdeflate_decompressor *d = libdeflate_malloc(sizeof(*d));
+
+	if (d == NULL)
+		return NULL;
+	memset(d, 0, sizeof(*d));
+	return d;
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_decompressor(struct libdeflate_decompressor *d)
+{
+	libdeflate_free(d);
 }

lib/gzip_compress.c

@@ -1,27 +1,42 @@
 /*
  * gzip_compress.c - compress with a gzip wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "crc32.h"
 #include "deflate_compress.h"
 #include "gzip_constants.h"
 #include "unaligned.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT size_t
-gzip_compress(struct deflate_compressor *c, const void *in, size_t in_size,
-	      void *out, size_t out_nbytes_avail)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress(struct libdeflate_compressor *c,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
 	unsigned compression_level;
@@ -46,33 +61,35 @@ gzip_compress(struct deflate_compressor *c, const void *in, size_t in_size,
 	xfl = 0;
 	compression_level = deflate_get_compression_level(c);
 	if (compression_level < 2)
-		xfl |= GZIP_XFL_FASTEST_COMRESSION;
+		xfl |= GZIP_XFL_FASTEST_COMPRESSION;
 	else if (compression_level >= 8)
-		xfl |= GZIP_XFL_SLOWEST_COMRESSION;
+		xfl |= GZIP_XFL_SLOWEST_COMPRESSION;
 	*out_next++ = xfl;
 	/* OS */
 	*out_next++ = GZIP_OS_UNKNOWN;	/* OS  */
 
 	/* Compressed data  */
-	deflate_size = deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - GZIP_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* CRC32 */
-	put_unaligned_le32(crc32_gzip(in, in_size), out_next);
+	put_unaligned_le32(libdeflate_crc32(0, in, in_nbytes), out_next);
 	out_next += 4;
 
 	/* ISIZE */
-	put_unaligned_le32((u32)in_size, out_next);
+	put_unaligned_le32((u32)in_nbytes, out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBEXPORT size_t
-gzip_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *c,
+			       size_t in_nbytes)
 {
-	return GZIP_MIN_OVERHEAD + deflate_compress_bound(c, in_nbytes);
+	return GZIP_MIN_OVERHEAD +
+	       libdeflate_deflate_compress_bound(c, in_nbytes);
 }

lib/gzip_constants.h

@@ -2,8 +2,8 @@
  * gzip_constants.h - constants for the gzip wrapper format
  */
 
-#ifndef _LIB_GZIP_CONSTANTS_H
-#define _LIB_GZIP_CONSTANTS_H
+#ifndef LIB_GZIP_CONSTANTS_H
+#define LIB_GZIP_CONSTANTS_H
 
 #define GZIP_MIN_HEADER_SIZE	10
 #define GZIP_FOOTER_SIZE	8
@@ -23,8 +23,8 @@
 
 #define GZIP_MTIME_UNAVAILABLE	0
 
-#define GZIP_XFL_SLOWEST_COMRESSION	0x02
-#define GZIP_XFL_FASTEST_COMRESSION	0x04
+#define GZIP_XFL_SLOWEST_COMPRESSION	0x02
+#define GZIP_XFL_FASTEST_COMPRESSION	0x04
 
 #define GZIP_OS_FAT		0
 #define GZIP_OS_AMIGA		1
@@ -42,4 +42,4 @@
 #define GZIP_OS_RISCOS		13
 #define GZIP_OS_UNKNOWN		255
 
-#endif /* _LIB_GZIP_CONSTANTS_H */
+#endif /* LIB_GZIP_CONSTANTS_H */

lib/gzip_decompress.c

@@ -1,47 +1,63 @@
 /*
  * gzip_decompress.c - decompress with a gzip wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "crc32.h"
 #include "gzip_constants.h"
 #include "unaligned.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT enum decompress_result
-gzip_decompress(struct deflate_decompressor *d,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret)
 {
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
 	u8 flg;
+	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
-	enum decompress_result result;
+	enum libdeflate_result result;
 
 	if (in_nbytes < GZIP_MIN_OVERHEAD)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* ID1 */
 	if (*in_next++ != GZIP_ID1)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	/* ID2 */
 	if (*in_next++ != GZIP_ID2)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	/* CM */
 	if (*in_next++ != GZIP_CM_DEFLATE)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 	flg = *in_next++;
 	/* MTIME */
 	in_next += 4;
@@ -51,7 +67,7 @@ gzip_decompress(struct deflate_decompressor *d,
 	in_next += 1;
 
 	if (flg & GZIP_FRESERVED)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* Extra field */
 	if (flg & GZIP_FEXTRA) {
@@ -59,7 +75,7 @@ gzip_decompress(struct deflate_decompressor *d,
 		in_next += 2;
 
 		if (in_end - in_next < (u32)xlen + GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 
 		in_next += xlen;
 	}
@@ -69,7 +85,7 @@ gzip_decompress(struct deflate_decompressor *d,
 		while (*in_next++ != 0 && in_next != in_end)
 			;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* File comment (zero terminated) */
@@ -77,22 +93,23 @@ gzip_decompress(struct deflate_decompressor *d,
 		while (*in_next++ != 0 && in_next != in_end)
 			;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* CRC16 for gzip header */
 	if (flg & GZIP_FHCRC) {
 		in_next += 2;
 		if (in_end - in_next < GZIP_FOOTER_SIZE)
-			return DECOMPRESS_BAD_DATA;
+			return LIBDEFLATE_BAD_DATA;
 	}
 
 	/* Compressed data  */
-	result = deflate_decompress(d, in_next,
-				    in_end - GZIP_FOOTER_SIZE - in_next,
-				    out, out_nbytes_avail,
-				    actual_out_nbytes_ret);
-	if (result != DECOMPRESS_SUCCESS)
+	result = libdeflate_deflate_decompress_ex(d, in_next,
+					in_end - GZIP_FOOTER_SIZE - in_next,
+					out, out_nbytes_avail,
+					&actual_in_nbytes,
+					actual_out_nbytes_ret);
+	if (result != LIBDEFLATE_SUCCESS)
 		return result;
 
 	if (actual_out_nbytes_ret)
@@ -100,16 +117,32 @@ gzip_decompress(struct deflate_decompressor *d,
 	else
 		actual_out_nbytes = out_nbytes_avail;
 
-	in_next = in_end - GZIP_FOOTER_SIZE;
+	in_next += actual_in_nbytes;
 
 	/* CRC32 */
-	if (crc32_gzip(out, actual_out_nbytes) != get_unaligned_le32(in_next))
-		return DECOMPRESS_BAD_DATA;
+	if (libdeflate_crc32(0, out, actual_out_nbytes) !=
+	    get_unaligned_le32(in_next))
+		return LIBDEFLATE_BAD_DATA;
 	in_next += 4;
 
 	/* ISIZE */
 	if ((u32)actual_out_nbytes != get_unaligned_le32(in_next))
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
+	in_next += 4;
 
-	return DECOMPRESS_SUCCESS;
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	return LIBDEFLATE_SUCCESS;
+}
+
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress(struct libdeflate_decompressor *d,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret)
+{
+	return libdeflate_gzip_decompress_ex(d, in, in_nbytes,
+					     out, out_nbytes_avail,
+					     NULL, actual_out_nbytes_ret);
 }

lib/hc_matchfinder.h

@@ -1,15 +1,30 @@
 /*
  * hc_matchfinder.h - Lempel-Ziv matchfinding with a hash table of linked lists
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
  * ---------------------------------------------------------------------------
  *
@@ -91,14 +106,17 @@
  * ----------------------------------------------------------------------------
  */
 
+#ifndef LIB_HC_MATCHFINDER_H
+#define LIB_HC_MATCHFINDER_H
+
 #include "matchfinder_common.h"
 
 #define HC_MATCHFINDER_HASH3_ORDER	15
 #define HC_MATCHFINDER_HASH4_ORDER	16
 
-#define HC_MATCHFINDER_TOTAL_HASH_LENGTH		\
-	((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
-	 (1UL << HC_MATCHFINDER_HASH4_ORDER))
+#define HC_MATCHFINDER_TOTAL_HASH_SIZE			\
+	(((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
+	  (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
 struct hc_matchfinder {
 
@@ -115,7 +133,7 @@ struct hc_matchfinder {
 
 }
 #ifdef _aligned_attribute
-  _aligned_attribute(MATCHFINDER_ALIGNMENT)
+  _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
 #endif
 ;
 
@@ -123,14 +141,18 @@ struct hc_matchfinder {
 static forceinline void
 hc_matchfinder_init(struct hc_matchfinder *mf)
 {
-	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_LENGTH);
+	STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE %
+		      MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE);
 }
 
 static forceinline void
 hc_matchfinder_slide_window(struct hc_matchfinder *mf)
 {
-	matchfinder_rebase((mf_pos_t *)mf,
-			   sizeof(struct hc_matchfinder) / sizeof(mf_pos_t));
+	STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+	matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
 }
 
 /*
@@ -179,7 +201,7 @@ hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf,
 	const u8 *best_matchptr = in_next;
 	mf_pos_t cur_node3, cur_node4;
 	u32 hash3, hash4;
-	u32 next_seq3, next_seq4;
+	u32 next_hashseq;
 	u32 seq4;
 	const u8 *matchptr;
 	u32 len;
@@ -217,10 +239,9 @@ hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf,
 	mf->next_tab[cur_pos] = cur_node4;
 
 	/* Compute the next hash codes.  */
-	next_seq4 = load_u32_unaligned(in_next + 1);
-	next_seq3 = loaded_u32_to_u24(next_seq4);
-	next_hashes[0] = lz_hash(next_seq3, HC_MATCHFINDER_HASH3_ORDER);
-	next_hashes[1] = lz_hash(next_seq4, HC_MATCHFINDER_HASH4_ORDER);
+	next_hashseq = get_unaligned_le32(in_next + 1);
+	next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+	next_hashes[1] = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
 	prefetchw(&mf->hash3_tab[next_hashes[0]]);
 	prefetchw(&mf->hash4_tab[next_hashes[1]]);
 
@@ -355,7 +376,7 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 {
 	u32 cur_pos;
 	u32 hash3, hash4;
-	u32 next_seq3, next_seq4;
+	u32 next_hashseq;
 	u32 remaining = count;
 
 	if (unlikely(count + 5 > in_end - in_next))
@@ -374,10 +395,9 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 		mf->next_tab[cur_pos] = mf->hash4_tab[hash4];
 		mf->hash4_tab[hash4] = cur_pos;
 
-		next_seq4 = load_u32_unaligned(++in_next);
-		next_seq3 = loaded_u32_to_u24(next_seq4);
-		hash3 = lz_hash(next_seq3, HC_MATCHFINDER_HASH3_ORDER);
-		hash4 = lz_hash(next_seq4, HC_MATCHFINDER_HASH4_ORDER);
+		next_hashseq = get_unaligned_le32(++in_next);
+		hash3 = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+		hash4 = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
 		cur_pos++;
 	} while (--remaining);
 
@@ -388,3 +408,5 @@ hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
 
 	return in_next;
 }
+
+#endif /* LIB_HC_MATCHFINDER_H */

lib/lib_common.h

@@ -0,0 +1,67 @@
+/*
+ * lib_common.h - internal header included by all library code
+ */
+
+#ifndef LIB_LIB_COMMON_H
+#define LIB_LIB_COMMON_H
+
+#ifdef LIBDEFLATE_H
+#  error "lib_common.h must always be included before libdeflate.h"
+   /* because BUILDING_LIBDEFLATE must be set first */
+#endif
+
+#define BUILDING_LIBDEFLATE
+
+#include "../common/common_defs.h"
+
+/*
+ * Prefix with "_libdeflate_" all global symbols which are not part of the API
+ * and don't already have a "libdeflate" prefix.  This avoids exposing overly
+ * generic names when libdeflate is built as a static library.
+ *
+ * Note that the chosen prefix is not really important and can be changed
+ * without breaking library users.  It was just chosen so that the resulting
+ * symbol names are unlikely to conflict with those from any other software.
+ * Also note that this fixup has no useful effect when libdeflate is built as a
+ * shared library, since these symbols are not exported.
+ */
+#define SYM_FIXUP(sym)			_libdeflate_##sym
+#define deflate_get_compression_level	SYM_FIXUP(deflate_get_compression_level)
+#define _cpu_features			SYM_FIXUP(_cpu_features)
+#define setup_cpu_features		SYM_FIXUP(setup_cpu_features)
+
+void *libdeflate_malloc(size_t size);
+void libdeflate_free(void *ptr);
+
+void *libdeflate_aligned_malloc(size_t alignment, size_t size);
+void libdeflate_aligned_free(void *ptr);
+
+#ifdef FREESTANDING
+/*
+ * With -ffreestanding, <string.h> may be missing, and we must provide
+ * implementations of memset(), memcpy(), memmove(), and memcmp().
+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+ *
+ * Also, -ffreestanding disables interpreting calls to these functions as
+ * built-ins.  E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
+ * not be optimized to a single load instruction.  For performance reasons we
+ * don't want that.  So, declare these functions as macros that expand to the
+ * corresponding built-ins.  This approach is recommended in the gcc man page.
+ * We still need the actual function definitions in case gcc calls them.
+ */
+void *memset(void *s, int c, size_t n);
+#define memset(s, c, n)		__builtin_memset((s), (c), (n))
+
+void *memcpy(void *dest, const void *src, size_t n);
+#define memcpy(dest, src, n)	__builtin_memcpy((dest), (src), (n))
+
+void *memmove(void *dest, const void *src, size_t n);
+#define memmove(dest, src, n)	__builtin_memmove((dest), (src), (n))
+
+int memcmp(const void *s1, const void *s2, size_t n);
+#define memcmp(s1, s2, n)	__builtin_memcmp((s1), (s2), (n))
+#else
+#include <string.h>
+#endif
+
+#endif /* LIB_LIB_COMMON_H */

lib/matchfinder_avx2.h

@@ -1,53 +0,0 @@
-/*
- * matchfinder_avx2.h - matchfinding routines optimized for Intel AVX2 (Advanced
- * Vector Extensions)
- */
-
-#include <immintrin.h>
-
-static forceinline bool
-matchfinder_init_avx2(mf_pos_t *data, size_t size)
-{
-	__m256i v, *p;
-	size_t n;
-
-	if (size % sizeof(__m256i) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
-{
-	__m256i v, *p;
-	size_t n;
-
-	if ((size % sizeof(__m256i) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m256i *)data;
-	n = size / (sizeof(__m256i) * 4);
-	do {
-		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
-		p[0] = _mm256_adds_epi16(p[0], v);
-		p[1] = _mm256_adds_epi16(p[1], v);
-		p[2] = _mm256_adds_epi16(p[2], v);
-		p[3] = _mm256_adds_epi16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/matchfinder_common.h

@@ -2,10 +2,10 @@
  * matchfinder_common.h - common code for Lempel-Ziv matchfinding
  */
 
-#ifndef _LIB_MATCHFINDER_COMMON_H
-#define _LIB_MATCHFINDER_COMMON_H
+#ifndef LIB_MATCHFINDER_COMMON_H
+#define LIB_MATCHFINDER_COMMON_H
 
-#include "common_defs.h"
+#include "lib_common.h"
 #include "unaligned.h"
 
 #ifndef MATCHFINDER_WINDOW_ORDER
@@ -18,29 +18,20 @@ typedef s16 mf_pos_t;
 
 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
 
-#define MATCHFINDER_ALIGNMENT 8
+/*
+ * Required alignment of the matchfinder buffer pointer and size.  The values
+ * here come from the AVX-2 implementation, which is the worst case.
+ */
+#define MATCHFINDER_MEM_ALIGNMENT	32
+#define MATCHFINDER_SIZE_ALIGNMENT	128
 
-#ifdef __AVX2__
-#  include "matchfinder_avx2.h"
-#  if MATCHFINDER_ALIGNMENT < 32
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 32
-#  endif
-#endif
-
-#ifdef __SSE2__
-#  include "matchfinder_sse2.h"
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
-#  endif
-#endif
-
-#ifdef __ARM_NEON__
-#  include "matchfinder_neon.h"
-#  if MATCHFINDER_ALIGNMENT < 16
-#    undef MATCHFINDER_ALIGNMENT
-#    define MATCHFINDER_ALIGNMENT 16
+#undef matchfinder_init
+#undef matchfinder_rebase
+#ifdef _aligned_attribute
+#  if defined(__arm__) || defined(__aarch64__)
+#    include "arm/matchfinder_impl.h"
+#  elif defined(__i386__) || defined(__x86_64__)
+#    include "x86/matchfinder_impl.h"
 #  endif
 #endif
 
@@ -49,31 +40,20 @@ typedef s16 mf_pos_t;
  *
  * Essentially, this is an optimized memset().
  *
- * 'data' must be aligned to a MATCHFINDER_ALIGNMENT boundary.
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_init
 static forceinline void
-matchfinder_init(mf_pos_t *data, size_t num_entries)
+matchfinder_init(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-#if defined(__AVX2__) && defined(_aligned_attribute)
-	if (matchfinder_init_avx2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__SSE2__) && defined(_aligned_attribute)
-	if (matchfinder_init_sse2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__ARM_NEON__) && defined(_aligned_attribute)
-	if (matchfinder_init_neon(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
 	for (i = 0; i < num_entries; i++)
 		data[i] = MATCHFINDER_INITVAL;
 }
+#endif
 
 /*
  * Slide the matchfinder by WINDOW_SIZE bytes.
@@ -95,27 +75,17 @@ matchfinder_init(mf_pos_t *data, size_t num_entries)
  * hash bucket.  Note that in the latter table, the links are 1-ary in the case
  * of "hash chains", and 2-ary in the case of "binary trees".  In either case,
  * the links need to be rebased in the same way.
+ *
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
+#ifndef matchfinder_rebase
 static forceinline void
-matchfinder_rebase(mf_pos_t *data, size_t num_entries)
+matchfinder_rebase(mf_pos_t *data, size_t size)
 {
+	size_t num_entries = size / sizeof(*data);
 	size_t i;
 
-#if defined(__AVX2__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_avx2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__SSE2__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_sse2(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
-#if defined(__ARM_NEON__) && defined(_aligned_attribute)
-	if (matchfinder_rebase_neon(data, num_entries * sizeof(data[0])))
-		return;
-#endif
-
 	if (MATCHFINDER_WINDOW_SIZE == 32768) {
 		/* Branchless version for 32768 byte windows.  If the value was
 		 * already negative, clear all bits except the sign bit; this
@@ -138,6 +108,7 @@ matchfinder_rebase(mf_pos_t *data, size_t num_entries)
 			data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
 	}
 }
+#endif
 
 /*
  * The hash function: given a sequence prefix held in the low-order bits of a
@@ -202,4 +173,4 @@ word_differs:
 	return len;
 }
 
-#endif /* _LIB_MATCHFINDER_COMMON_H */
+#endif /* LIB_MATCHFINDER_COMMON_H */

lib/matchfinder_neon.h

@@ -1,61 +0,0 @@
-/*
- * matchfinder_neon.h - matchfinding routines optimized for ARM NEON (Advanced
- * SIMD) instructions
- */
-
-#include <arm_neon.h>
-
-static forceinline bool
-matchfinder_init_neon(mf_pos_t *data, size_t size)
-{
-	int16x8_t v, *p;
-	size_t n;
-
-	if (size % sizeof(int16x8_t) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_neon(mf_pos_t *data, size_t size)
-{
-	int16x8_t v, *p;
-	size_t n;
-
-	if ((size % sizeof(int16x8_t) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = (int16x8_t) {
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-	};
-	p = (int16x8_t *)data;
-	n = size / (sizeof(int16x8_t) * 4);
-	do {
-		p[0] = vqaddq_s16(p[0], v);
-		p[1] = vqaddq_s16(p[1], v);
-		p[2] = vqaddq_s16(p[2], v);
-		p[3] = vqaddq_s16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/matchfinder_sse2.h

@@ -1,53 +0,0 @@
-/*
- * matchfinder_sse2.h - matchfinding routines optimized for Intel SSE2
- * (Streaming SIMD Extensions).
- */
-
-#include <emmintrin.h>
-
-static forceinline bool
-matchfinder_init_sse2(mf_pos_t *data, size_t size)
-{
-	__m128i v, *p;
-	size_t n;
-
-	if (size % sizeof(__m128i) * 4)
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16(MATCHFINDER_INITVAL);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
-	do {
-		p[0] = v;
-		p[1] = v;
-		p[2] = v;
-		p[3] = v;
-		p += 4;
-	} while (--n);
-	return true;
-}
-
-static forceinline bool
-matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
-{
-	__m128i v, *p;
-	size_t n;
-
-	if ((size % sizeof(__m128i) * 4 != 0))
-		return false;
-
-	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
-	v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
-	p = (__m128i *)data;
-	n = size / (sizeof(__m128i) * 4);
-	do {
-		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
-		p[0] = _mm_adds_epi16(p[0], v);
-		p[1] = _mm_adds_epi16(p[1], v);
-		p[2] = _mm_adds_epi16(p[2], v);
-		p[3] = _mm_adds_epi16(p[3], v);
-		p += 4;
-	} while (--n);
-	return true;
-}

lib/unaligned.h

@@ -5,15 +5,41 @@
 #ifndef LIB_UNALIGNED_H
 #define LIB_UNALIGNED_H
 
-#include "common_defs.h"
+#include "lib_common.h"
+
+/***** Unaligned loads and stores without endianness conversion *****/
 
 /*
- * Naming note:
+ * memcpy() is portable, and it usually gets optimized appropriately by modern
+ * compilers.  I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled
+ * to a load or store instruction, not to an actual function call.
  *
- * {load,store}_*_unaligned() deal with raw bytes without endianness conversion.
- * {get,put}_unaligned_*() deal with a specific endianness.
+ * We no longer use the "packed struct" approach, as that is nonstandard, has
+ * unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366).  However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
  */
 
+#define DEFINE_UNALIGNED_TYPE(type)				\
+static forceinline type						\
+load_##type##_unaligned(const void *p)				\
+{								\
+	type v;							\
+	memcpy(&v, p, sizeof(v));				\
+	return v;						\
+}								\
+								\
+static forceinline void						\
+store_##type##_unaligned(type v, void *p)			\
+{								\
+	memcpy(p, &v, sizeof(v));				\
+}
+
 DEFINE_UNALIGNED_TYPE(u16)
 DEFINE_UNALIGNED_TYPE(u32)
 DEFINE_UNALIGNED_TYPE(u64)
@@ -22,7 +48,7 @@ DEFINE_UNALIGNED_TYPE(machine_word_t)
 #define load_word_unaligned	load_machine_word_t_unaligned
 #define store_word_unaligned	store_machine_word_t_unaligned
 
-/***** Unaligned loads  *****/
+/***** Unaligned loads with endianness conversion *****/
 
 static forceinline u16
 get_unaligned_le16(const u8 *p)
@@ -84,7 +110,7 @@ get_unaligned_leword(const u8 *p)
 		return get_unaligned_le64(p);
 }
 
-/***** Unaligned stores  *****/
+/***** Unaligned stores with endianness conversion *****/
 
 static forceinline void
 put_unaligned_le16(u16 v, u8 *p)

lib/utils.c

@@ -0,0 +1,142 @@
+/*
+ * utils.c - utility functions for libdeflate
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+#include "libdeflate.h"
+
+#ifdef FREESTANDING
+#  define malloc NULL
+#  define free NULL
+#else
+#  include <stdlib.h>
+#endif
+
+static void *(*libdeflate_malloc_func)(size_t) = malloc;
+static void (*libdeflate_free_func)(void *) = free;
+
+void *
+libdeflate_malloc(size_t size)
+{
+	return (*libdeflate_malloc_func)(size);
+}
+
+void
+libdeflate_free(void *ptr)
+{
+	(*libdeflate_free_func)(ptr);
+}
+
+void *
+libdeflate_aligned_malloc(size_t alignment, size_t size)
+{
+	void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size);
+	if (ptr) {
+		void *orig_ptr = ptr;
+		ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
+		((void **)ptr)[-1] = orig_ptr;
+	}
+	return ptr;
+}
+
+void
+libdeflate_aligned_free(void *ptr)
+{
+	if (ptr)
+		libdeflate_free(((void **)ptr)[-1]);
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *))
+{
+	libdeflate_malloc_func = malloc_func;
+	libdeflate_free_func = free_func;
+}
+
+/*
+ * Implementations of libc functions for freestanding library builds.
+ * Normal library builds don't use these.  Not optimized yet; usually the
+ * compiler expands these functions and doesn't actually call them anyway.
+ */
+#ifdef FREESTANDING
+#undef memset
+void * __attribute__((weak))
+memset(void *s, int c, size_t n)
+{
+	u8 *p = s;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		p[i] = c;
+	return s;
+}
+
+#undef memcpy
+void * __attribute__((weak))
+memcpy(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	for (i = 0; i < n; i++)
+		d[i] = s[i];
+	return dest;
+}
+
+#undef memmove
+void * __attribute__((weak))
+memmove(void *dest, const void *src, size_t n)
+{
+	u8 *d = dest;
+	const u8 *s = src;
+	size_t i;
+
+	if (d <= s)
+		return memcpy(d, s, n);
+
+	for (i = n; i > 0; i--)
+		d[i - 1] = s[i - 1];
+	return dest;
+}
+
+#undef memcmp
+int __attribute__((weak))
+memcmp(const void *s1, const void *s2, size_t n)
+{
+	const u8 *p1 = s1;
+	const u8 *p2 = s2;
+	size_t i;
+
+	for (i = 0; i < n; i++) {
+		if (p1[i] != p2[i])
+			return (int)p1[i] - (int)p2[i];
+	}
+	return 0;
+}
+#endif /* FREESTANDING */

lib/x86/adler32_impl.h

@@ -0,0 +1,337 @@
+/*
+ * x86/adler32_impl.h - x86 implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_ADLER32_IMPL_H
+#define LIB_X86_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * The following macros horizontally sum the s1 counters and add them to the
+ * real s1, and likewise for s2.  They do this via a series of reductions, each
+ * of which halves the vector length, until just one counter remains.
+ *
+ * The s1 reductions don't depend on the s2 reductions and vice versa, so for
+ * efficiency they are interleaved.  Also, every other s1 counter is 0 due to
+ * the 'psadbw' instruction (_mm_sad_epu8) summing groups of 8 bytes rather than
+ * 4; hence, one of the s1 reductions is skipped when going from 128 => 32 bits.
+ */
+
+#define ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_last = (v_s1), s2_last = (v_s2);			    \
+									    \
+	/* 128 => 32 bits */						    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x31);	    \
+	s1_last += (__v4su)_mm_shuffle_epi32((__m128i)s1_last, 0x02);	    \
+	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x02);	    \
+									    \
+	*(s1) += (u32)_mm_cvtsi128_si32((__m128i)s1_last);		    \
+	*(s2) += (u32)_mm_cvtsi128_si32((__m128i)s2_last);		    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v4su s1_128bit, s2_128bit;					    \
+									    \
+	/* 256 => 128 bits */						    \
+	s1_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 1);   \
+	s2_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 0) +  \
+		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 1);   \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_128((s1), (s2), s1_128bit, s2_128bit);	    \
+}
+
+#define ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2)		    \
+{									    \
+	__v8su s1_256bit, s2_256bit;					    \
+									    \
+	/* 512 => 256 bits */						    \
+	s1_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 1);  \
+	s2_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 0) + \
+		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 1);  \
+									    \
+	ADLER32_FINISH_VEC_CHUNK_256((s1), (s2), s1_256bit, s2_256bit);	    \
+}
+
+/* AVX-512BW implementation: like the AVX2 one, but does 64 bytes at a time */
+#undef DISPATCH_AVX512BW
+#if !defined(DEFAULT_IMPL) &&						\
+    /*
+     * clang before v3.9 is missing some AVX-512BW intrinsics including
+     * _mm512_sad_epu8(), a.k.a. __builtin_ia32_psadbw512.  So just make using
+     * AVX-512BW, even when __AVX512BW__ is defined, conditional on
+     * COMPILER_SUPPORTS_AVX512BW_TARGET where we check for that builtin.
+     */									\
+    COMPILER_SUPPORTS_AVX512BW_TARGET &&				\
+    (defined(__AVX512BW__) || (X86_CPU_FEATURES_ENABLED &&		\
+			       COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_avx512bw
+#  define FUNCNAME_CHUNK	adler32_avx512bw_chunk
+#  define IMPL_ALIGNMENT	64
+#  define IMPL_SEGMENT_SIZE	64
+#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
+#  ifdef __AVX512BW__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_avx512bw
+#  else
+#    define ATTRIBUTES		__attribute__((target("avx512bw")))
+#    define DISPATCH		1
+#    define DISPATCH_AVX512BW	1
+#  endif
+#  include <immintrin.h>
+static forceinline ATTRIBUTES void
+adler32_avx512bw_chunk(const __m512i *p, const __m512i *const end,
+		       u32 *s1, u32 *s2)
+{
+	const __m512i zeroes = _mm512_setzero_si512();
+	const __v64qi multipliers = (__v64qi){
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v32hi ones = (__v32hi)_mm512_set1_epi16(1);
+	__v16si v_s1 = (__v16si)zeroes;
+	__v16si v_s1_sums = (__v16si)zeroes;
+	__v16si v_s2 = (__v16si)zeroes;
+
+	do {
+		/* Load the next 64-byte segment */
+		__m512i bytes = *p++;
+
+		/* Multiply the bytes by 64...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v32hi sums = (__v32hi)_mm512_maddubs_epi16(
+						bytes, (__m512i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 64 to the end. */
+		v_s1_sums += v_s1;
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v16si)_mm512_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 64...1 to the corresponding s2 counter */
+		v_s2 += (__v16si)_mm512_madd_epi16((__m512i)sums,
+						   (__m512i)ones);
+	} while (p != end);
+
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (64) */
+	v_s2 += (__v16si)_mm512_slli_epi32((__m512i)v_s1_sums, 6);
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* AVX-512BW implementation */
+
+/* AVX2 implementation: like the AVX-512BW one, but does 32 bytes at a time */
+#undef DISPATCH_AVX2
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__AVX2__) || (X86_CPU_FEATURES_ENABLED &&	\
+			       COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_avx2
+#  define FUNCNAME_CHUNK	adler32_avx2_chunk
+#  define IMPL_ALIGNMENT	32
+#  define IMPL_SEGMENT_SIZE	32
+#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
+#  ifdef __AVX2__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_avx2
+#  else
+#    define ATTRIBUTES		__attribute__((target("avx2")))
+#    define DISPATCH		1
+#    define DISPATCH_AVX2	1
+#  endif
+#  include <immintrin.h>
+static forceinline ATTRIBUTES void
+adler32_avx2_chunk(const __m256i *p, const __m256i *const end, u32 *s1, u32 *s2)
+{
+	const __m256i zeroes = _mm256_setzero_si256();
+	const __v32qu multipliers = (__v32qu){
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const __v16hu ones = (__v16hu)_mm256_set1_epi16(1);
+	__v8su v_s1 = (__v8su)zeroes;
+	__v8su v_s1_sums = (__v8su)zeroes;
+	__v8su v_s2 = (__v8su)zeroes;
+
+	do {
+		/* Load the next 32-byte segment */
+		__m256i bytes = *p++;
+
+		/* Multiply the bytes by 32...1 (the number of times they need
+		 * to be added to s2) and add adjacent products */
+		__v16hu sums = (__v16hu)_mm256_maddubs_epi16(
+						bytes, (__m256i)multipliers);
+
+		/* Keep sum of all previous s1 counters, for adding to s2 later.
+		 * This allows delaying the multiplication by 32 to the end. */
+		v_s1_sums += v_s1;
+
+		/* Add the sum of each group of 8 bytes to the corresponding s1
+		 * counter */
+		v_s1 += (__v8su)_mm256_sad_epu8(bytes, zeroes);
+
+		/* Add the sum of each group of 4 products of the bytes by
+		 * 32...1 to the corresponding s2 counter */
+		v_s2 += (__v8su)_mm256_madd_epi16((__m256i)sums, (__m256i)ones);
+	} while (p != end);
+
+	/* Finish the s2 counters by adding the sum of the s1 values at the
+	 * beginning of each segment, multiplied by the segment size (32) */
+	v_s2 += (__v8su)_mm256_slli_epi32((__m256i)v_s1_sums, 5);
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* AVX2 implementation */
+
+/* SSE2 implementation */
+#undef DISPATCH_SSE2
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__SSE2__) || (X86_CPU_FEATURES_ENABLED &&	\
+			       COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS))
+#  define FUNCNAME		adler32_sse2
+#  define FUNCNAME_CHUNK	adler32_sse2_chunk
+#  define IMPL_ALIGNMENT	16
+#  define IMPL_SEGMENT_SIZE	32
+/*
+ * The 16-bit precision byte counters must not be allowed to undergo *signed*
+ * overflow, otherwise the signed multiplications at the end (_mm_madd_epi16)
+ * would behave incorrectly.
+ */
+#  define IMPL_MAX_CHUNK_SIZE	(32 * (0x7FFF / 0xFF))
+#  ifdef __SSE2__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	adler32_sse2
+#  else
+#    define ATTRIBUTES		__attribute__((target("sse2")))
+#    define DISPATCH		1
+#    define DISPATCH_SSE2	1
+#  endif
+#  include <emmintrin.h>
+static forceinline ATTRIBUTES void
+adler32_sse2_chunk(const __m128i *p, const __m128i *const end, u32 *s1, u32 *s2)
+{
+	const __m128i zeroes = _mm_setzero_si128();
+
+	/* s1 counters: 32-bit, sum of bytes */
+	__v4su v_s1 = (__v4su)zeroes;
+
+	/* s2 counters: 32-bit, sum of s1 values */
+	__v4su v_s2 = (__v4su)zeroes;
+
+	/*
+	 * Thirty-two 16-bit counters for byte sums.  Each accumulates the bytes
+	 * that eventually need to be multiplied by a number 32...1 for addition
+	 * into s2.
+	 */
+	__v8hu v_byte_sums_a = (__v8hu)zeroes;
+	__v8hu v_byte_sums_b = (__v8hu)zeroes;
+	__v8hu v_byte_sums_c = (__v8hu)zeroes;
+	__v8hu v_byte_sums_d = (__v8hu)zeroes;
+
+	do {
+		/* Load the next 32 bytes */
+		const __m128i bytes1 = *p++;
+		const __m128i bytes2 = *p++;
+
+		/*
+		 * Accumulate the previous s1 counters into the s2 counters.
+		 * Logically, this really should be v_s2 += v_s1 * 32, but we
+		 * can do the multiplication (or left shift) later.
+		 */
+		v_s2 += v_s1;
+
+		/*
+		 * s1 update: use "Packed Sum of Absolute Differences" to add
+		 * the bytes horizontally with 8 bytes per sum.  Then add the
+		 * sums to the s1 counters.
+		 */
+		v_s1 += (__v4su)_mm_sad_epu8(bytes1, zeroes);
+		v_s1 += (__v4su)_mm_sad_epu8(bytes2, zeroes);
+
+		/*
+		 * Also accumulate the bytes into 32 separate counters that have
+		 * 16-bit precision.
+		 */
+		v_byte_sums_a += (__v8hu)_mm_unpacklo_epi8(bytes1, zeroes);
+		v_byte_sums_b += (__v8hu)_mm_unpackhi_epi8(bytes1, zeroes);
+		v_byte_sums_c += (__v8hu)_mm_unpacklo_epi8(bytes2, zeroes);
+		v_byte_sums_d += (__v8hu)_mm_unpackhi_epi8(bytes2, zeroes);
+
+	} while (p != end);
+
+	/* Finish calculating the s2 counters */
+	v_s2 = (__v4su)_mm_slli_epi32((__m128i)v_s2, 5);
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_a,
+				       (__m128i)(__v8hu){ 32, 31, 30, 29, 28, 27, 26, 25 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_b,
+				       (__m128i)(__v8hu){ 24, 23, 22, 21, 20, 19, 18, 17 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_c,
+				       (__m128i)(__v8hu){ 16, 15, 14, 13, 12, 11, 10, 9 });
+	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_d,
+				       (__m128i)(__v8hu){ 8,  7,  6,  5,  4,  3,  2,  1 });
+
+	/* Add the counters to the real s1 and s2 */
+	ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2);
+}
+#  include "../adler32_vec_template.h"
+#endif /* SSE2 implementation */
+
+#ifdef DISPATCH
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_AVX512BW
+	if (features & X86_CPU_FEATURE_AVX512BW)
+		return adler32_avx512bw;
+#endif
+#ifdef DISPATCH_AVX2
+	if (features & X86_CPU_FEATURE_AVX2)
+		return adler32_avx2;
+#endif
+#ifdef DISPATCH_SSE2
+	if (features & X86_CPU_FEATURE_SSE2)
+		return adler32_sse2;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_ADLER32_IMPL_H */

lib/x86/cpu_features.c

@@ -0,0 +1,152 @@
+/*
+ * x86/cpu_features.c - feature detection for x86 processors
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#if X86_CPU_FEATURES_ENABLED
+
+volatile u32 _cpu_features = 0;
+
+/* With old GCC versions we have to manually save and restore the x86_32 PIC
+ * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
+#if defined(__i386__) && defined(__PIC__)
+#  define EBX_CONSTRAINT "=&r"
+#else
+#  define EBX_CONSTRAINT "=b"
+#endif
+
+/* Execute the CPUID instruction.  */
+static inline void
+cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
+{
+	__asm__(".ifnc %%ebx, %1; mov  %%ebx, %1; .endif\n"
+		"cpuid                                  \n"
+		".ifnc %%ebx, %1; xchg %%ebx, %1; .endif\n"
+		: "=a" (*a), EBX_CONSTRAINT (*b), "=c" (*c), "=d" (*d)
+		: "a" (leaf), "c" (subleaf));
+}
+
+/* Read an extended control register.  */
+static inline u64
+read_xcr(u32 index)
+{
+	u32 edx, eax;
+
+	/* Execute the "xgetbv" instruction.  Old versions of binutils do not
+	 * recognize this instruction, so list the raw bytes instead.  */
+	__asm__ (".byte 0x0f, 0x01, 0xd0" : "=d" (edx), "=a" (eax) : "c" (index));
+
+	return ((u64)edx << 32) | eax;
+}
+
+#undef BIT
+#define BIT(nr)			(1UL << (nr))
+
+#define XCR0_BIT_SSE		BIT(1)
+#define XCR0_BIT_AVX		BIT(2)
+#define XCR0_BIT_OPMASK		BIT(5)
+#define XCR0_BIT_ZMM_HI256	BIT(6)
+#define XCR0_BIT_HI16_ZMM	BIT(7)
+
+#define IS_SET(reg, nr)		((reg) & BIT(nr))
+#define IS_ALL_SET(reg, mask)	(((reg) & (mask)) == (mask))
+
+static const struct cpu_feature x86_cpu_feature_table[] = {
+	{X86_CPU_FEATURE_SSE2,		"sse2"},
+	{X86_CPU_FEATURE_PCLMUL,	"pclmul"},
+	{X86_CPU_FEATURE_AVX,		"avx"},
+	{X86_CPU_FEATURE_AVX2,		"avx2"},
+	{X86_CPU_FEATURE_BMI2,		"bmi2"},
+	{X86_CPU_FEATURE_AVX512BW,	"avx512bw"},
+};
+
+/* Initialize _cpu_features with bits for interesting processor features. */
+void setup_cpu_features(void)
+{
+	u32 features = 0;
+	u32 dummy1, dummy2, dummy3, dummy4;
+	u32 max_function;
+	u32 features_1, features_2, features_3, features_4;
+	bool os_avx_support = false;
+	bool os_avx512_support = false;
+
+	/* Get maximum supported function  */
+	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
+	if (max_function < 1)
+		goto out;
+
+	/* Standard feature flags  */
+	cpuid(1, 0, &dummy1, &dummy2, &features_2, &features_1);
+
+	if (IS_SET(features_1, 26))
+		features |= X86_CPU_FEATURE_SSE2;
+
+	if (IS_SET(features_2, 1))
+		features |= X86_CPU_FEATURE_PCLMUL;
+
+	if (IS_SET(features_2, 27)) { /* OSXSAVE set? */
+		u64 xcr0 = read_xcr(0);
+
+		os_avx_support = IS_ALL_SET(xcr0,
+					    XCR0_BIT_SSE |
+					    XCR0_BIT_AVX);
+
+		os_avx512_support = IS_ALL_SET(xcr0,
+					       XCR0_BIT_SSE |
+					       XCR0_BIT_AVX |
+					       XCR0_BIT_OPMASK |
+					       XCR0_BIT_ZMM_HI256 |
+					       XCR0_BIT_HI16_ZMM);
+	}
+
+	if (os_avx_support && IS_SET(features_2, 28))
+		features |= X86_CPU_FEATURE_AVX;
+
+	if (max_function < 7)
+		goto out;
+
+	/* Extended feature flags  */
+	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
+
+	if (os_avx_support && IS_SET(features_3, 5))
+		features |= X86_CPU_FEATURE_AVX2;
+
+	if (IS_SET(features_3, 8))
+		features |= X86_CPU_FEATURE_BMI2;
+
+	if (os_avx512_support && IS_SET(features_3, 30))
+		features |= X86_CPU_FEATURE_AVX512BW;
+
+out:
+	disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
+					 ARRAY_LEN(x86_cpu_feature_table));
+
+	_cpu_features = features | X86_CPU_FEATURES_KNOWN;
+}
+
+#endif /* X86_CPU_FEATURES_ENABLED */

lib/x86/cpu_features.h

@@ -0,0 +1,41 @@
+/*
+ * x86/cpu_features.h - feature detection for x86 processors
+ */
+
+#ifndef LIB_X86_CPU_FEATURES_H
+#define LIB_X86_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if (defined(__i386__) || defined(__x86_64__)) && \
+	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+#  define X86_CPU_FEATURES_ENABLED 1
+#else
+#  define X86_CPU_FEATURES_ENABLED 0
+#endif
+
+#if X86_CPU_FEATURES_ENABLED
+
+#define X86_CPU_FEATURE_SSE2		0x00000001
+#define X86_CPU_FEATURE_PCLMUL		0x00000002
+#define X86_CPU_FEATURE_AVX		0x00000004
+#define X86_CPU_FEATURE_AVX2		0x00000008
+#define X86_CPU_FEATURE_BMI2		0x00000010
+#define X86_CPU_FEATURE_AVX512BW	0x00000020
+
+#define X86_CPU_FEATURES_KNOWN		0x80000000
+
+extern volatile u32 _cpu_features;
+
+void setup_cpu_features(void);
+
+static inline u32 get_cpu_features(void)
+{
+	if (_cpu_features == 0)
+		setup_cpu_features();
+	return _cpu_features;
+}
+
+#endif /* X86_CPU_FEATURES_ENABLED */
+
+#endif /* LIB_X86_CPU_FEATURES_H */

lib/x86/crc32_impl.h

@@ -0,0 +1,92 @@
+/*
+ * x86/crc32_impl.h - x86 implementations of CRC-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_CRC32_IMPL_H
+#define LIB_X86_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * Include the PCLMUL/AVX implementation?  Although our PCLMUL-optimized CRC-32
+ * function doesn't use any AVX intrinsics specifically, it can benefit a lot
+ * from being compiled for an AVX target: on Skylake, ~16700 MB/s vs. ~10100
+ * MB/s.  I expect this is related to the PCLMULQDQ instructions being assembled
+ * in the newer three-operand form rather than the older two-operand form.
+ *
+ * Note: this is only needed if __AVX__ is *not* defined, since otherwise the
+ * "regular" PCLMUL implementation would already be AVX enabled.
+ */
+#undef DISPATCH_PCLMUL_AVX
+#if !defined(DEFAULT_IMPL) && !defined(__AVX__) &&	\
+	X86_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_AVX_TARGET &&	\
+	(defined(__PCLMUL__) || COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS)
+#  define FUNCNAME		crc32_pclmul_avx
+#  define FUNCNAME_ALIGNED	crc32_pclmul_avx_aligned
+#  define ATTRIBUTES		__attribute__((target("pclmul,avx")))
+#  define DISPATCH		1
+#  define DISPATCH_PCLMUL_AVX	1
+#  include "crc32_pclmul_template.h"
+#endif
+
+/* PCLMUL implementation */
+#undef DISPATCH_PCLMUL
+#if !defined(DEFAULT_IMPL) &&	\
+	(defined(__PCLMUL__) || (X86_CPU_FEATURES_ENABLED &&	\
+				 COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS))
+#  define FUNCNAME		crc32_pclmul
+#  define FUNCNAME_ALIGNED	crc32_pclmul_aligned
+#  ifdef __PCLMUL__
+#    define ATTRIBUTES
+#    define DEFAULT_IMPL	crc32_pclmul
+#  else
+#    define ATTRIBUTES		__attribute__((target("pclmul")))
+#    define DISPATCH		1
+#    define DISPATCH_PCLMUL	1
+#  endif
+#  include "crc32_pclmul_template.h"
+#endif
+
+#ifdef DISPATCH
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_PCLMUL_AVX
+	if ((features & X86_CPU_FEATURE_PCLMUL) &&
+	    (features & X86_CPU_FEATURE_AVX))
+		return crc32_pclmul_avx;
+#endif
+#ifdef DISPATCH_PCLMUL
+	if (features & X86_CPU_FEATURE_PCLMUL)
+		return crc32_pclmul;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_CRC32_IMPL_H */

lib/x86/crc32_pclmul_template.h

@@ -0,0 +1,262 @@
+/*
+ * x86/crc32_pclmul_template.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <immintrin.h>
+
+/*
+ * CRC-32 folding with PCLMULQDQ.
+ *
+ * The basic idea is to repeatedly "fold" each 512 bits into the next 512 bits,
+ * producing an abbreviated message which is congruent the original message
+ * modulo the generator polynomial G(x).
+ *
+ * Folding each 512 bits is implemented as eight 64-bit folds, each of which
+ * uses one carryless multiplication instruction.  It's expected that CPUs may
+ * be able to execute some of these multiplications in parallel.
+ *
+ * Explanation of "folding": let A(x) be 64 bits from the message, and let B(x)
+ * be 95 bits from a constant distance D later in the message.  The relevant
+ * portion of the message can be written as:
+ *
+ *	M(x) = A(x)*x^D + B(x)
+ *
+ * ... where + and * represent addition and multiplication, respectively, of
+ * polynomials over GF(2).  Note that when implemented on a computer, these
+ * operations are equivalent to XOR and carryless multiplication, respectively.
+ *
+ * For the purpose of CRC calculation, only the remainder modulo the generator
+ * polynomial G(x) matters:
+ *
+ *	M(x) mod G(x) = (A(x)*x^D + B(x)) mod G(x)
+ *
+ * Since the modulo operation can be applied anywhere in a sequence of additions
+ * and multiplications without affecting the result, this is equivalent to:
+ *
+ *	M(x) mod G(x) = (A(x)*(x^D mod G(x)) + B(x)) mod G(x)
+ *
+ * For any D, 'x^D mod G(x)' will be a polynomial with maximum degree 31, i.e.
+ * a 32-bit quantity.  So 'A(x) * (x^D mod G(x))' is equivalent to a carryless
+ * multiplication of a 64-bit quantity by a 32-bit quantity, producing a 95-bit
+ * product.  Then, adding (XOR-ing) the product to B(x) produces a polynomial
+ * with the same length as B(x) but with the same remainder as 'A(x)*x^D +
+ * B(x)'.  This is the basic fold operation with 64 bits.
+ *
+ * Note that the carryless multiplication instruction PCLMULQDQ actually takes
+ * two 64-bit inputs and produces a 127-bit product in the low-order bits of a
+ * 128-bit XMM register.  This works fine, but care must be taken to account for
+ * "bit endianness".  With the CRC version implemented here, bits are always
+ * ordered such that the lowest-order bit represents the coefficient of highest
+ * power of x and the highest-order bit represents the coefficient of the lowest
+ * power of x.  This is backwards from the more intuitive order.  Still,
+ * carryless multiplication works essentially the same either way.  It just must
+ * be accounted for that when we XOR the 95-bit product in the low-order 95 bits
+ * of a 128-bit XMM register into 128-bits of later data held in another XMM
+ * register, we'll really be XOR-ing the product into the mathematically higher
+ * degree end of those later bits, not the lower degree end as may be expected.
+ *
+ * So given that caveat and the fact that we process 512 bits per iteration, the
+ * 'D' values we need for the two 64-bit halves of each 128 bits of data are:
+ *
+ *	D = (512 + 95) - 64	 for the higher-degree half of each 128 bits,
+ *				 i.e. the lower order bits in the XMM register
+ *
+ *	D = (512 + 95) - 128	 for the lower-degree half of each 128 bits,
+ *				 i.e. the higher order bits in the XMM register
+ *
+ * The required 'x^D mod G(x)' values were precomputed.
+ *
+ * When <= 512 bits remain in the message, we finish up by folding across
+ * smaller distances.  This works similarly; the distance D is just different,
+ * so different constant multipliers must be used.  Finally, once the remaining
+ * message is just 64 bits, it is reduced to the CRC-32 using Barrett reduction
+ * (explained later).
+ *
+ * For more information see the original paper from Intel:
+ *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
+ *	December 2009
+ *	http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+ */
+static u32 ATTRIBUTES
+FUNCNAME_ALIGNED(u32 remainder, const __m128i *p, size_t nr_segs)
+{
+	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
+	const __v2di multipliers_4 = (__v2di){ 0x8F352D95, 0x1D9513D7 };
+	const __v2di multipliers_2 = (__v2di){ 0xF1DA05AA, 0x81256527 };
+	const __v2di multipliers_1 = (__v2di){ 0xAE689191, 0xCCAA009E };
+	const __v2di final_multiplier = (__v2di){ 0xB8BC6765 };
+	const __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };
+	const __v2di barrett_reduction_constants =
+			(__v2di){ 0x00000001F7011641, 0x00000001DB710641 };
+
+	const __m128i * const end = p + nr_segs;
+	const __m128i * const end512 = p + (nr_segs & ~3);
+	__m128i x0, x1, x2, x3;
+
+	/*
+	 * Account for the current 'remainder', i.e. the CRC of the part of the
+	 * message already processed.  Explanation: rewrite the message
+	 * polynomial M(x) in terms of the first part A(x), the second part
+	 * B(x), and the length of the second part in bits |B(x)| >= 32:
+	 *
+	 *	M(x) = A(x)*x^|B(x)| + B(x)
+	 *
+	 * Then the CRC of M(x) is:
+	 *
+	 *	CRC(M(x)) = CRC(A(x)*x^|B(x)| + B(x))
+	 *	          = CRC(A(x)*x^32*x^(|B(x)| - 32) + B(x))
+	 *	          = CRC(CRC(A(x))*x^(|B(x)| - 32) + B(x))
+	 *
+	 * Note: all arithmetic is modulo G(x), the generator polynomial; that's
+	 * why A(x)*x^32 can be replaced with CRC(A(x)) = A(x)*x^32 mod G(x).
+	 *
+	 * So the CRC of the full message is the CRC of the second part of the
+	 * message where the first 32 bits of the second part of the message
+	 * have been XOR'ed with the CRC of the first part of the message.
+	 */
+	x0 = *p++;
+	x0 ^= (__m128i)(__v4si){ remainder };
+
+	if (p > end512) /* only 128, 256, or 384 bits of input? */
+		goto _128_bits_at_a_time;
+	x1 = *p++;
+	x2 = *p++;
+	x3 = *p++;
+
+	/* Fold 512 bits at a time */
+	for (; p != end512; p += 4) {
+		__m128i y0, y1, y2, y3;
+
+		y0 = p[0];
+		y1 = p[1];
+		y2 = p[2];
+		y3 = p[3];
+
+		/*
+		 * Note: the immediate constant for PCLMULQDQ specifies which
+		 * 64-bit halves of the 128-bit vectors to multiply:
+		 *
+		 * 0x00 means low halves (higher degree polynomial terms for us)
+		 * 0x11 means high halves (lower degree polynomial terms for us)
+		 */
+		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x00);
+		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x00);
+		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x00);
+		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x00);
+		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x11);
+		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x11);
+		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x11);
+		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x11);
+
+		x0 = y0;
+		x1 = y1;
+		x2 = y2;
+		x3 = y3;
+	}
+
+	/* Fold 512 bits => 128 bits */
+	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x00);
+	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x00);
+	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x11);
+	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x11);
+	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x00);
+	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x11);
+	x0 = x3;
+
+_128_bits_at_a_time:
+	while (p != end) {
+		/* Fold 128 bits into next 128 bits */
+		x1 = *p++;
+		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x00);
+		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x11);
+		x0 = x1;
+	}
+
+	/* Now there are just 128 bits left, stored in 'x0'. */
+
+	/*
+	 * Fold 128 => 96 bits.  This also implicitly appends 32 zero bits,
+	 * which is equivalent to multiplying by x^32.  This is needed because
+	 * the CRC is defined as M(x)*x^32 mod G(x), not just M(x) mod G(x).
+	 */
+	x0 = _mm_srli_si128(x0, 8) ^
+	     _mm_clmulepi64_si128(x0, multipliers_1, 0x10);
+
+	/* Fold 96 => 64 bits */
+	x0 = _mm_srli_si128(x0, 4) ^
+	     _mm_clmulepi64_si128(x0 & mask32, final_multiplier, 0x00);
+
+        /*
+	 * Finally, reduce 64 => 32 bits using Barrett reduction.
+	 *
+	 * Let M(x) = A(x)*x^32 + B(x) be the remaining message.  The goal is to
+	 * compute R(x) = M(x) mod G(x).  Since degree(B(x)) < degree(G(x)):
+	 *
+	 *	R(x) = (A(x)*x^32 + B(x)) mod G(x)
+	 *	     = (A(x)*x^32) mod G(x) + B(x)
+	 *
+	 * Then, by the Division Algorithm there exists a unique q(x) such that:
+	 *
+	 *	A(x)*x^32 mod G(x) = A(x)*x^32 - q(x)*G(x)
+	 *
+	 * Since the left-hand side is of maximum degree 31, the right-hand side
+	 * must be too.  This implies that we can apply 'mod x^32' to the
+	 * right-hand side without changing its value:
+	 *
+	 *	(A(x)*x^32 - q(x)*G(x)) mod x^32 = q(x)*G(x) mod x^32
+	 *
+	 * Note that '+' is equivalent to '-' in polynomials over GF(2).
+	 *
+	 * We also know that:
+	 *
+	 *	              / A(x)*x^32 \
+	 *	q(x) = floor (  ---------  )
+	 *	              \    G(x)   /
+	 *
+	 * To compute this efficiently, we can multiply the top and bottom by
+	 * x^32 and move the division by G(x) to the top:
+	 *
+	 *	              / A(x) * floor(x^64 / G(x)) \
+	 *	q(x) = floor (  -------------------------  )
+	 *	              \           x^32            /
+	 *
+	 * Note that floor(x^64 / G(x)) is a constant.
+	 *
+	 * So finally we have:
+	 *
+	 *	                          / A(x) * floor(x^64 / G(x)) \
+	 *	R(x) = B(x) + G(x)*floor (  -------------------------  )
+	 *	                          \           x^32            /
+	 */
+	x1 = x0;
+	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x00);
+	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x10);
+	return _mm_cvtsi128_si32(_mm_srli_si128(x0 ^ x1, 4));
+}
+
+#define IMPL_ALIGNMENT		16
+#define IMPL_SEGMENT_SIZE	16
+#include "../crc32_vec_template.h"

lib/x86/decompress_impl.h

@@ -0,0 +1,31 @@
+#ifndef LIB_X86_DECOMPRESS_IMPL_H
+#define LIB_X86_DECOMPRESS_IMPL_H
+
+#include "cpu_features.h"
+
+/* Include the BMI2-optimized version? */
+#undef DISPATCH_BMI2
+#if !defined(__BMI2__) && X86_CPU_FEATURES_ENABLED && \
+	COMPILER_SUPPORTS_BMI2_TARGET
+#  define FUNCNAME	deflate_decompress_bmi2
+#  define ATTRIBUTES	__attribute__((target("bmi2")))
+#  define DISPATCH	1
+#  define DISPATCH_BMI2	1
+#  include "../decompress_template.h"
+#endif
+
+#ifdef DISPATCH
+static inline decompress_func_t
+arch_select_decompress_func(void)
+{
+	u32 features = get_cpu_features();
+
+#ifdef DISPATCH_BMI2
+	if (features & X86_CPU_FEATURE_BMI2)
+		return deflate_decompress_bmi2;
+#endif
+	return NULL;
+}
+#endif /* DISPATCH */
+
+#endif /* LIB_X86_DECOMPRESS_IMPL_H */

lib/x86/matchfinder_impl.h

@@ -0,0 +1,122 @@
+/*
+ * x86/matchfinder_impl.h - x86 implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_MATCHFINDER_IMPL_H
+#define LIB_X86_MATCHFINDER_IMPL_H
+
+#ifdef __AVX2__
+#  include <immintrin.h>
+static forceinline void
+matchfinder_init_avx2(mf_pos_t *data, size_t size)
+{
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_avx2
+
+static forceinline void
+matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
+{
+	__m256i *p = (__m256i *)data;
+	__m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
+		p[0] = _mm256_adds_epi16(p[0], v);
+		p[1] = _mm256_adds_epi16(p[1], v);
+		p[2] = _mm256_adds_epi16(p[2], v);
+		p[3] = _mm256_adds_epi16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_avx2
+
+#elif defined(__SSE2__)
+#  include <emmintrin.h>
+static forceinline void
+matchfinder_init_sse2(mf_pos_t *data, size_t size)
+{
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		p[0] = v;
+		p[1] = v;
+		p[2] = v;
+		p[3] = v;
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_sse2
+
+static forceinline void
+matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
+{
+	__m128i *p = (__m128i *)data;
+	__m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+	STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+	do {
+		/* PADDSW: Add Packed Signed Integers With Signed Saturation  */
+		p[0] = _mm_adds_epi16(p[0], v);
+		p[1] = _mm_adds_epi16(p[1], v);
+		p[2] = _mm_adds_epi16(p[2], v);
+		p[3] = _mm_adds_epi16(p[3], v);
+		p += 4;
+		size -= 4 * sizeof(*p);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_sse2
+#endif /* __SSE2__ */
+
+#endif /* LIB_X86_MATCHFINDER_IMPL_H */

lib/x86_cpu_features.c

@@ -1,149 +0,0 @@
-/*
- * x86_cpu_features.c - feature detection for x86 processors
- *
- * Written in 2015 by Eric Biggers <ebiggers3@gmail.com>
- *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
- *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
- */
-
-#include "x86_cpu_features.h"
-
-#if X86_CPU_FEATURES_ENABLED
-
-#define DEBUG 0
-
-#if DEBUG
-#  include <stdio.h>
-#endif
-
-u32 _x86_cpu_features = 0;
-
-/* With old GCC versions we have to manually save and restore the x86_32 PIC
- * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
-#if defined(__i386__) && defined(__PIC__)
-#  define EBX_CONSTRAINT "=r"
-#else
-#  define EBX_CONSTRAINT "=b"
-#endif
-
-/* Execute the CPUID instruction.  */
-static inline void
-cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
-{
-	__asm__(".ifnc %%ebx, %1; mov  %%ebx, %1; .endif\n"
-		"cpuid                                  \n"
-		".ifnc %%ebx, %1; xchg %%ebx, %1; .endif\n"
-		: "=a" (*a), EBX_CONSTRAINT (*b), "=c" (*c), "=d" (*d)
-		: "a" (leaf), "c" (subleaf));
-}
-
-/* Read an extended control register.  */
-static inline u64
-read_xcr(u32 index)
-{
-	u32 edx, eax;
-
-	/* Execute the "xgetbv" instruction.  Old versions of binutils do not
-	 * recognize this instruction, so list the raw bytes instead.  */
-	__asm__ (".byte 0x0f, 0x01, 0xd0" : "=d" (edx), "=a" (eax) : "c" (index));
-
-	return ((u64)edx << 32) | eax;
-}
-
-#define IS_SET(reg, bit) ((reg) & ((u32)1 << (bit)))
-
-/* Initialize _x86_cpu_features with bits for interesting processor features. */
-void
-x86_setup_cpu_features(void)
-{
-	u32 features = 0;
-	u32 dummy1, dummy2, dummy3, dummy4;
-	u32 max_function;
-	u32 features_1, features_2, features_3, features_4;
-	bool os_saves_ymm_regs = false;
-
-	/* Get maximum supported function  */
-	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
-	if (max_function < 1)
-		goto out;
-
-	/* Standard feature flags  */
-	cpuid(1, 0, &dummy1, &dummy2, &features_2, &features_1);
-
-	if (IS_SET(features_1, 25))
-		features |= X86_CPU_FEATURE_SSE;
-
-	if (IS_SET(features_1, 26))
-		features |= X86_CPU_FEATURE_SSE2;
-
-	if (IS_SET(features_2, 0))
-		features |= X86_CPU_FEATURE_SSE3;
-
-	if (IS_SET(features_2, 9))
-		features |= X86_CPU_FEATURE_SSSE3;
-
-	if (IS_SET(features_2, 19))
-		features |= X86_CPU_FEATURE_SSE4_1;
-
-	if (IS_SET(features_2, 20))
-		features |= X86_CPU_FEATURE_SSE4_2;
-
-	if (IS_SET(features_2, 27)) /* OSXSAVE set?  */
-		if ((read_xcr(0) & 0x6) == 0x6)
-			os_saves_ymm_regs = true;
-
-	if (os_saves_ymm_regs && IS_SET(features_2, 28))
-		features |= X86_CPU_FEATURE_AVX;
-
-	if (max_function < 7)
-		goto out;
-
-	/* Extended feature flags  */
-	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
-
-	if (IS_SET(features_3, 3))
-		features |= X86_CPU_FEATURE_BMI;
-
-	if (os_saves_ymm_regs && IS_SET(features_3, 5))
-		features |= X86_CPU_FEATURE_AVX2;
-
-	if (IS_SET(features_3, 8))
-		features |= X86_CPU_FEATURE_BMI2;
-
-out:
-
-#if DEBUG
-	printf("Detected x86 CPU features: ");
-	if (features & X86_CPU_FEATURE_SSE)
-		printf("SSE ");
-	if (features & X86_CPU_FEATURE_SSE2)
-		printf("SSE2 ");
-	if (features & X86_CPU_FEATURE_SSE3)
-		printf("SSE3 ");
-	if (features & X86_CPU_FEATURE_SSSE3)
-		printf("SSSE3 ");
-	if (features & X86_CPU_FEATURE_SSE4_1)
-		printf("SSE4.1 ");
-	if (features & X86_CPU_FEATURE_SSE4_2)
-		printf("SSE4.2 ");
-	if (features & X86_CPU_FEATURE_BMI)
-		printf("BMI ");
-	if (features & X86_CPU_FEATURE_AVX)
-		printf("AVX ");
-	if (features & X86_CPU_FEATURE_BMI2)
-		printf("BMI2 ");
-	if (features & X86_CPU_FEATURE_AVX2)
-		printf("AVX2 ");
-	printf("\n");
-#endif /* DEBUG */
-
-	_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
-}
-
-#endif /* X86_CPU_FEATURES_ENABLED */

lib/x86_cpu_features.h

@@ -1,47 +0,0 @@
-/*
- * x86_cpu_features.h - feature detection for x86 processors
- */
-
-#ifndef _LIB_X86_CPU_FEATURES_H
-#define _LIB_X86_CPU_FEATURES_H
-
-#include "common_defs.h"
-
-#if defined(__x86_64__) && COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-#  define X86_CPU_FEATURES_ENABLED 1
-#else
-#  define X86_CPU_FEATURES_ENABLED 0
-#endif
-
-#if X86_CPU_FEATURES_ENABLED
-
-#define X86_CPU_FEATURE_SSE		0x00000001
-#define X86_CPU_FEATURE_SSE2		0x00000002
-#define X86_CPU_FEATURE_SSE3		0x00000004
-#define X86_CPU_FEATURE_SSSE3		0x00000008
-#define X86_CPU_FEATURE_SSE4_1		0x00000010
-#define X86_CPU_FEATURE_SSE4_2		0x00000020
-#define X86_CPU_FEATURE_AVX		0x00000040
-#define X86_CPU_FEATURE_BMI		0x00000080
-#define X86_CPU_FEATURE_AVX2		0x00000100
-#define X86_CPU_FEATURE_BMI2		0x00000200
-
-#define X86_CPU_FEATURES_KNOWN		0x80000000
-
-extern u32 _x86_cpu_features;
-
-extern void
-x86_setup_cpu_features(void);
-
-/* Does the processor have the specified feature?  */
-static inline bool
-x86_have_cpu_feature(u32 feature)
-{
-	if (_x86_cpu_features == 0)
-		x86_setup_cpu_features();
-	return _x86_cpu_features & feature;
-}
-
-#endif /* X86_CPU_FEATURES_ENABLED */
-
-#endif /* _LIB_X86_CPU_FEATURES_H */

lib/zlib_compress.c

@@ -1,27 +1,42 @@
 /*
  * zlib_compress.c - compress with a zlib wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
 #include "deflate_compress.h"
 #include "unaligned.h"
 #include "zlib_constants.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT size_t
-zlib_compress(struct deflate_compressor *c, const void *in, size_t in_size,
-	      void *out, size_t out_nbytes_avail)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress(struct libdeflate_compressor *c,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail)
 {
 	u8 *out_next = out;
 	u16 hdr;
@@ -50,21 +65,23 @@ zlib_compress(struct deflate_compressor *c, const void *in, size_t in_size,
 	out_next += 2;
 
 	/* Compressed data  */
-	deflate_size = deflate_compress(c, in, in_size, out_next,
+	deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
 					out_nbytes_avail - ZLIB_MIN_OVERHEAD);
 	if (deflate_size == 0)
 		return 0;
 	out_next += deflate_size;
 
 	/* ADLER32  */
-	put_unaligned_be32(adler32(in, in_size), out_next);
+	put_unaligned_be32(libdeflate_adler32(1, in, in_nbytes), out_next);
 	out_next += 4;
 
 	return out_next - (u8 *)out;
 }
 
-LIBEXPORT size_t
-zlib_compress_bound(struct deflate_compressor *c, size_t in_nbytes)
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *c,
+			       size_t in_nbytes)
 {
-	return ZLIB_MIN_OVERHEAD + deflate_compress_bound(c, in_nbytes);
+	return ZLIB_MIN_OVERHEAD +
+	       libdeflate_deflate_compress_bound(c, in_nbytes);
 }

lib/zlib_constants.h

@@ -2,8 +2,8 @@
  * zlib_constants.h - constants for the zlib wrapper format
  */
 
-#ifndef _LIB_ZLIB_CONSTANTS_H
-#define _LIB_ZLIB_CONSTANTS_H
+#ifndef LIB_ZLIB_CONSTANTS_H
+#define LIB_ZLIB_CONSTANTS_H
 
 #define ZLIB_MIN_HEADER_SIZE	2
 #define ZLIB_FOOTER_SIZE	4
@@ -18,4 +18,4 @@
 #define ZLIB_DEFAULT_COMPRESSION	2
 #define ZLIB_SLOWEST_COMPRESSION	3
 
-#endif /* _LIB_ZLIB_CONSTANTS_H */
+#endif /* LIB_ZLIB_CONSTANTS_H */

lib/zlib_decompress.c

@@ -1,37 +1,53 @@
 /*
  * zlib_decompress.c - decompress with a zlib wrapper
  *
- * Written in 2014-2016 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "adler32.h"
 #include "unaligned.h"
 #include "zlib_constants.h"
 
 #include "libdeflate.h"
 
-LIBEXPORT enum decompress_result
-zlib_decompress(struct deflate_decompressor *d,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret)
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret)
 {
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
 	u16 hdr;
+	size_t actual_in_nbytes;
 	size_t actual_out_nbytes;
-	enum decompress_result result;
+	enum libdeflate_result result;
 
 	if (in_nbytes < ZLIB_MIN_OVERHEAD)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* 2 byte header: CMF and FLG  */
 	hdr = get_unaligned_be16(in_next);
@@ -39,26 +55,26 @@ zlib_decompress(struct deflate_decompressor *d,
 
 	/* FCHECK */
 	if ((hdr % 31) != 0)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* CM */
 	if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* CINFO */
 	if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* FDICT */
 	if ((hdr >> 5) & 1)
-		return DECOMPRESS_BAD_DATA;
+		return LIBDEFLATE_BAD_DATA;
 
 	/* Compressed data  */
-	result = deflate_decompress(d, in_next,
-				    in_end - ZLIB_FOOTER_SIZE - in_next,
-				    out, out_nbytes_avail,
-				    actual_out_nbytes_ret);
-	if (result != DECOMPRESS_SUCCESS)
+	result = libdeflate_deflate_decompress_ex(d, in_next,
+					in_end - ZLIB_FOOTER_SIZE - in_next,
+					out, out_nbytes_avail,
+					&actual_in_nbytes, actual_out_nbytes_ret);
+	if (result != LIBDEFLATE_SUCCESS)
 		return result;
 
 	if (actual_out_nbytes_ret)
@@ -66,11 +82,27 @@ zlib_decompress(struct deflate_decompressor *d,
 	else
 		actual_out_nbytes = out_nbytes_avail;
 
-	in_next = in_end - ZLIB_FOOTER_SIZE;
+	in_next += actual_in_nbytes;
 
 	/* ADLER32  */
-	if (adler32(out, actual_out_nbytes) != get_unaligned_be32(in_next))
-		return DECOMPRESS_BAD_DATA;
+	if (libdeflate_adler32(1, out, actual_out_nbytes) !=
+	    get_unaligned_be32(in_next))
+		return LIBDEFLATE_BAD_DATA;
+	in_next += 4;
 
-	return DECOMPRESS_SUCCESS;
+	if (actual_in_nbytes_ret)
+		*actual_in_nbytes_ret = in_next - (u8 *)in;
+
+	return LIBDEFLATE_SUCCESS;
+}
+
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret)
+{
+	return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
+					     out, out_nbytes_avail,
+					     NULL, actual_out_nbytes_ret);
 }

libdeflate.h

@@ -9,69 +9,100 @@
 extern "C" {
 #endif
 
-#define LIBDEFLATE_VERSION_MAJOR	0
-#define LIBDEFLATE_VERSION_MINOR	3
-#define LIBDEFLATE_VERSION_STRING	"0.3"
+#define LIBDEFLATE_VERSION_MAJOR	1
+#define LIBDEFLATE_VERSION_MINOR	8
+#define LIBDEFLATE_VERSION_STRING	"1.8"
 
 #include <stddef.h>
+#include <stdint.h>
 
-/* Microsoft C / Visual Studio garbage.  If you want to link to the DLL version
- * of libdeflate, then #define LIBDEFLATE_DLL.  */
-#ifdef _MSC_VER
+/*
+ * On Windows, if you want to link to the DLL version of libdeflate, then
+ * #define LIBDEFLATE_DLL.  Note that the calling convention is "stdcall".
+ */
+#ifdef LIBDEFLATE_DLL
 #  ifdef BUILDING_LIBDEFLATE
-#    define LIBDEFLATEAPI __declspec(dllexport)
-#  elif defined(LIBDEFLATE_DLL)
-#    define LIBDEFLATEAPI __declspec(dllimport)
+#    define LIBDEFLATEEXPORT	LIBEXPORT
+#  elif defined(_WIN32) || defined(__CYGWIN__)
+#    define LIBDEFLATEEXPORT	__declspec(dllimport)
 #  endif
 #endif
-#ifndef LIBDEFLATEAPI
-#  define LIBDEFLATEAPI
+#ifndef LIBDEFLATEEXPORT
+#  define LIBDEFLATEEXPORT
 #endif
 
+#if defined(_WIN32) && !defined(_WIN64)
+#  define LIBDEFLATEAPI_ABI	__stdcall
+#else
+#  define LIBDEFLATEAPI_ABI
+#endif
+
+#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \
+	defined(_WIN32) && !defined(_WIN64)
+    /*
+     * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4.
+     * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX
+     * code when called from an MSVC-compiled application.
+     */
+#  define LIBDEFLATEAPI_STACKALIGN	__attribute__((force_align_arg_pointer))
+#else
+#  define LIBDEFLATEAPI_STACKALIGN
+#endif
+
+#define LIBDEFLATEAPI	LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN
+
 /* ========================================================================== */
 /*                             Compression                                    */
 /* ========================================================================== */
 
-struct deflate_compressor;
+struct libdeflate_compressor;
 
 /*
- * deflate_alloc_compressor() allocates a new DEFLATE compressor.
- * 'compression_level' is the compression level on a zlib-like scale but with a
- * higher maximum value (1 = fastest, 6 = medium/default, 9 = slow, 12 =
- * slowest).  The return value is a pointer to the new DEFLATE compressor, or
- * NULL if out of memory.
+ * libdeflate_alloc_compressor() allocates a new compressor that supports
+ * DEFLATE, zlib, and gzip compression.  'compression_level' is the compression
+ * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
+ * medium/default, 9 = slow, 12 = slowest).  Level 0 is also supported and means
+ * "no compression", specifically "create a valid stream, but only emit
+ * uncompressed blocks" (this will expand the data slightly).
+ *
+ * The return value is a pointer to the new compressor, or NULL if out of memory
+ * or if the compression level is invalid (i.e. outside the range [0, 12]).
  *
  * Note: for compression, the sliding window size is defined at compilation time
  * to 32768, the largest size permissible in the DEFLATE format.  It cannot be
  * changed at runtime.
+ *
+ * A single compressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different compressors concurrently.
  */
-LIBDEFLATEAPI struct deflate_compressor *
-deflate_alloc_compressor(unsigned int compression_level);
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+libdeflate_alloc_compressor(int compression_level);
 
 /*
- * deflate_compress() performs DEFLATE compression on a buffer of data.  The
- * function attempts to compress 'in_nbytes' bytes of data located at 'in' and
- * write the results to 'out', which has space for 'out_nbytes_avail' bytes.
- * The return value is the compressed size in bytes, or 0 if the data could not
- * be compressed to 'out_nbytes_avail' bytes or fewer.
+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
+ * data.  The function attempts to compress 'in_nbytes' bytes of data located at
+ * 'in' and write the results to 'out', which has space for 'out_nbytes_avail'
+ * bytes.  The return value is the compressed size in bytes, or 0 if the data
+ * could not be compressed to 'out_nbytes_avail' bytes or fewer.
  */
-LIBDEFLATEAPI size_t
-deflate_compress(struct deflate_compressor *compressor,
-		 const void *in, size_t in_nbytes,
-		 void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
+			    const void *in, size_t in_nbytes,
+			    void *out, size_t out_nbytes_avail);
 
 /*
- * deflate_compress_bound() returns a worst-case upper bound on the number of
- * bytes of compressed data that may be produced by compressing any buffer of
- * length less than or equal to 'in_nbytes' using deflate_compress() with the
- * specified compressor.  Mathematically, this bound will necessarily be a
- * number greater than or equal to 'in_nbytes'.  It may be an overestimate of
- * the true upper bound.  The return value is guaranteed to be the same for all
- * invocations with the same compressor and same 'in_nbytes'.
+ * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
+ * number of bytes of compressed data that may be produced by compressing any
+ * buffer of length less than or equal to 'in_nbytes' using
+ * libdeflate_deflate_compress() with the specified compressor.  Mathematically,
+ * this bound will necessarily be a number greater than or equal to 'in_nbytes'.
+ * It may be an overestimate of the true upper bound.  The return value is
+ * guaranteed to be the same for all invocations with the same compressor and
+ * same 'in_nbytes'.
  *
  * As a special case, 'compressor' may be NULL.  This causes the bound to be
- * taken across *any* deflate_compressor that could ever be allocated with this
- * build of the library, with any options.
+ * taken across *any* libdeflate_compressor that could ever be allocated with
+ * this build of the library, with any options.
  *
  * Note that this function is not necessary in many applications.  With
  * block-based compression, it is usually preferable to separately store the
@@ -79,154 +110,254 @@ deflate_compress(struct deflate_compressor *compressor,
  * to less than their original size uncompressed.  In that scenario, there is no
  * need to know the worst-case compressed size, since the maximum number of
  * bytes of compressed data that may be used would always be one less than the
- * input length.  You can just pass a buffer of that size to deflate_compress()
- * and store the data uncompressed if deflate_compress() returns 0, indicating
- * that the compressed data did not fit into the provided output buffer.
+ * input length.  You can just pass a buffer of that size to
+ * libdeflate_deflate_compress() and store the data uncompressed if
+ * libdeflate_deflate_compress() returns 0, indicating that the compressed data
+ * did not fit into the provided output buffer.
  */
-LIBDEFLATEAPI size_t
-deflate_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
+				  size_t in_nbytes);
 
 /*
- * Like deflate_compress(), but stores the data in the zlib wrapper format.
+ * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper
+ * format.
  */
-LIBDEFLATEAPI size_t
-zlib_compress(struct deflate_compressor *compressor,
-	      const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail);
 
 /*
- * Like deflate_compress_bound(), but assumes the data will be compressed with
- * zlib_compress() rather than with deflate_compress().
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_zlib_compress() rather than with
+ * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
-zlib_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
+			       size_t in_nbytes);
 
 /*
- * Like deflate_compress(), but stores the data in the gzip wrapper format.
+ * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper
+ * format.
  */
-LIBDEFLATEAPI size_t
-gzip_compress(struct deflate_compressor *compressor,
-	      const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
+			 const void *in, size_t in_nbytes,
+			 void *out, size_t out_nbytes_avail);
 
 /*
- * Like deflate_compress_bound(), but assumes the data will be compressed with
- * gzip_compress() rather than with deflate_compress().
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_gzip_compress() rather than with
+ * libdeflate_deflate_compress().
  */
-LIBDEFLATEAPI size_t
-gzip_compress_bound(struct deflate_compressor *compressor, size_t in_nbytes);
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
+			       size_t in_nbytes);
 
 /*
- * deflate_free_compressor() frees a DEFLATE compressor that was allocated with
- * deflate_alloc_compressor().  If a NULL pointer is passed in, no action is
+ * libdeflate_free_compressor() frees a compressor that was allocated with
+ * libdeflate_alloc_compressor().  If a NULL pointer is passed in, no action is
  * taken.
  */
-LIBDEFLATEAPI void
-deflate_free_compressor(struct deflate_compressor *compressor);
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_compressor(struct libdeflate_compressor *compressor);
 
 /* ========================================================================== */
 /*                             Decompression                                  */
 /* ========================================================================== */
 
-struct deflate_decompressor;
+struct libdeflate_decompressor;
 
 /*
- * deflate_alloc_decompressor() allocates a new DEFLATE decompressor.  The
- * return value is a pointer to the new DEFLATE decompressor, or NULL if out of
- * memory.
+ * libdeflate_alloc_decompressor() allocates a new decompressor that can be used
+ * for DEFLATE, zlib, and gzip decompression.  The return value is a pointer to
+ * the new decompressor, or NULL if out of memory.
  *
  * This function takes no parameters, and the returned decompressor is valid for
  * decompressing data that was compressed at any compression level and with any
- * sliding window size.  It can also be used for any wrapper format (raw
- * DEFLATE, zlib, or gzip); however, the appropriate decompression function must
- * be called.
+ * sliding window size.
+ *
+ * A single decompressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different decompressors concurrently.
  */
-LIBDEFLATEAPI struct deflate_decompressor *
-deflate_alloc_decompressor(void);
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
+libdeflate_alloc_decompressor(void);
 
 /*
- * Result of a call to deflate_decompress(), zlib_decompress(), or
- * gzip_decompress().
+ * Result of a call to libdeflate_deflate_decompress(),
+ * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
  */
-enum decompress_result {
+enum libdeflate_result {
 	/* Decompression was successful.  */
-	DECOMPRESS_SUCCESS = 0,
+	LIBDEFLATE_SUCCESS = 0,
 
 	/* Decompressed failed because the compressed data was invalid, corrupt,
 	 * or otherwise unsupported.  */
-	DECOMPRESS_BAD_DATA = 1,
+	LIBDEFLATE_BAD_DATA = 1,
 
 	/* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
 	 * decompressed to fewer than 'out_nbytes_avail' bytes.  */
-	DECOMPRESS_SHORT_OUTPUT = 2,
+	LIBDEFLATE_SHORT_OUTPUT = 2,
 
 	/* The data would have decompressed to more than 'out_nbytes_avail'
 	 * bytes.  */
-	DECOMPRESS_INSUFFICIENT_SPACE = 3,
+	LIBDEFLATE_INSUFFICIENT_SPACE = 3,
 };
 
 /*
- * deflate_decompress() decompresses 'in_nbytes' bytes of DEFLATE-compressed
- * data at 'in' and writes the uncompressed data to 'out', which is a buffer of
- * at least 'out_nbytes_avail' bytes.  If decompression was successful, then 0
- * (DECOMPRESS_SUCCESS) is returned; otherwise, a nonzero result code such as
- * DECOMPRESS_BAD_DATA is returned.  If a nonzero result code is returned, then
- * the contents of the output buffer are undefined.
+ * libdeflate_deflate_decompress() decompresses the DEFLATE-compressed stream
+ * from the buffer 'in' with compressed size up to 'in_nbytes' bytes.  The
+ * uncompressed data is written to 'out', a buffer with size 'out_nbytes_avail'
+ * bytes.  If decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned.
+ * Otherwise, a nonzero result code such as LIBDEFLATE_BAD_DATA is returned.  If
+ * a nonzero result code is returned, then the contents of the output buffer are
+ * undefined.
  *
- * deflate_decompress() can be used in cases where the actual uncompressed size
- * is known (recommended) or unknown (not recommended):
+ * Decompression stops at the end of the DEFLATE stream (as indicated by the
+ * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
+ *
+ * libdeflate_deflate_decompress() can be used in cases where the actual
+ * uncompressed size is known (recommended) or unknown (not recommended):
  *
  *   - If the actual uncompressed size is known, then pass the actual
  *     uncompressed size as 'out_nbytes_avail' and pass NULL for
- *     'actual_out_nbytes_ret'.  This makes deflate_decompress() fail with
- *     DECOMPRESS_SHORT_OUTPUT if the data decompressed to fewer than the
+ *     'actual_out_nbytes_ret'.  This makes libdeflate_deflate_decompress() fail
+ *     with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
  *     specified number of bytes.
  *
  *   - If the actual uncompressed size is unknown, then provide a non-NULL
  *     'actual_out_nbytes_ret' and provide a buffer with some size
  *     'out_nbytes_avail' that you think is large enough to hold all the
  *     uncompressed data.  In this case, if the data decompresses to less than
- *     or equal to 'out_nbytes_avail' bytes, then deflate_decompress() will
- *     write the actual uncompressed size to *actual_out_nbytes_ret and return 0
- *     (DECOMPRESS_SUCCESS).  Otherwise, it will return
- *     DECOMPRESS_INSUFFICIENT_SPACE if the provided buffer was not large enough
- *     but no other problems were encountered, or another nonzero result code if
- *     decompression failed for another reason.
+ *     or equal to 'out_nbytes_avail' bytes, then
+ *     libdeflate_deflate_decompress() will write the actual uncompressed size
+ *     to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS).  Otherwise,
+ *     it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
+ *     not large enough but no other problems were encountered, or another
+ *     nonzero result code if decompression failed for another reason.
  */
-LIBDEFLATEAPI enum decompress_result
-deflate_decompress(struct deflate_decompressor *decompressor,
-		   const void *in, size_t in_nbytes,
-		   void *out, size_t out_nbytes_avail,
-		   size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_out_nbytes_ret);
 
 /*
- * Like deflate_decompress(), but assumes the zlib wrapper format instead of raw
- * DEFLATE.
+ * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
+ * then the actual compressed size of the DEFLATE stream (aligned to the next
+ * byte boundary) is written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEAPI enum decompress_result
-zlib_decompress(struct deflate_decompressor *decompressor,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
+				 const void *in, size_t in_nbytes,
+				 void *out, size_t out_nbytes_avail,
+				 size_t *actual_in_nbytes_ret,
+				 size_t *actual_out_nbytes_ret);
 
 /*
- * Like deflate_decompress(), but assumes the gzip wrapper format instead of raw
- * DEFLATE.
+ * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
+ * instead of raw DEFLATE.
+ *
+ * Decompression will stop at the end of the zlib stream, even if it is shorter
+ * than 'in_nbytes'.  If you need to know exactly where the zlib stream ended,
+ * use libdeflate_zlib_decompress_ex().
  */
-LIBDEFLATEAPI enum decompress_result
-gzip_decompress(struct deflate_decompressor *decompressor,
-		const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail,
-		size_t *actual_out_nbytes_ret);
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret);
 
 /*
- * deflate_free_decompressor() frees a DEFLATE decompressor that was allocated
- * with deflate_alloc_decompressor().  If a NULL pointer is passed in, no action
+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first zlib-compressed stream in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
+ * instead of raw DEFLATE.
+ *
+ * If multiple gzip-compressed members are concatenated, then only the first
+ * will be decompressed.  Use libdeflate_gzip_decompress_ex() if you need
+ * multi-member support.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail,
+			   size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument.  If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first gzip-compressed member in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
+			      const void *in, size_t in_nbytes,
+			      void *out, size_t out_nbytes_avail,
+			      size_t *actual_in_nbytes_ret,
+			      size_t *actual_out_nbytes_ret);
+
+/*
+ * libdeflate_free_decompressor() frees a decompressor that was allocated with
+ * libdeflate_alloc_decompressor().  If a NULL pointer is passed in, no action
  * is taken.
  */
-LIBDEFLATEAPI void
-deflate_free_decompressor(struct deflate_decompressor *decompressor);
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
 
+/* ========================================================================== */
+/*                                Checksums                                   */
+/* ========================================================================== */
+
+/*
+ * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
+ * data and returns the updated checksum.  When starting a new checksum, the
+ * required initial value for 'adler' is 1.  This value is also returned when
+ * 'buffer' is specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
+
+
+/*
+ * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
+ * and returns the updated checksum.  When starting a new checksum, the required
+ * initial value for 'crc' is 0.  This value is also returned when 'buffer' is
+ * specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
+
+/* ========================================================================== */
+/*                           Custom memory allocator                          */
+/* ========================================================================== */
+
+/*
+ * Install a custom memory allocator which libdeflate will use for all memory
+ * allocations.  'malloc_func' is a function that must behave like malloc(), and
+ * 'free_func' is a function that must behave like free().
+ *
+ * There must not be any libdeflate_compressor or libdeflate_decompressor
+ * structures in existence when calling this function.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+				void (*free_func)(void *));
 
 #ifdef __cplusplus
 }

programs/benchmark.c

@@ -25,31 +25,390 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <zlib.h> /* for comparison purposes */
+#include "test_util.h"
 
-#include "prog_util.h"
+static const tchar *const optstring = T("0::1::2::3::4::5::6::7::8::9::C:D:eghs:VYZz");
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::ghs:VYZz");
+enum format {
+	DEFLATE_FORMAT,
+	ZLIB_FORMAT,
+	GZIP_FORMAT,
+};
+
+struct compressor {
+	int level;
+	enum format format;
+	const struct engine *engine;
+	void *private;
+};
+
+struct decompressor {
+	enum format format;
+	const struct engine *engine;
+	void *private;
+};
+
+struct engine {
+	const tchar *name;
+
+	bool (*init_compressor)(struct compressor *);
+	size_t (*compress_bound)(struct compressor *, size_t);
+	size_t (*compress)(struct compressor *, const void *, size_t,
+			   void *, size_t);
+	void (*destroy_compressor)(struct compressor *);
+
+	bool (*init_decompressor)(struct decompressor *);
+	bool (*decompress)(struct decompressor *, const void *, size_t,
+			   void *, size_t);
+	void (*destroy_decompressor)(struct decompressor *);
+};
+
+/******************************************************************************/
+
+static bool
+libdeflate_engine_init_compressor(struct compressor *c)
+{
+	c->private = alloc_compressor(c->level);
+	return c->private != NULL;
+}
+
+static size_t
+libdeflate_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	switch (c->format) {
+	case ZLIB_FORMAT:
+		return libdeflate_zlib_compress_bound(c->private, in_nbytes);
+	case GZIP_FORMAT:
+		return libdeflate_gzip_compress_bound(c->private, in_nbytes);
+	default:
+		return libdeflate_deflate_compress_bound(c->private, in_nbytes);
+	}
+}
+
+static size_t
+libdeflate_engine_compress(struct compressor *c, const void *in,
+			   size_t in_nbytes, void *out, size_t out_nbytes_avail)
+{
+	switch (c->format) {
+	case ZLIB_FORMAT:
+		return libdeflate_zlib_compress(c->private, in, in_nbytes,
+						out, out_nbytes_avail);
+	case GZIP_FORMAT:
+		return libdeflate_gzip_compress(c->private, in, in_nbytes,
+						out, out_nbytes_avail);
+	default:
+		return libdeflate_deflate_compress(c->private, in, in_nbytes,
+						   out, out_nbytes_avail);
+	}
+}
+
+static void
+libdeflate_engine_destroy_compressor(struct compressor *c)
+{
+	libdeflate_free_compressor(c->private);
+}
+
+static bool
+libdeflate_engine_init_decompressor(struct decompressor *d)
+{
+	d->private = alloc_decompressor();
+	return d->private != NULL;
+}
+
+static bool
+libdeflate_engine_decompress(struct decompressor *d, const void *in,
+			     size_t in_nbytes, void *out, size_t out_nbytes)
+{
+	switch (d->format) {
+	case ZLIB_FORMAT:
+		return !libdeflate_zlib_decompress(d->private, in, in_nbytes,
+						   out, out_nbytes, NULL);
+	case GZIP_FORMAT:
+		return !libdeflate_gzip_decompress(d->private, in, in_nbytes,
+						   out, out_nbytes, NULL);
+	default:
+		return !libdeflate_deflate_decompress(d->private, in, in_nbytes,
+						      out, out_nbytes, NULL);
+	}
+}
+
+static void
+libdeflate_engine_destroy_decompressor(struct decompressor *d)
+{
+	libdeflate_free_decompressor(d->private);
+}
+
+static const struct engine libdeflate_engine = {
+	.name			= T("libdeflate"),
+
+	.init_compressor	= libdeflate_engine_init_compressor,
+	.compress_bound		= libdeflate_engine_compress_bound,
+	.compress		= libdeflate_engine_compress,
+	.destroy_compressor	= libdeflate_engine_destroy_compressor,
+
+	.init_decompressor	= libdeflate_engine_init_decompressor,
+	.decompress		= libdeflate_engine_decompress,
+	.destroy_decompressor	= libdeflate_engine_destroy_decompressor,
+};
+
+/******************************************************************************/
+
+static int
+get_libz_window_bits(enum format format)
+{
+	const int windowBits = 15;
+	switch (format) {
+	case ZLIB_FORMAT:
+		return windowBits;
+	case GZIP_FORMAT:
+		return windowBits + 16;
+	default:
+		return -windowBits;
+	}
+}
+
+static bool
+libz_engine_init_compressor(struct compressor *c)
+{
+	z_stream *z;
+
+	if (c->level > 9) {
+		msg("libz only supports up to compression level 9");
+		return false;
+	}
+
+	z = xmalloc(sizeof(*z));
+	if (z == NULL)
+		return false;
+
+	z->next_in = NULL;
+	z->avail_in = 0;
+	z->zalloc = NULL;
+	z->zfree = NULL;
+	z->opaque = NULL;
+	if (deflateInit2(z, c->level, Z_DEFLATED,
+			 get_libz_window_bits(c->format),
+			 8, Z_DEFAULT_STRATEGY) != Z_OK)
+	{
+		msg("unable to initialize deflater");
+		free(z);
+		return false;
+	}
+
+	c->private = z;
+	return true;
+}
+
+static size_t
+libz_engine_compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return deflateBound(c->private, in_nbytes);
+}
+
+static size_t
+libz_engine_compress(struct compressor *c, const void *in, size_t in_nbytes,
+		     void *out, size_t out_nbytes_avail)
+{
+	z_stream *z = c->private;
+
+	deflateReset(z);
+
+	z->next_in = (void *)in;
+	z->avail_in = in_nbytes;
+	z->next_out = out;
+	z->avail_out = out_nbytes_avail;
+
+	if (deflate(z, Z_FINISH) != Z_STREAM_END)
+		return 0;
+
+	return out_nbytes_avail - z->avail_out;
+}
+
+static void
+libz_engine_destroy_compressor(struct compressor *c)
+{
+	z_stream *z = c->private;
+
+	deflateEnd(z);
+	free(z);
+}
+
+static bool
+libz_engine_init_decompressor(struct decompressor *d)
+{
+	z_stream *z;
+
+	z = xmalloc(sizeof(*z));
+	if (z == NULL)
+		return false;
+
+	z->next_in = NULL;
+	z->avail_in = 0;
+	z->zalloc = NULL;
+	z->zfree = NULL;
+	z->opaque = NULL;
+	if (inflateInit2(z, get_libz_window_bits(d->format)) != Z_OK) {
+		msg("unable to initialize inflater");
+		free(z);
+		return false;
+	}
+
+	d->private = z;
+	return true;
+}
+
+static bool
+libz_engine_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
+		       void *out, size_t out_nbytes)
+{
+	z_stream *z = d->private;
+
+	inflateReset(z);
+
+	z->next_in = (void *)in;
+	z->avail_in = in_nbytes;
+	z->next_out = out;
+	z->avail_out = out_nbytes;
+
+	return inflate(z, Z_FINISH) == Z_STREAM_END && z->avail_out == 0;
+}
+
+static void
+libz_engine_destroy_decompressor(struct decompressor *d)
+{
+	z_stream *z = d->private;
+
+	inflateEnd(z);
+	free(z);
+}
+
+static const struct engine libz_engine = {
+	.name			= T("libz"),
+
+	.init_compressor	= libz_engine_init_compressor,
+	.compress_bound		= libz_engine_compress_bound,
+	.compress		= libz_engine_compress,
+	.destroy_compressor	= libz_engine_destroy_compressor,
+
+	.init_decompressor	= libz_engine_init_decompressor,
+	.decompress		= libz_engine_decompress,
+	.destroy_decompressor	= libz_engine_destroy_decompressor,
+};
+
+/******************************************************************************/
+
+static const struct engine * const all_engines[] = {
+	&libdeflate_engine,
+	&libz_engine,
+};
+
+#define DEFAULT_ENGINE libdeflate_engine
+
+static const struct engine *
+name_to_engine(const tchar *name)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_LEN(all_engines); i++)
+		if (tstrcmp(all_engines[i]->name, name) == 0)
+			return all_engines[i];
+	return NULL;
+}
+
+/******************************************************************************/
+
+static bool
+compressor_init(struct compressor *c, int level, enum format format,
+		const struct engine *engine)
+{
+	c->level = level;
+	c->format = format;
+	c->engine = engine;
+	return engine->init_compressor(c);
+}
+
+static size_t
+compress_bound(struct compressor *c, size_t in_nbytes)
+{
+	return c->engine->compress_bound(c, in_nbytes);
+}
+
+static size_t
+do_compress(struct compressor *c, const void *in, size_t in_nbytes,
+	    void *out, size_t out_nbytes_avail)
+{
+	return c->engine->compress(c, in, in_nbytes, out, out_nbytes_avail);
+}
+
+static void
+compressor_destroy(struct compressor *c)
+{
+	if (c->engine != NULL)
+		c->engine->destroy_compressor(c);
+}
+
+static bool
+decompressor_init(struct decompressor *d, enum format format,
+		  const struct engine *engine)
+{
+	d->format = format;
+	d->engine = engine;
+	return engine->init_decompressor(d);
+}
+
+static bool
+do_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
+	      void *out, size_t out_nbytes)
+{
+	return d->engine->decompress(d, in, in_nbytes, out, out_nbytes);
+}
+
+static void
+decompressor_destroy(struct decompressor *d)
+{
+	if (d->engine != NULL)
+		d->engine->destroy_decompressor(d);
+}
+
+/******************************************************************************/
+
+static void
+show_available_engines(FILE *fp)
+{
+	size_t i;
+
+	fprintf(fp, "Available ENGINEs are: ");
+	for (i = 0; i < ARRAY_LEN(all_engines); i++) {
+		fprintf(fp, "%"TS, all_engines[i]->name);
+		if (i < ARRAY_LEN(all_engines) - 1)
+			fprintf(fp, ", ");
+	}
+	fprintf(fp, ".  Default is %"TS"\n", DEFAULT_ENGINE.name);
+}
 
 static void
 show_usage(FILE *fp)
 {
 	fprintf(fp,
-"Usage: %"TS" [-LVL] [-ghVYZz] [-s SIZE] [FILE]...\n"
+"Usage: %"TS" [-LVL] [-C ENGINE] [-D ENGINE] [-ghVz] [-s SIZE] [FILE]...\n"
 "Benchmark DEFLATE compression and decompression on the specified FILEs.\n"
 "\n"
 "Options:\n"
+"  -0        no compression\n"
 "  -1        fastest (worst) compression\n"
 "  -6        medium compression (default)\n"
 "  -12       slowest (best) compression\n"
-"  -g        use gzip wrapper\n"
+"  -C ENGINE compression engine\n"
+"  -D ENGINE decompression engine\n"
+"  -e        allow chunks to be expanded (implied by -0)\n"
+"  -g        use gzip format instead of raw DEFLATE\n"
 "  -h        print this help\n"
 "  -s SIZE   chunk size\n"
 "  -V        show version and legal information\n"
-"  -Y        compress with libz, not libdeflate\n"
-"  -Z        decompress with libz, not libdeflate\n"
-"  -z        use zlib wrapper\n",
-	program_invocation_name);
+"  -z        use zlib format instead of raw DEFLATE\n"
+"\n", prog_invocation_name);
+
+	show_available_engines(fp);
 }
 
 static void
@@ -65,273 +424,13 @@ show_version(void)
 	);
 }
 
-enum wrapper {
-	NO_WRAPPER,
-	ZLIB_WRAPPER,
-	GZIP_WRAPPER,
-};
 
-struct compressor {
-	void *private;
-	size_t (*compress)(void *, const void *, size_t, void *, size_t);
-	void (*free_private)(void *);
-};
-
-struct decompressor {
-	void *private;
-	bool (*decompress)(void *, const void *, size_t, void *, size_t);
-	void (*free_private)(void *);
-};
-
-static int
-get_libz_window_bits(enum wrapper wrapper)
-{
-	const int windowBits = 15;
-	switch (wrapper) {
-	case ZLIB_WRAPPER:
-		return windowBits;
-	case GZIP_WRAPPER:
-		return windowBits + 16;
-	default:
-		return -windowBits;
-	}
-}
-
-static size_t
-libz_compress(void *private, const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail)
-{
-	z_stream *z = private;
-
-	deflateReset(z);
-
-	z->next_in = (void *)in;
-	z->avail_in = in_nbytes;
-	z->next_out = out;
-	z->avail_out = out_nbytes_avail;
-
-	if (deflate(z, Z_FINISH) != Z_STREAM_END)
-		return 0;
-
-	return out_nbytes_avail - z->avail_out;
-}
-
-static bool
-libz_decompress(void *private, const void *in, size_t in_nbytes,
-		void *out, size_t out_nbytes_avail)
-{
-	z_stream *z = private;
-
-	inflateReset(z);
-
-	z->next_in = (void *)in;
-	z->avail_in = in_nbytes;
-	z->next_out = out;
-	z->avail_out = out_nbytes_avail;
-
-	return (inflate(z, Z_FINISH) == Z_STREAM_END && z->avail_out == 0);
-}
-
-static void
-libz_free_compressor(void *private)
-{
-	deflateEnd((z_stream *)private);
-	free(private);
-}
-
-static void
-libz_free_decompressor(void *private)
-{
-	inflateEnd((z_stream *)private);
-	free(private);
-}
-
-static size_t
-libdeflate_deflate_compress(void *private, const void *in, size_t in_nbytes,
-			    void *out, size_t out_nbytes_avail)
-{
-	return deflate_compress(private, in, in_nbytes,
-				out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_deflate_decompress(void *private, const void *in, size_t in_nbytes,
-			      void *out, size_t out_nbytes_avail)
-{
-	return 0 == deflate_decompress(private, in, in_nbytes,
-				       out, out_nbytes_avail, NULL);
-}
-
-static size_t
-libdeflate_zlib_compress(void *private, const void *in, size_t in_nbytes,
-			 void *out, size_t out_nbytes_avail)
-{
-	return zlib_compress(private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_zlib_decompress(void *private, const void *in, size_t in_nbytes,
-			   void *out, size_t out_nbytes_avail)
-{
-	return 0 == zlib_decompress(private, in, in_nbytes,
-				    out, out_nbytes_avail, NULL);
-}
-
-static size_t
-libdeflate_gzip_compress(void *private, const void *in, size_t in_nbytes,
-			 void *out, size_t out_nbytes_avail)
-{
-	return gzip_compress(private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static bool
-libdeflate_gzip_decompress(void *private, const void *in, size_t in_nbytes,
-			   void *out, size_t out_nbytes_avail)
-{
-	return 0 == gzip_decompress(private, in, in_nbytes,
-				    out, out_nbytes_avail, NULL);
-}
-
-static void
-libdeflate_free_compressor(void *private)
-{
-	deflate_free_compressor(private);
-}
-
-static void
-libdeflate_free_decompressor(void *private)
-{
-	deflate_free_decompressor(private);
-}
-
-static int
-compressor_init(struct compressor *c, int level,
-		enum wrapper wrapper, bool use_libz)
-{
-	if (use_libz) {
-		z_stream *z;
-
-		if (level > 9) {
-			msg("libz only supports up to compression level 9");
-			return -1;
-		}
-
-		z = xmalloc(sizeof(z_stream));
-		if (z == NULL)
-			return -1;
-
-		c->private = z;
-
-		z->next_in = NULL;
-		z->avail_in = 0;
-		z->zalloc = NULL;
-		z->zfree = NULL;
-		z->opaque = NULL;
-		if (deflateInit2(z, level, Z_DEFLATED,
-				 get_libz_window_bits(wrapper),
-				 8, Z_DEFAULT_STRATEGY) != Z_OK)
-		{
-			msg("unable to initialize deflater");
-			free(z);
-			return -1;
-		}
-		c->compress = libz_compress;
-		c->free_private = libz_free_compressor;
-	} else {
-		c->private = alloc_compressor(level);
-		if (c->private == NULL)
-			return -1;
-		switch (wrapper) {
-		case ZLIB_WRAPPER:
-			c->compress = libdeflate_zlib_compress;
-			break;
-		case GZIP_WRAPPER:
-			c->compress = libdeflate_gzip_compress;
-			break;
-		default:
-			c->compress = libdeflate_deflate_compress;
-			break;
-		}
-		c->free_private = libdeflate_free_compressor;
-	}
-	return 0;
-}
-
-static size_t
-do_compress(struct compressor *c, const void *in, size_t in_nbytes,
-	    void *out, size_t out_nbytes_avail)
-{
-	return (*c->compress)(c->private, in, in_nbytes, out, out_nbytes_avail);
-}
-
-static void
-compressor_destroy(struct compressor *c)
-{
-	(*c->free_private)(c->private);
-}
-
-static int
-decompressor_init(struct decompressor *d, enum wrapper wrapper, bool use_libz)
-{
-	if (use_libz) {
-		z_stream *z;
-
-		z = xmalloc(sizeof(z_stream));
-		if (z == NULL)
-			return -1;
-
-		d->private = z;
-
-		z->next_in = NULL;
-		z->avail_in = 0;
-		z->zalloc = NULL;
-		z->zfree = NULL;
-		z->opaque = NULL;
-		if (inflateInit2(z, get_libz_window_bits(wrapper)) != Z_OK) {
-			msg("unable to initialize inflater");
-			free(z);
-			return -1;
-		}
-
-		d->decompress = libz_decompress;
-		d->free_private = libz_free_decompressor;
-	} else {
-		d->private = alloc_decompressor();
-		if (d->private == NULL)
-			return -1;
-		switch (wrapper) {
-		case ZLIB_WRAPPER:
-			d->decompress = libdeflate_zlib_decompress;
-			break;
-		case GZIP_WRAPPER:
-			d->decompress = libdeflate_gzip_decompress;
-			break;
-		default:
-			d->decompress = libdeflate_deflate_decompress;
-			break;
-		}
-		d->free_private = libdeflate_free_decompressor;
-	}
-	return 0;
-}
-
-static bool
-do_decompress(struct decompressor *d, const void *in, size_t in_nbytes,
-	      void *out, size_t out_nbytes_avail)
-{
-	return (*d->decompress)(d->private, in, in_nbytes,
-				out, out_nbytes_avail);
-}
-
-static void
-decompressor_destroy(struct decompressor *d)
-{
-	(*d->free_private)(d->private);
-}
+/******************************************************************************/
 
 static int
 do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 	     void *decompressed_buf, u32 chunk_size,
+	     bool allow_expansion, size_t compressed_buf_size,
 	     struct compressor *compressor,
 	     struct decompressor *decompressor)
 {
@@ -343,35 +442,45 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 	while ((ret = xread(in, original_buf, chunk_size)) > 0) {
 		u32 original_size = ret;
+		size_t out_nbytes_avail;
 		u32 compressed_size;
 		u64 start_time;
-		bool result;
+		bool ok;
 
 		total_uncompressed_size += original_size;
 
+		if (allow_expansion) {
+			out_nbytes_avail = compress_bound(compressor,
+							  original_size);
+			if (out_nbytes_avail > compressed_buf_size) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
+		} else {
+			out_nbytes_avail = original_size - 1;
+		}
+
 		/* Compress the chunk of data. */
-		start_time = current_time();
+		start_time = timer_ticks();
 		compressed_size = do_compress(compressor,
 					      original_buf,
 					      original_size,
 					      compressed_buf,
-					      original_size - 1);
-		total_compress_time += current_time() - start_time;
+					      out_nbytes_avail);
+		total_compress_time += timer_ticks() - start_time;
 
 		if (compressed_size) {
 			/* Successfully compressed the chunk of data. */
 
 			/* Decompress the data we just compressed and compare
 			 * the result with the original. */
-			start_time = current_time();
-			result = do_decompress(decompressor,
-					       compressed_buf,
-					       compressed_size,
-					       decompressed_buf,
-					       original_size);
-			total_decompress_time += current_time() - start_time;
+			start_time = timer_ticks();
+			ok = do_decompress(decompressor,
+					   compressed_buf, compressed_size,
+					   decompressed_buf, original_size);
+			total_decompress_time += timer_ticks() - start_time;
 
-			if (!result) {
+			if (!ok) {
 				msg("%"TS": failed to decompress data",
 				    in->name);
 				return -1;
@@ -387,7 +496,14 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 
 			total_compressed_size += compressed_size;
 		} else {
-			/* Compression did not make the chunk smaller. */
+			/*
+			 * The chunk would have compressed to more than
+			 * out_nbytes_avail bytes.
+			 */
+			if (allow_expansion) {
+				msg("%"TS": bug in compress_bound()", in->name);
+				return -1;
+			}
 			total_compressed_size += original_size;
 		}
 	}
@@ -412,11 +528,11 @@ do_benchmark(struct file_stream *in, void *original_buf, void *compressed_buf,
 	       (unsigned int)(total_compressed_size * 100000 /
 				total_uncompressed_size % 1000));
 	printf("\tCompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n",
-	       total_compress_time / 1000000,
-	       1000 * total_uncompressed_size / total_compress_time);
+	       timer_ticks_to_ms(total_compress_time),
+	       timer_MB_per_s(total_uncompressed_size, total_compress_time));
 	printf("\tDecompression time: %"PRIu64" ms (%"PRIu64" MB/s)\n",
-	       total_decompress_time / 1000000,
-	       1000 * total_uncompressed_size / total_decompress_time);
+	       timer_ticks_to_ms(total_decompress_time),
+	       timer_MB_per_s(total_uncompressed_size, total_decompress_time));
 
 	return 0;
 }
@@ -426,23 +542,26 @@ tmain(int argc, tchar *argv[])
 {
 	u32 chunk_size = 1048576;
 	int level = 6;
-	enum wrapper wrapper = NO_WRAPPER;
-	bool compress_with_libz = false;
-	bool decompress_with_libz = false;
+	enum format format = DEFLATE_FORMAT;
+	const struct engine *compress_engine = &DEFAULT_ENGINE;
+	const struct engine *decompress_engine = &DEFAULT_ENGINE;
+	bool allow_expansion = false;
+	struct compressor compressor = { 0 };
+	struct decompressor decompressor = { 0 };
+	size_t compressed_buf_size;
 	void *original_buf = NULL;
 	void *compressed_buf = NULL;
 	void *decompressed_buf = NULL;
-	struct compressor compressor;
-	struct decompressor decompressor;
 	tchar *default_file_list[] = { NULL };
 	int opt_char;
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
+		case '0':
 		case '1':
 		case '2':
 		case '3':
@@ -453,11 +572,30 @@ tmain(int argc, tchar *argv[])
 		case '8':
 		case '9':
 			level = parse_compression_level(opt_char, toptarg);
-			if (level == 0)
-				return -1;
+			if (level < 0)
+				return 1;
+			break;
+		case 'C':
+			compress_engine = name_to_engine(toptarg);
+			if (compress_engine == NULL) {
+				msg("invalid compression engine: \"%"TS"\"", toptarg);
+				show_available_engines(stderr);
+				return 1;
+			}
+			break;
+		case 'D':
+			decompress_engine = name_to_engine(toptarg);
+			if (decompress_engine == NULL) {
+				msg("invalid decompression engine: \"%"TS"\"", toptarg);
+				show_available_engines(stderr);
+				return 1;
+			}
+			break;
+		case 'e':
+			allow_expansion = true;
 			break;
 		case 'g':
-			wrapper = GZIP_WRAPPER;
+			format = GZIP_FORMAT;
 			break;
 		case 'h':
 			show_usage(stdout);
@@ -472,14 +610,14 @@ tmain(int argc, tchar *argv[])
 		case 'V':
 			show_version();
 			return 0;
-		case 'Y':
-			compress_with_libz = true;
+		case 'Y': /* deprecated, use '-C libz' instead */
+			compress_engine = &libz_engine;
 			break;
-		case 'Z':
-			decompress_with_libz = true;
+		case 'Z': /* deprecated, use '-D libz' instead */
+			decompress_engine = &libz_engine;
 			break;
 		case 'z':
-			wrapper = ZLIB_WRAPPER;
+			format = ZLIB_FORMAT;
 			break;
 		default:
 			show_usage(stderr);
@@ -490,22 +628,28 @@ tmain(int argc, tchar *argv[])
 	argc -= toptind;
 	argv += toptind;
 
+	if (level == 0)
+		allow_expansion = true;
+
+	ret = -1;
+	if (!compressor_init(&compressor, level, format, compress_engine))
+		goto out;
+	if (!decompressor_init(&decompressor, format, decompress_engine))
+		goto out;
+
+	if (allow_expansion)
+		compressed_buf_size = compress_bound(&compressor, chunk_size);
+	else
+		compressed_buf_size = chunk_size - 1;
+
 	original_buf = xmalloc(chunk_size);
-	compressed_buf = xmalloc(chunk_size - 1);
+	compressed_buf = xmalloc(compressed_buf_size);
 	decompressed_buf = xmalloc(chunk_size);
 
 	ret = -1;
 	if (original_buf == NULL || compressed_buf == NULL ||
 	    decompressed_buf == NULL)
-		goto out0;
-
-	ret = compressor_init(&compressor, level, wrapper, compress_with_libz);
-	if (ret)
-		goto out0;
-
-	ret = decompressor_init(&decompressor, wrapper, decompress_with_libz);
-	if (ret)
-		goto out1;
+		goto out;
 
 	if (argc == 0) {
 		argv = default_file_list;
@@ -516,41 +660,37 @@ tmain(int argc, tchar *argv[])
 				argv[i] = NULL;
 	}
 
-	printf("Benchmarking DEFLATE compression:\n");
+	printf("Benchmarking %s compression:\n",
+	       format == DEFLATE_FORMAT ? "DEFLATE" :
+	       format == ZLIB_FORMAT ? "zlib" : "gzip");
 	printf("\tCompression level: %d\n", level);
 	printf("\tChunk size: %"PRIu32"\n", chunk_size);
-	printf("\tWrapper: %s\n",
-	       wrapper == NO_WRAPPER ? "None" :
-	       wrapper == ZLIB_WRAPPER ? "zlib" : "gzip");
-	printf("\tCompression engine: %s\n",
-	       compress_with_libz ? "libz" : "libdeflate");
-	printf("\tDecompression engine: %s\n",
-	       decompress_with_libz ? "libz" : "libdeflate");
+	printf("\tCompression engine: %"TS"\n", compress_engine->name);
+	printf("\tDecompression engine: %"TS"\n", decompress_engine->name);
 
 	for (i = 0; i < argc; i++) {
 		struct file_stream in;
 
-		ret = xopen_for_read(argv[i], &in);
+		ret = xopen_for_read(argv[i], true, &in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 
 		printf("Processing %"TS"...\n", in.name);
 
 		ret = do_benchmark(&in, original_buf, compressed_buf,
-				   decompressed_buf, chunk_size, &compressor,
-				   &decompressor);
+				   decompressed_buf, chunk_size,
+				   allow_expansion, compressed_buf_size,
+				   &compressor, &decompressor);
 		xclose(&in);
 		if (ret != 0)
-			goto out2;
+			goto out;
 	}
 	ret = 0;
-out2:
-	decompressor_destroy(&decompressor);
-out1:
-	compressor_destroy(&compressor);
-out0:
+out:
 	free(decompressed_buf);
 	free(compressed_buf);
 	free(original_buf);
+	decompressor_destroy(&decompressor);
+	compressor_destroy(&compressor);
 	return -ret;
 }

programs/checksum.c

@@ -0,0 +1,207 @@
+/*
+ * checksum.c - Adler-32 and CRC-32 checksumming program
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "test_util.h"
+
+static const tchar *const optstring = T("Ahs:tZ");
+
+static void
+show_usage(FILE *fp)
+{
+	fprintf(fp,
+"Usage: %"TS" [-A] [-h] [-s SIZE] [-t] [-Z] [FILE]...\n"
+"Calculate Adler-32 or CRC-32 checksums of the specified FILEs.\n"
+"\n"
+"Options:\n"
+"  -A        use Adler-32 (default is CRC-32)\n"
+"  -h        print this help\n"
+"  -s SIZE   chunk size\n"
+"  -t        show checksum speed, excluding I/O\n"
+"  -Z        use zlib implementation instead of libdeflate\n",
+	prog_invocation_name);
+}
+
+typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
+
+static u32
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
+}
+
+static u32
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
+{
+	return adler32(adler, buf, len);
+}
+
+static u32
+crc32_zlib(u32 crc, const void *buf, size_t len)
+{
+	return crc32(crc, buf, len);
+}
+
+static int
+checksum_stream(struct file_stream *in, cksum_fn_t cksum, u32 *sum,
+		void *buf, size_t bufsize, u64 *size_ret, u64 *elapsed_ret)
+{
+	u64 size = 0;
+	u64 elapsed = 0;
+
+	for (;;) {
+		ssize_t ret;
+		u64 start_time;
+
+		ret = xread(in, buf, bufsize);
+		if (ret < 0)
+			return ret;
+		if (ret == 0)
+			break;
+
+		size += ret;
+		start_time = timer_ticks();
+		*sum = cksum(*sum, buf, ret);
+		elapsed += timer_ticks() - start_time;
+	}
+
+	if (elapsed == 0)
+		elapsed = 1;
+	*size_ret = size;
+	*elapsed_ret = elapsed;
+	return 0;
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	bool use_adler32 = false;
+	bool use_zlib_impl = false;
+	bool do_timing = false;
+	void *buf;
+	size_t bufsize = 131072;
+	tchar *default_file_list[] = { NULL };
+	cksum_fn_t cksum;
+	int opt_char;
+	int i;
+	int ret;
+
+	begin_program(argv);
+
+	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
+		switch (opt_char) {
+		case 'A':
+			use_adler32 = true;
+			break;
+		case 'h':
+			show_usage(stdout);
+			return 0;
+		case 's':
+			bufsize = tstrtoul(toptarg, NULL, 10);
+			if (bufsize == 0) {
+				msg("invalid chunk size: \"%"TS"\"", toptarg);
+				return 1;
+			}
+			break;
+		case 't':
+			do_timing = true;
+			break;
+		case 'Z':
+			use_zlib_impl = true;
+			break;
+		default:
+			show_usage(stderr);
+			return 1;
+		}
+	}
+
+	argc -= toptind;
+	argv += toptind;
+
+	if (use_adler32) {
+		if (use_zlib_impl)
+			cksum = adler32_zlib;
+		else
+			cksum = adler32_libdeflate;
+	} else {
+		if (use_zlib_impl)
+			cksum = crc32_zlib;
+		else
+			cksum = crc32_libdeflate;
+	}
+
+	buf = xmalloc(bufsize);
+	if (buf == NULL)
+		return 1;
+
+	if (argc == 0) {
+		argv = default_file_list;
+		argc = ARRAY_LEN(default_file_list);
+	} else {
+		for (i = 0; i < argc; i++)
+			if (argv[i][0] == '-' && argv[i][1] == '\0')
+				argv[i] = NULL;
+	}
+
+	for (i = 0; i < argc; i++) {
+		struct file_stream in;
+		u32 sum = cksum(0, NULL, 0);
+		u64 size = 0;
+		u64 elapsed = 0;
+
+		ret = xopen_for_read(argv[i], true, &in);
+		if (ret != 0)
+			goto out;
+
+		ret = checksum_stream(&in, cksum, &sum, buf, bufsize,
+				      &size, &elapsed);
+		if (ret == 0) {
+			if (do_timing) {
+				printf("%08"PRIx32"\t%"TS"\t"
+				       "%"PRIu64" ms\t%"PRIu64" MB/s\n",
+				       sum, in.name, timer_ticks_to_ms(elapsed),
+				       timer_MB_per_s(size, elapsed));
+			} else {
+				printf("%08"PRIx32"\t%"TS"\t\n", sum, in.name);
+			}
+		}
+
+		xclose(&in);
+
+		if (ret != 0)
+			goto out;
+	}
+	ret = 0;
+out:
+	free(buf);
+	return -ret;
+}

programs/detect.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-if [ -z "$CC" ]; then
-	CC=cc
-fi
-
-echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
-echo "#ifndef _CONFIG_H"
-echo "#define _CONFIG_H"
-
-tmpfile="$(mktemp -t libdeflate_config.XXXXXXXX)"
-trap "rm -f \"$tmpfile\"" EXIT
-
-check_function() {
-	funcname="$1"
-	macro="HAVE_$(echo $funcname | tr a-z A-Z)"
-	echo "int main() { $funcname(); }" > "$tmpfile"
-	echo
-	echo "/* Is the $funcname() function available? */"
-	if $CC -x c $tmpfile -o /dev/null > /dev/null 2>&1; then
-		echo "#define $macro 1"
-	else
-		echo "/* $macro is not set */"
-	fi
-}
-
-check_function clock_gettime
-check_function futimens
-check_function futimes
-
-echo
-echo "#endif /* _CONFIG_H */"

programs/gzip.c

@@ -27,6 +27,7 @@
 
 #include "prog_util.h"
 
+#include <errno.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #ifdef _WIN32
@@ -42,11 +43,12 @@ struct options {
 	bool decompress;
 	bool force;
 	bool keep;
+	bool test;
 	int compression_level;
 	const tchar *suffix;
 };
 
-static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhkS:V");
+static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknS:tV");
 
 static void
 show_usage(FILE *fp)
@@ -64,9 +66,10 @@ show_usage(FILE *fp)
 "  -f        overwrite existing output files\n"
 "  -h        print this help\n"
 "  -k        don't delete input files\n"
-"  -S SUF    use suffix .SUF instead of .gz\n"
+"  -S SUF    use suffix SUF instead of .gz\n"
+"  -t        test file integrity\n"
 "  -V        show version and legal information\n",
-	program_invocation_name);
+	prog_invocation_name);
 }
 
 static void
@@ -86,10 +89,14 @@ show_version(void)
 static bool
 is_gunzip(void)
 {
-	if (tstrxcmp(program_invocation_name, T("gunzip")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip")) == 0)
+		return true;
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip")) == 0)
 		return true;
 #ifdef _WIN32
-	if (tstrxcmp(program_invocation_name, T("gunzip.exe")) == 0)
+	if (tstrxcmp(prog_invocation_name, T("gunzip.exe")) == 0)
+		return true;
+	if (tstrxcmp(prog_invocation_name, T("libdeflate-gunzip.exe")) == 0)
 		return true;
 #endif
 	return false;
@@ -98,10 +105,15 @@ is_gunzip(void)
 static const tchar *
 get_suffix(const tchar *path, const tchar *suffix)
 {
-	const tchar *dot = tstrrchr(get_filename(path), '.');
+	size_t path_len = tstrlen(path);
+	size_t suffix_len = tstrlen(suffix);
+	const tchar *p;
 
-	if (dot != NULL && tstrxcmp(dot + 1, suffix) == 0)
-		return dot;
+	if (path_len <= suffix_len)
+		return NULL;
+	p = &path[path_len - suffix_len];
+	if (tstrxcmp(p, suffix) == 0)
+		return p;
 	return NULL;
 }
 
@@ -111,8 +123,23 @@ has_suffix(const tchar *path, const tchar *suffix)
 	return get_suffix(path, suffix) != NULL;
 }
 
+static tchar *
+append_suffix(const tchar *path, const tchar *suffix)
+{
+	size_t path_len = tstrlen(path);
+	size_t suffix_len = tstrlen(suffix);
+	tchar *suffixed_path;
+
+	suffixed_path = xmalloc((path_len + suffix_len + 1) * sizeof(tchar));
+	if (suffixed_path == NULL)
+		return NULL;
+	tmemcpy(suffixed_path, path, path_len);
+	tmemcpy(&suffixed_path[path_len], suffix, suffix_len + 1);
+	return suffixed_path;
+}
+
 static int
-do_compress(struct deflate_compressor *compressor,
+do_compress(struct libdeflate_compressor *compressor,
 	    struct file_stream *in, struct file_stream *out)
 {
 	const void *uncompressed_data = in->mmap_mem;
@@ -122,7 +149,8 @@ do_compress(struct deflate_compressor *compressor,
 	size_t max_compressed_size;
 	int ret;
 
-	max_compressed_size = gzip_compress_bound(compressor, uncompressed_size);
+	max_compressed_size = libdeflate_gzip_compress_bound(compressor,
+							     uncompressed_size);
 	compressed_data = xmalloc(max_compressed_size);
 	if (compressed_data == NULL) {
 		msg("%"TS": file is probably too large to be processed by this "
@@ -131,12 +159,13 @@ do_compress(struct deflate_compressor *compressor,
 		goto out;
 	}
 
-	actual_compressed_size =
-		gzip_compress(compressor, uncompressed_data, uncompressed_size,
-			      compressed_data, max_compressed_size);
-
+	actual_compressed_size = libdeflate_gzip_compress(compressor,
+							  uncompressed_data,
+							  uncompressed_size,
+							  compressed_data,
+							  max_compressed_size);
 	if (actual_compressed_size == 0) {
-		msg("Bug in gzip_compress_bound()!");
+		msg("Bug in libdeflate_gzip_compress_bound()!");
 		ret = -1;
 		goto out;
 	}
@@ -155,15 +184,18 @@ load_u32_gzip(const u8 *p)
 }
 
 static int
-do_decompress(struct deflate_decompressor *decompressor,
-	      struct file_stream *in, struct file_stream *out)
+do_decompress(struct libdeflate_decompressor *decompressor,
+	      struct file_stream *in, struct file_stream *out,
+	      const struct options *options)
 {
 	const u8 *compressed_data = in->mmap_mem;
 	size_t compressed_size = in->mmap_size;
 	void *uncompressed_data = NULL;
 	size_t uncompressed_size;
-	enum decompress_result result;
-	int ret;
+	size_t actual_in_nbytes;
+	size_t actual_out_nbytes;
+	enum libdeflate_result result;
+	int ret = 0;
 
 	if (compressed_size < sizeof(u32)) {
 	       msg("%"TS": not in gzip format", in->name);
@@ -171,42 +203,83 @@ do_decompress(struct deflate_decompressor *decompressor,
 	       goto out;
 	}
 
+	/*
+	 * Use the ISIZE field as a hint for the decompressed data size.  It may
+	 * need to be increased later, however, because the file may contain
+	 * multiple gzip members and the particular ISIZE we happen to use may
+	 * not be the largest; or the real size may be >= 4 GiB, causing ISIZE
+	 * to overflow.  In any case, make sure to allocate at least one byte.
+	 */
 	uncompressed_size = load_u32_gzip(&compressed_data[compressed_size - 4]);
+	if (uncompressed_size == 0)
+		uncompressed_size = 1;
 
-	uncompressed_data = xmalloc(uncompressed_size);
-	if (uncompressed_data == NULL) {
-		msg("%"TS": file is probably too large to be processed by this "
-		    "program", in->name);
-		ret = -1;
-		goto out;
-	}
+	do {
+		if (uncompressed_data == NULL) {
+			uncompressed_data = xmalloc(uncompressed_size);
+			if (uncompressed_data == NULL) {
+				msg("%"TS": file is probably too large to be "
+				    "processed by this program", in->name);
+				ret = -1;
+				goto out;
+			}
+		}
 
-	result = gzip_decompress(decompressor, compressed_data, compressed_size,
-				 uncompressed_data, uncompressed_size, NULL);
+		result = libdeflate_gzip_decompress_ex(decompressor,
+						       compressed_data,
+						       compressed_size,
+						       uncompressed_data,
+						       uncompressed_size,
+						       &actual_in_nbytes,
+						       &actual_out_nbytes);
 
-	if (result == DECOMPRESS_INSUFFICIENT_SPACE) {
-		msg("%"TS": file corrupt or too large to be processed by this "
-		    "program", in->name);
-		ret = -1;
-		goto out;
-	}
+		if (result == LIBDEFLATE_INSUFFICIENT_SPACE) {
+			if (uncompressed_size * 2 <= uncompressed_size) {
+				msg("%"TS": file corrupt or too large to be "
+				    "processed by this program", in->name);
+				ret = -1;
+				goto out;
+			}
+			uncompressed_size *= 2;
+			free(uncompressed_data);
+			uncompressed_data = NULL;
+			continue;
+		}
 
-	if (result != DECOMPRESS_SUCCESS) {
-		msg("%"TS": file corrupt or not in gzip format", in->name);
-		ret = -1;
-		goto out;
-	}
+		if (result != LIBDEFLATE_SUCCESS) {
+			msg("%"TS": file corrupt or not in gzip format",
+			    in->name);
+			ret = -1;
+			goto out;
+		}
 
-	ret = full_write(out, uncompressed_data, uncompressed_size);
+		if (actual_in_nbytes == 0 ||
+		    actual_in_nbytes > compressed_size ||
+		    actual_out_nbytes > uncompressed_size) {
+			msg("Bug in libdeflate_gzip_decompress_ex()!");
+			ret = -1;
+			goto out;
+		}
+
+		if (!options->test) {
+			ret = full_write(out, uncompressed_data, actual_out_nbytes);
+			if (ret != 0)
+				goto out;
+		}
+
+		compressed_data += actual_in_nbytes;
+		compressed_size -= actual_in_nbytes;
+
+	} while (compressed_size != 0);
 out:
 	free(uncompressed_data);
 	return ret;
 }
 
 static int
-stat_file(struct file_stream *in, struct stat *stbuf, bool allow_hard_links)
+stat_file(struct file_stream *in, stat_t *stbuf, bool allow_hard_links)
 {
-	if (fstat(in->fd, stbuf) != 0) {
+	if (tfstat(in->fd, stbuf) != 0) {
 		msg("%"TS": unable to stat file", in->name);
 		return -1;
 	}
@@ -228,7 +301,7 @@ stat_file(struct file_stream *in, struct stat *stbuf, bool allow_hard_links)
 }
 
 static void
-restore_mode(struct file_stream *out, const struct stat *stbuf)
+restore_mode(struct file_stream *out, const stat_t *stbuf)
 {
 #ifndef _WIN32
 	if (fchmod(out->fd, stbuf->st_mode) != 0)
@@ -237,7 +310,7 @@ restore_mode(struct file_stream *out, const struct stat *stbuf)
 }
 
 static void
-restore_owner_and_group(struct file_stream *out, const struct stat *stbuf)
+restore_owner_and_group(struct file_stream *out, const stat_t *stbuf)
 {
 #ifndef _WIN32
 	if (fchown(out->fd, stbuf->st_uid, stbuf->st_gid) != 0) {
@@ -249,33 +322,33 @@ restore_owner_and_group(struct file_stream *out, const struct stat *stbuf)
 
 static void
 restore_timestamps(struct file_stream *out, const tchar *newpath,
-		   const struct stat *stbuf)
+		   const stat_t *stbuf)
 {
 	int ret;
-#if defined(HAVE_FUTIMENS)
+#if defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION)
 	struct timespec times[2] = {
 		stbuf->st_atim, stbuf->st_mtim,
 	};
 	ret = futimens(out->fd, times);
-#elif defined(HAVE_FUTIMES)
+#elif defined(HAVE_FUTIMES) && defined(HAVE_STAT_NANOSECOND_PRECISION)
 	struct timeval times[2] = {
 		{ stbuf->st_atim.tv_sec, stbuf->st_atim.tv_nsec / 1000, },
 		{ stbuf->st_mtim.tv_sec, stbuf->st_mtim.tv_nsec / 1000, },
 	};
 	ret = futimes(out->fd, times);
-#else /* HAVE_FUTIMES */
+#else
 	struct tutimbuf times = {
 		stbuf->st_atime, stbuf->st_mtime,
 	};
 	ret = tutime(newpath, &times);
-#endif /* !HAVE_FUTIMES */
+#endif
 	if (ret != 0)
 		msg_errno("%"TS": unable to preserve timestamps", out->name);
 }
 
 static void
 restore_metadata(struct file_stream *out, const tchar *newpath,
-		 const struct stat *stbuf)
+		 const stat_t *stbuf)
 {
 	restore_mode(out, stbuf);
 	restore_owner_and_group(out, stbuf);
@@ -283,32 +356,58 @@ restore_metadata(struct file_stream *out, const tchar *newpath,
 }
 
 static int
-decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
+decompress_file(struct libdeflate_decompressor *decompressor, const tchar *path,
 		const struct options *options)
 {
+	tchar *oldpath = (tchar *)path;
 	tchar *newpath = NULL;
 	struct file_stream in;
 	struct file_stream out;
-	struct stat stbuf;
+	stat_t stbuf;
 	int ret;
 	int ret2;
 
-	if (path != NULL && !options->to_stdout) {
+	if (path != NULL) {
 		const tchar *suffix = get_suffix(path, options->suffix);
 		if (suffix == NULL) {
-			msg("\"%"TS"\" does not end with the .%"TS" suffix -- "
-			    "skipping", path, options->suffix);
-			ret = -2;
-			goto out;
+			/*
+			 * Input file is unsuffixed.  If the file doesn't exist,
+			 * then try it suffixed.  Otherwise, if we're not
+			 * writing to stdout, skip the file with warning status.
+			 * Otherwise, go ahead and try to open the file anyway
+			 * (which will very likely fail).
+			 */
+			if (tstat(path, &stbuf) != 0 && errno == ENOENT) {
+				oldpath = append_suffix(path, options->suffix);
+				if (oldpath == NULL)
+					return -1;
+				if (!options->to_stdout)
+					newpath = (tchar *)path;
+			} else if (!options->to_stdout) {
+				msg("\"%"TS"\" does not end with the %"TS" "
+				    "suffix -- skipping",
+				    path, options->suffix);
+				return -2;
+			}
+		} else if (!options->to_stdout) {
+			/*
+			 * Input file is suffixed, and we're not writing to
+			 * stdout.  Strip the suffix to get the path to the
+			 * output file.
+			 */
+			newpath = xmalloc((suffix - oldpath + 1) *
+					  sizeof(tchar));
+			if (newpath == NULL)
+				return -1;
+			tmemcpy(newpath, oldpath, suffix - oldpath);
+			newpath[suffix - oldpath] = '\0';
 		}
-		newpath = xmalloc((suffix - path + 1) * sizeof(tchar));
-		tmemcpy(newpath, path, suffix - path);
-		newpath[suffix - path] = '\0';
 	}
 
-	ret = xopen_for_read(path, &in);
+	ret = xopen_for_read(oldpath, options->force || options->to_stdout,
+			     &in);
 	if (ret != 0)
-		goto out_free_newpath;
+		goto out_free_paths;
 
 	if (!options->force && isatty(in.fd)) {
 		msg("Refusing to read compressed data from terminal.  "
@@ -317,7 +416,8 @@ decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
 		goto out_close_in;
 	}
 
-	ret = stat_file(&in, &stbuf, options->force || newpath == NULL);
+	ret = stat_file(&in, &stbuf, options->force || options->keep ||
+			oldpath == NULL || newpath == NULL);
 	if (ret != 0)
 		goto out_close_in;
 
@@ -325,15 +425,16 @@ decompress_file(struct deflate_decompressor *decompressor, const tchar *path,
 	if (ret != 0)
 		goto out_close_in;
 
+	/* TODO: need a streaming-friendly solution */
 	ret = map_file_contents(&in, stbuf.st_size);
 	if (ret != 0)
 		goto out_close_out;
 
-	ret = do_decompress(decompressor, &in, &out);
+	ret = do_decompress(decompressor, &in, &out, options);
 	if (ret != 0)
 		goto out_close_out;
 
-	if (path != NULL && newpath != NULL)
+	if (oldpath != NULL && newpath != NULL)
 		restore_metadata(&out, newpath, &stbuf);
 	ret = 0;
 out_close_out:
@@ -344,49 +445,44 @@ out_close_out:
 		tunlink(newpath);
 out_close_in:
 	xclose(&in);
-	if (ret == 0 && path != NULL && newpath != NULL && !options->keep)
-		tunlink(path);
-out_free_newpath:
-	free(newpath);
-out:
+	if (ret == 0 && oldpath != NULL && newpath != NULL && !options->keep)
+		tunlink(oldpath);
+out_free_paths:
+	if (newpath != path)
+		free(newpath);
+	if (oldpath != path)
+		free(oldpath);
 	return ret;
 }
 
 static int
-compress_file(struct deflate_compressor *compressor, const tchar *path,
+compress_file(struct libdeflate_compressor *compressor, const tchar *path,
 	      const struct options *options)
 {
 	tchar *newpath = NULL;
 	struct file_stream in;
 	struct file_stream out;
-	struct stat stbuf;
+	stat_t stbuf;
 	int ret;
 	int ret2;
 
 	if (path != NULL && !options->to_stdout) {
-		size_t path_nchars, suffix_nchars;
-
 		if (!options->force && has_suffix(path, options->suffix)) {
-			msg("%"TS": already has .%"TS" suffix -- skipping",
+			msg("%"TS": already has %"TS" suffix -- skipping",
 			    path, options->suffix);
-			ret = -2;
-			goto out;
+			return 0;
 		}
-		path_nchars = tstrlen(path);
-		suffix_nchars = tstrlen(options->suffix);
-		newpath = xmalloc((path_nchars + 1 + suffix_nchars + 1) *
-					sizeof(tchar));
-		tmemcpy(newpath, path, path_nchars);
-		newpath[path_nchars] = '.';
-		tmemcpy(&newpath[path_nchars + 1], options->suffix,
-			suffix_nchars + 1);
+		newpath = append_suffix(path, options->suffix);
+		if (newpath == NULL)
+			return -1;
 	}
 
-	ret = xopen_for_read(path, &in);
+	ret = xopen_for_read(path, options->force || options->to_stdout, &in);
 	if (ret != 0)
 		goto out_free_newpath;
 
-	ret = stat_file(&in, &stbuf, options->force || newpath == NULL);
+	ret = stat_file(&in, &stbuf, options->force || options->keep ||
+			path == NULL || newpath == NULL);
 	if (ret != 0)
 		goto out_close_in;
 
@@ -401,8 +497,9 @@ compress_file(struct deflate_compressor *compressor, const tchar *path,
 		goto out_close_out;
 	}
 
+	/* TODO: need a streaming-friendly solution */
 	ret = map_file_contents(&in, stbuf.st_size);
-	if (ret)
+	if (ret != 0)
 		goto out_close_out;
 
 	ret = do_compress(compressor, &in, &out);
@@ -424,26 +521,27 @@ out_close_in:
 		tunlink(path);
 out_free_newpath:
 	free(newpath);
-out:
 	return ret;
 }
 
 int
 tmain(int argc, tchar *argv[])
 {
+	tchar *default_file_list[] = { NULL };
 	struct options options;
 	int opt_char;
 	int i;
 	int ret;
 
-	program_invocation_name = get_filename(argv[0]);
+	begin_program(argv);
 
 	options.to_stdout = false;
 	options.decompress = is_gunzip();
 	options.force = false;
 	options.keep = false;
+	options.test = false;
 	options.compression_level = 6;
-	options.suffix = T("gz");
+	options.suffix = T(".gz");
 
 	while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
 		switch (opt_char) {
@@ -458,8 +556,8 @@ tmain(int argc, tchar *argv[])
 		case '9':
 			options.compression_level =
 				parse_compression_level(opt_char, toptarg);
-			if (options.compression_level == 0)
-				return -1;
+			if (options.compression_level < 0)
+				return 1;
 			break;
 		case 'c':
 			options.to_stdout = true;
@@ -476,8 +574,31 @@ tmain(int argc, tchar *argv[])
 		case 'k':
 			options.keep = true;
 			break;
+		case 'n':
+			/*
+			 * -n means don't save or restore the original filename
+			 *  in the gzip header.  Currently this implementation
+			 *  already behaves this way by default, so accept the
+			 *  option as a no-op.
+			 */
+			break;
 		case 'S':
 			options.suffix = toptarg;
+			if (options.suffix[0] == T('\0')) {
+				msg("invalid suffix");
+				return 1;
+			}
+			break;
+		case 't':
+			options.test = true;
+			options.decompress = true;
+			options.to_stdout = true;
+			/*
+			 * -t behaves just like the more commonly used -c
+			 * option, except that -t doesn't actually write
+			 * anything.  For ease of implementation, just pretend
+			 * that -c was specified too.
+			 */
 			break;
 		case 'V':
 			show_version();
@@ -492,20 +613,17 @@ tmain(int argc, tchar *argv[])
 	argc -= toptind;
 
 	if (argc == 0) {
-		show_usage(stderr);
-		return 1;
-	}
-	for (i = 0; i < argc; i++) {
-		if (argv[i][0] == '-' && argv[i][1] == '\0') {
-			msg("This implementation of gzip does not yet "
-			    "support reading from standard input.");
-			return 1;
-		}
+		argv = default_file_list;
+		argc = ARRAY_LEN(default_file_list);
+	} else {
+		for (i = 0; i < argc; i++)
+			if (argv[i][0] == '-' && argv[i][1] == '\0')
+				argv[i] = NULL;
 	}
 
 	ret = 0;
 	if (options.decompress) {
-		struct deflate_decompressor *d;
+		struct libdeflate_decompressor *d;
 
 		d = alloc_decompressor();
 		if (d == NULL)
@@ -514,9 +632,9 @@ tmain(int argc, tchar *argv[])
 		for (i = 0; i < argc; i++)
 			ret |= -decompress_file(d, argv[i], &options);
 
-		deflate_free_decompressor(d);
+		libdeflate_free_decompressor(d);
 	} else {
-		struct deflate_compressor *c;
+		struct libdeflate_compressor *c;
 
 		c = alloc_compressor(options.compression_level);
 		if (c == NULL)
@@ -525,7 +643,7 @@ tmain(int argc, tchar *argv[])
 		for (i = 0; i < argc; i++)
 			ret |= -compress_file(c, argv[i], &options);
 
-		deflate_free_compressor(c);
+		libdeflate_free_compressor(c);
 	}
 
 	/*

programs/prog_util.c

@@ -30,24 +30,38 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdarg.h>
-#include <time.h>
 #ifdef _WIN32
 #  include <windows.h>
 #else
 #  include <unistd.h>
 #  include <sys/mman.h>
-#  include <sys/time.h>
+#endif
+
+#ifndef O_BINARY
+#  define O_BINARY 0
+#endif
+#ifndef O_SEQUENTIAL
+#  define O_SEQUENTIAL 0
+#endif
+#ifndef O_NOFOLLOW
+#  define O_NOFOLLOW 0
+#endif
+#ifndef O_NONBLOCK
+#  define O_NONBLOCK 0
+#endif
+#ifndef O_NOCTTY
+#  define O_NOCTTY 0
 #endif
 
 /* The invocation name of the program (filename component only) */
-const tchar *program_invocation_name;
+const tchar *prog_invocation_name;
 
 static void
 do_msg(const char *format, bool with_errno, va_list va)
 {
 	int saved_errno = errno;
 
-	fprintf(stderr, "%"TS": ", program_invocation_name);
+	fprintf(stderr, "%"TS": ", prog_invocation_name);
 	vfprintf(stderr, format, va);
 	if (with_errno)
 		fprintf(stderr, ": %s\n", strerror(saved_errno));
@@ -91,28 +105,6 @@ xmalloc(size_t size)
 	return p;
 }
 
-/*
- * Retrieve the current time in nanoseconds since a start time which is fixed
- * for the duration of program execution but is otherwise unspecified
- */
-u64
-current_time(void)
-{
-#ifdef _WIN32
-	FILETIME ft;
-	GetSystemTimeAsFileTime(&ft);
-	return 100 * (((u64)ft.dwHighDateTime << 32) | ft.dwLowDateTime);
-#elif defined(HAVE_CLOCK_GETTIME)
-	struct timespec ts;
-	clock_gettime(CLOCK_MONOTONIC, &ts);
-	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
-#else
-	struct timeval tv;
-	gettimeofday(&tv, NULL);
-	return (1000000000 * (u64)tv.tv_sec) + (1000 * (u64)tv.tv_usec);
-#endif
-}
-
 /*
  * Retrieve a pointer to the filename component of the specified path.
  *
@@ -120,7 +112,7 @@ current_time(void)
  * properly for directories, since a path to a directory might have trailing
  * slashes.
  */
-const tchar *
+static const tchar *
 get_filename(const tchar *path)
 {
 	const tchar *slash = tstrrchr(path, '/');
@@ -134,6 +126,17 @@ get_filename(const tchar *path)
 	return path;
 }
 
+void
+begin_program(tchar *argv[])
+{
+	prog_invocation_name = get_filename(argv[0]);
+
+#ifdef FREESTANDING
+	/* This allows testing freestanding library builds. */
+	libdeflate_set_memory_allocator(malloc, free);
+#endif
+}
+
 /* Create a copy of 'path' surrounded by double quotes */
 static tchar *
 quote_path(const tchar *path)
@@ -153,8 +156,9 @@ quote_path(const tchar *path)
 
 /* Open a file for reading, or set up standard input for reading */
 int
-xopen_for_read(const tchar *path, struct file_stream *strm)
+xopen_for_read(const tchar *path, bool symlink_ok, struct file_stream *strm)
 {
+	strm->mmap_token = NULL;
 	strm->mmap_mem = NULL;
 
 	if (path == NULL) {
@@ -173,13 +177,18 @@ xopen_for_read(const tchar *path, struct file_stream *strm)
 	if (strm->name == NULL)
 		return -1;
 
-	strm->fd = topen(path, O_RDONLY | O_BINARY | O_NOFOLLOW);
+	strm->fd = topen(path, O_RDONLY | O_BINARY | O_NONBLOCK | O_NOCTTY |
+			 (symlink_ok ? 0 : O_NOFOLLOW) | O_SEQUENTIAL);
 	if (strm->fd < 0) {
 		msg_errno("Can't open %"TS" for reading", strm->name);
 		free(strm->name);
 		return -1;
 	}
 
+#if defined(HAVE_POSIX_FADVISE) && (O_SEQUENTIAL == 0)
+	(void)posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+
 	return 0;
 }
 
@@ -187,6 +196,9 @@ xopen_for_read(const tchar *path, struct file_stream *strm)
 int
 xopen_for_write(const tchar *path, bool overwrite, struct file_stream *strm)
 {
+	int ret = -1;
+
+	strm->mmap_token = NULL;
 	strm->mmap_mem = NULL;
 
 	if (path == NULL) {
@@ -216,11 +228,12 @@ retry:
 			if (!isatty(STDERR_FILENO) || !isatty(STDIN_FILENO)) {
 				msg("%"TS" already exists; use -f to overwrite",
 				    strm->name);
+				ret = -2; /* warning only */
 				goto err;
 			}
 			fprintf(stderr, "%"TS": %"TS" already exists; "
 				"overwrite? (y/n) ",
-				program_invocation_name, strm->name);
+				prog_invocation_name, strm->name);
 			if (getchar() != 'y') {
 				msg("Not overwriting.");
 				goto err;
@@ -237,13 +250,60 @@ retry:
 
 err:
 	free(strm->name);
-	return -1;
+	return ret;
+}
+
+/* Read the full contents of a file into memory */
+static int
+read_full_contents(struct file_stream *strm)
+{
+	size_t filled = 0;
+	size_t capacity = 4096;
+	char *buf;
+	int ret;
+
+	buf = xmalloc(capacity);
+	if (buf == NULL)
+		return -1;
+	do {
+		if (filled == capacity) {
+			char *newbuf;
+
+			if (capacity == SIZE_MAX)
+				goto oom;
+			capacity += MIN(SIZE_MAX - capacity, capacity);
+			newbuf = realloc(buf, capacity);
+			if (newbuf == NULL)
+				goto oom;
+			buf = newbuf;
+		}
+		ret = xread(strm, &buf[filled], capacity - filled);
+		if (ret < 0)
+			goto err;
+		filled += ret;
+	} while (ret != 0);
+
+	strm->mmap_mem = buf;
+	strm->mmap_size = filled;
+	return 0;
+
+err:
+	free(buf);
+	return ret;
+oom:
+	msg("Out of memory!  %"TS" is too large to be processed by "
+	    "this program as currently implemented.", strm->name);
+	ret = -1;
+	goto err;
 }
 
 /* Map the contents of a file into memory */
 int
 map_file_contents(struct file_stream *strm, u64 size)
 {
+	if (size == 0) /* mmap isn't supported on empty files */
+		return read_full_contents(strm);
+
 	if (size > SIZE_MAX) {
 		msg("%"TS" is too large to be processed by this program",
 		    strm->name);
@@ -254,8 +314,11 @@ map_file_contents(struct file_stream *strm, u64 size)
 				(HANDLE)(intptr_t)_get_osfhandle(strm->fd),
 				NULL, PAGE_READONLY, 0, 0, NULL);
 	if (strm->mmap_token == NULL) {
+		DWORD err = GetLastError();
+		if (err == ERROR_BAD_EXE_FORMAT) /* mmap unsupported */
+			return read_full_contents(strm);
 		msg("Unable create file mapping for %"TS": Windows error %u",
-		    strm->name, (unsigned int)GetLastError());
+		    strm->name, (unsigned int)err);
 		return -1;
 	}
 
@@ -267,10 +330,12 @@ map_file_contents(struct file_stream *strm, u64 size)
 		CloseHandle((HANDLE)strm->mmap_token);
 		return -1;
 	}
-#else
+#else /* _WIN32 */
 	strm->mmap_mem = mmap(NULL, size, PROT_READ, MAP_SHARED, strm->fd, 0);
 	if (strm->mmap_mem == MAP_FAILED) {
 		strm->mmap_mem = NULL;
+		if (errno == ENODEV) /* mmap isn't supported on this file */
+			return read_full_contents(strm);
 		if (errno == ENOMEM) {
 			msg("%"TS" is too large to be processed by this "
 			    "program", strm->name);
@@ -280,7 +345,13 @@ map_file_contents(struct file_stream *strm, u64 size)
 		}
 		return -1;
 	}
+
+#ifdef HAVE_POSIX_MADVISE
+	(void)posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL);
 #endif
+	strm->mmap_token = strm; /* anything that's not NULL */
+
+#endif /* !_WIN32 */
 	strm->mmap_size = size;
 	return 0;
 }
@@ -300,6 +371,8 @@ xread(struct file_stream *strm, void *buf, size_t count)
 		if (res == 0)
 			break;
 		if (res < 0) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
 			msg_errno("Error reading from %"TS, strm->name);
 			return -1;
 		}
@@ -309,39 +382,6 @@ xread(struct file_stream *strm, void *buf, size_t count)
 	return orig_count - count;
 }
 
-/* Skip over 'count' bytes from a file, returning 0 on success or -1 on error */
-int
-skip_bytes(struct file_stream *strm, size_t count)
-{
-	size_t bufsize;
-	char *buffer;
-	ssize_t ret;
-
-	if (count == 0)
-		return 0;
-
-	bufsize = MIN(count, 4096);
-	buffer = xmalloc(bufsize);
-	if (buffer == NULL)
-		return -1;
-	do {
-		size_t n = MIN(count, bufsize);
-		ret = xread(strm, buffer, n);
-		if (ret < 0)
-			goto out;
-		if (ret != n) {
-			msg("%"TS": unexpected end-of-file", strm->name);
-			ret = -1;
-			goto out;
-		}
-		count -= ret;
-	} while (count != 0);
-	ret = 0;
-out:
-	free(buffer);
-	return ret;
-}
-
 /* Write to a file, returning 0 if all bytes were written or -1 on error */
 int
 full_write(struct file_stream *strm, const void *buf, size_t count)
@@ -365,23 +405,27 @@ int
 xclose(struct file_stream *strm)
 {
 	int ret = 0;
-	if (strm->fd >= 0 && !strm->is_standard_stream) {
+
+	if (!strm->is_standard_stream) {
 		if (close(strm->fd) != 0) {
 			msg_errno("Error closing %"TS, strm->name);
 			ret = -1;
 		}
 		free(strm->name);
-
-		if (strm->mmap_mem != NULL) {
-#ifdef _WIN32
-			UnmapViewOfFile(strm->mmap_mem);
-			CloseHandle((HANDLE)strm->mmap_token);
-#else
-			munmap(strm->mmap_mem, strm->mmap_size);
-#endif
-			strm->mmap_mem = NULL;
-		}
 	}
+
+	if (strm->mmap_token != NULL) {
+#ifdef _WIN32
+		UnmapViewOfFile(strm->mmap_mem);
+		CloseHandle((HANDLE)strm->mmap_token);
+#else
+		munmap(strm->mmap_mem, strm->mmap_size);
+#endif
+		strm->mmap_token = NULL;
+	} else {
+		free(strm->mmap_mem);
+	}
+	strm->mmap_mem = NULL;
 	strm->fd = -1;
 	strm->name = NULL;
 	return ret;
@@ -389,36 +433,48 @@ xclose(struct file_stream *strm)
 
 /*
  * Parse the compression level given on the command line, returning the
- * compression level on success or 0 on error
+ * compression level on success or -1 on error
  */
 int
 parse_compression_level(tchar opt_char, const tchar *arg)
 {
-	unsigned long level = opt_char - '0';
-	const tchar *p;
+	int level;
 
 	if (arg == NULL)
 		arg = T("");
 
-	for (p = arg; *p >= '0' && *p <= '9'; p++)
-		level = (level * 10) + (*p - '0');
+	if (opt_char < '0' || opt_char > '9')
+		goto invalid;
+	level = opt_char - '0';
 
-	if (level < 1 || level > 12 || *p != '\0') {
-		msg("Invalid compression level: \"%"TC"%"TS"\".  "
-		    "Must be an integer in the range [1, 12].", opt_char, arg);
-		return 0;
+	if (arg[0] != '\0') {
+		if (arg[0] < '0' || arg[0] > '9')
+			goto invalid;
+		if (arg[1] != '\0')	/* Levels are at most 2 digits */
+			goto invalid;
+		if (level == 0)		/* Don't allow arguments like "-01" */
+			goto invalid;
+		level = (level * 10) + (arg[0] - '0');
 	}
 
+	if (level < 0 || level > 12)
+		goto invalid;
+
 	return level;
+
+invalid:
+	msg("Invalid compression level: \"%"TC"%"TS"\".  "
+	    "Must be an integer in the range [0, 12].", opt_char, arg);
+	return -1;
 }
 
 /* Allocate a new DEFLATE compressor */
-struct deflate_compressor *
+struct libdeflate_compressor *
 alloc_compressor(int level)
 {
-	struct deflate_compressor *c;
+	struct libdeflate_compressor *c;
 
-	c = deflate_alloc_compressor(level);
+	c = libdeflate_alloc_compressor(level);
 	if (c == NULL) {
 		msg_errno("Unable to allocate compressor with "
 			  "compression level %d", level);
@@ -427,12 +483,12 @@ alloc_compressor(int level)
 }
 
 /* Allocate a new DEFLATE decompressor */
-struct deflate_decompressor *
+struct libdeflate_decompressor *
 alloc_decompressor(void)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 	if (d == NULL)
 		msg_errno("Unable to allocate decompressor");
 

programs/prog_util.h

@@ -38,8 +38,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#ifndef _WIN32
+#  include <sys/types.h>
+#endif
 
-#include "common_defs.h"
+#include "../common/common_defs.h"
 
 #ifdef __GNUC__
 # define _printf(str_idx, args_idx)	\
@@ -48,6 +51,27 @@
 # define _printf(str_idx, args_idx)
 #endif
 
+#ifdef _MSC_VER
+/*
+ * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
+ * inttypes.h.  Work around this by defining the PRI* macros ourselves.
+ */
+# define PRIu8  "hhu"
+# define PRIu16 "hu"
+# define PRIu32 "u"
+# define PRIu64 "llu"
+# define PRIi8  "hhi"
+# define PRIi16 "hi"
+# define PRIi32 "i"
+# define PRIi64 "lli"
+# define PRIx8  "hhx"
+# define PRIx16 "hx"
+# define PRIx32 "x"
+# define PRIx64 "llx"
+#else
+# include <inttypes.h>
+#endif
+
 #ifdef _WIN32
 
 /*
@@ -57,7 +81,7 @@
  */
 
 #include <wchar.h>
-extern int wmain(int argc, wchar_t **argv);
+int wmain(int argc, wchar_t **argv);
 #  define	tmain		wmain
 #  define	tchar		wchar_t
 #  define	_T(text)	L##text
@@ -68,27 +92,23 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		_wopen
 #  define	tstrchr		wcschr
 #  define	tstrcmp		wcscmp
-#  define	tstrcpy		wcscpy
 #  define	tstrlen		wcslen
 #  define	tstrrchr	wcsrchr
 #  define	tstrtoul	wcstoul
 #  define	tstrxcmp	wcsicmp
 #  define	tunlink		_wunlink
-#  define	tutimbuf	_utimbuf
-#  define	tutime		_wutime
+#  define	tutimbuf	__utimbuf64
+#  define	tutime		_wutime64
+#  define	tstat		_wstat64
+#  define	tfstat		_fstat64
+#  define	stat_t		struct _stat64
 #  ifdef _MSC_VER
-     /* Standard file descriptors?  What are those? */
 #    define	STDIN_FILENO	0
 #    define	STDOUT_FILENO	1
 #    define	STDERR_FILENO	2
-
-     /* Fix the broken stat-related definitions. */
-#    define	fstat		_fstat
-#    define	stat		_stat
 #    define	S_ISREG(m)      (((m) & S_IFMT) == S_IFREG)
 #    define	S_ISDIR(m)      (((m) & S_IFMT) == S_IFDIR)
 #  endif
-#  define	O_NOFOLLOW	0
 
 #else /* _WIN32 */
 
@@ -103,7 +123,6 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	topen		open
 #  define	tstrchr		strchr
 #  define	tstrcmp		strcmp
-#  define	tstrcpy		strcpy
 #  define	tstrlen		strlen
 #  define	tstrrchr	strrchr
 #  define	tstrtoul	strtoul
@@ -111,20 +130,20 @@ extern int wmain(int argc, wchar_t **argv);
 #  define	tunlink		unlink
 #  define	tutimbuf	utimbuf
 #  define	tutime		utime
-#  define	O_BINARY	0
+#  define	tstat		stat
+#  define	tfstat		fstat
+#  define	stat_t		struct stat
 
 #endif /* !_WIN32 */
 
-extern const tchar *program_invocation_name;
+extern const tchar *prog_invocation_name;
 
-extern void _printf(1, 2) msg(const char *fmt, ...);
-extern void _printf(1, 2) msg_errno(const char *fmt, ...);
+void _printf(1, 2) msg(const char *fmt, ...);
+void _printf(1, 2) msg_errno(const char *fmt, ...);
 
-extern void *xmalloc(size_t size);
+void *xmalloc(size_t size);
 
-extern u64 current_time(void);
-
-extern const tchar *get_filename(const tchar *path);
+void begin_program(tchar *argv[]);
 
 struct file_stream {
 	int fd;
@@ -135,27 +154,26 @@ struct file_stream {
 	size_t mmap_size;
 };
 
-extern int xopen_for_read(const tchar *path, struct file_stream *strm);
-extern int xopen_for_write(const tchar *path, bool force,
-			   struct file_stream *strm);
-extern int map_file_contents(struct file_stream *strm, u64 size);
+int xopen_for_read(const tchar *path, bool symlink_ok,
+		   struct file_stream *strm);
+int xopen_for_write(const tchar *path, bool force, struct file_stream *strm);
+int map_file_contents(struct file_stream *strm, u64 size);
 
-extern ssize_t xread(struct file_stream *strm, void *buf, size_t count);
-extern int skip_bytes(struct file_stream *strm, size_t count);
-extern int full_write(struct file_stream *strm, const void *buf, size_t count);
+ssize_t xread(struct file_stream *strm, void *buf, size_t count);
+int full_write(struct file_stream *strm, const void *buf, size_t count);
 
-extern int xclose(struct file_stream *strm);
+int xclose(struct file_stream *strm);
 
-extern int parse_compression_level(tchar opt_char, const tchar *arg);
+int parse_compression_level(tchar opt_char, const tchar *arg);
 
-extern struct deflate_compressor *alloc_compressor(int level);
-extern struct deflate_decompressor *alloc_decompressor(void);
+struct libdeflate_compressor *alloc_compressor(int level);
+struct libdeflate_decompressor *alloc_decompressor(void);
 
 /* tgetopt.c */
 
 extern tchar *toptarg;
 extern int toptind, topterr, toptopt;
 
-extern int tgetopt(int argc, tchar *argv[], const tchar *optstring);
+int tgetopt(int argc, tchar *argv[], const tchar *optstring);
 
 #endif /* PROGRAMS_PROG_UTIL_H */

programs/test_checksums.c

@@ -0,0 +1,196 @@
+/*
+ * test_checksums.c
+ *
+ * Verify that libdeflate's Adler-32 and CRC-32 functions produce the same
+ * results as their zlib equivalents.
+ */
+
+#include <stdlib.h>
+#include <time.h>
+
+#include "test_util.h"
+
+static unsigned int rng_seed;
+
+typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
+
+static u32
+adler32_libdeflate(u32 adler, const void *buf, size_t len)
+{
+	return libdeflate_adler32(adler, buf, len);
+}
+
+static u32
+crc32_libdeflate(u32 crc, const void *buf, size_t len)
+{
+	return libdeflate_crc32(crc, buf, len);
+}
+
+static u32
+adler32_zlib(u32 adler, const void *buf, size_t len)
+{
+	return adler32(adler, buf, len);
+}
+
+static u32
+crc32_zlib(u32 crc, const void *buf, size_t len)
+{
+	return crc32(crc, buf, len);
+}
+
+static u32
+select_initial_crc(void)
+{
+	if (rand() & 1)
+		return 0;
+	return ((u32)rand() << 16) | rand();
+}
+
+static u32
+select_initial_adler(void)
+{
+	u32 lo, hi;
+
+	if (rand() & 1)
+		return 1;
+
+	lo = (rand() % 4 == 0 ? 65520 : rand() % 65521);
+	hi = (rand() % 4 == 0 ? 65520 : rand() % 65521);
+	return (hi << 16) | lo;
+}
+
+static void
+test_initial_values(cksum_fn_t cksum, u32 expected)
+{
+	ASSERT(cksum(0, NULL, 0) == expected);
+	if (cksum != adler32_zlib) /* broken */
+		ASSERT(cksum(0, NULL, 1) == expected);
+	ASSERT(cksum(0, NULL, 1234) == expected);
+	ASSERT(cksum(1234, NULL, 0) == expected);
+	ASSERT(cksum(1234, NULL, 1234) == expected);
+}
+
+static void
+test_multipart(const u8 *buffer, size_t size, const char *name,
+	       cksum_fn_t cksum, u32 v, u32 expected)
+{
+	size_t division = rand() % (size + 1);
+	v = cksum(v, buffer, division);
+	v = cksum(v, buffer + division, size - division);
+	if (v != expected) {
+		fprintf(stderr, "%s checksum failed multipart test\n", name);
+		ASSERT(0);
+	}
+}
+
+static void
+test_checksums(const void *buffer, size_t size, const char *name,
+	       cksum_fn_t cksum1, cksum_fn_t cksum2, u32 initial_value)
+{
+	u32 v1 = cksum1(initial_value, buffer, size);
+	u32 v2 = cksum2(initial_value, buffer, size);
+
+	if (v1 != v2) {
+		fprintf(stderr, "%s checksum mismatch\n", name);
+		fprintf(stderr, "initial_value=0x%08"PRIx32", buffer=%p, "
+			"size=%zu, buffer=", initial_value, buffer, size);
+		for (size_t i = 0; i < MIN(size, 256); i++)
+			fprintf(stderr, "%02x", ((const u8 *)buffer)[i]);
+		if (size > 256)
+			fprintf(stderr, "...");
+		fprintf(stderr, "\n");
+		ASSERT(0);
+	}
+
+	if ((rand() & 15) == 0) {
+		test_multipart(buffer, size, name, cksum1, initial_value, v1);
+		test_multipart(buffer, size, name, cksum2, initial_value, v1);
+	}
+}
+
+static void
+test_crc32(const void *buffer, size_t size, u32 initial_value)
+{
+	test_checksums(buffer, size, "CRC-32",
+		       crc32_libdeflate, crc32_zlib, initial_value);
+}
+
+static void
+test_adler32(const void *buffer, size_t size, u32 initial_value)
+{
+	test_checksums(buffer, size, "Adler-32",
+		       adler32_libdeflate, adler32_zlib, initial_value);
+}
+
+static void test_random_buffers(u8 *buffer, u8 *guarded_buf_end,
+				size_t limit, u32 num_iter)
+{
+	for (u32 i = 0; i < num_iter; i++) {
+		size_t start = rand() % limit;
+		size_t len = rand() % (limit - start);
+		u32 a0 = select_initial_adler();
+		u32 c0 = select_initial_crc();
+
+		for (size_t j = start; j < start + len; j++)
+			buffer[j] = rand();
+
+		/* Test with chosen size and alignment */
+		test_adler32(&buffer[start], len, a0);
+		test_crc32(&buffer[start], len, c0);
+
+		/* Test with chosen size, with guard page after input buffer */
+		memcpy(guarded_buf_end - len, &buffer[start], len);
+		test_adler32(guarded_buf_end - len, len, a0);
+		test_crc32(guarded_buf_end - len, len, c0);
+	}
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 *buffer = xmalloc(32768);
+	u8 *guarded_buf_start, *guarded_buf_end;
+
+	begin_program(argv);
+
+	alloc_guarded_buffer(32768, &guarded_buf_start, &guarded_buf_end);
+
+	rng_seed = time(NULL);
+	srand(rng_seed);
+
+	test_initial_values(adler32_libdeflate, 1);
+	test_initial_values(adler32_zlib, 1);
+	test_initial_values(crc32_libdeflate, 0);
+	test_initial_values(crc32_zlib, 0);
+
+	/* Test different buffer sizes and alignments */
+	test_random_buffers(buffer, guarded_buf_end, 256, 5000);
+	test_random_buffers(buffer, guarded_buf_end, 1024, 500);
+	test_random_buffers(buffer, guarded_buf_end, 32768, 50);
+
+	/*
+	 * Test Adler-32 overflow cases.  For example, given all 0xFF bytes and
+	 * the highest possible initial (s1, s2) of (65520, 65520), then s2 if
+	 * stored as a 32-bit unsigned integer will overflow if > 5552 bytes are
+	 * processed.  Implementations must make sure to reduce s2 modulo 65521
+	 * before that point.  Also, some implementations make use of 16-bit
+	 * counters which can overflow earlier.
+	 */
+	memset(buffer, 0xFF, 32768);
+	for (u32 i = 0; i < 20; i++) {
+		u32 initial_value;
+
+		if (i == 0)
+			initial_value = ((u32)65520 << 16) | 65520;
+		else
+			initial_value = select_initial_adler();
+
+		test_adler32(buffer, 5553, initial_value);
+		test_adler32(buffer, rand() % 32769, initial_value);
+		buffer[rand() % 32768] = 0xFE;
+	}
+
+	free(buffer);
+	free_guarded_buffer(guarded_buf_start, guarded_buf_end);
+	return 0;
+}

programs/test_custom_malloc.c

@@ -0,0 +1,85 @@
+/*
+ * test_custom_malloc.c
+ *
+ * Test libdeflate_set_memory_allocator().
+ * Also test injecting allocation failures.
+ */
+
+#include "test_util.h"
+
+static int malloc_count = 0;
+static int free_count = 0;
+
+static void *do_malloc(size_t size)
+{
+	malloc_count++;
+	return malloc(size);
+}
+
+static void *do_fail_malloc(size_t size)
+{
+	malloc_count++;
+	return NULL;
+}
+
+static void do_free(void *ptr)
+{
+	free_count++;
+	free(ptr);
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	int level;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+
+	begin_program(argv);
+
+	/* Test that the custom allocator is actually used when requested. */
+
+	libdeflate_set_memory_allocator(do_malloc, do_free);
+	ASSERT(malloc_count == 0);
+	ASSERT(free_count == 0);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c != NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+		libdeflate_free_compressor(c);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 1);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+	libdeflate_free_decompressor(d);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 1);
+
+	/* As long as we're here, also test injecting allocation failures. */
+
+	libdeflate_set_memory_allocator(do_fail_malloc, do_free);
+
+	for (level = 0; level <= 12; level++) {
+		malloc_count = free_count = 0;
+		c = libdeflate_alloc_compressor(level);
+		ASSERT(c == NULL);
+		ASSERT(malloc_count == 1);
+		ASSERT(free_count == 0);
+	}
+
+	malloc_count = free_count = 0;
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d == NULL);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 0);
+
+	return 0;
+}

programs/test_incomplete_codes.c

@@ -0,0 +1,385 @@
+/*
+ * test_incomplete_codes.c
+ *
+ * Test that the decompressor accepts incomplete Huffman codes in certain
+ * specific cases.
+ */
+
+#include "test_util.h"
+
+static void
+verify_decompression_libdeflate(const u8 *in, size_t in_nbytes,
+				u8 *out, size_t out_nbytes_avail,
+				const u8 *expected_out,
+				size_t expected_out_nbytes)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	size_t actual_out_nbytes;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	res = libdeflate_deflate_decompress(d, in, in_nbytes,
+					    out, out_nbytes_avail,
+					    &actual_out_nbytes);
+	ASSERT(res == LIBDEFLATE_SUCCESS);
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	libdeflate_free_decompressor(d);
+}
+
+static void
+verify_decompression_zlib(const u8 *in, size_t in_nbytes,
+			  u8 *out, size_t out_nbytes_avail,
+			  const u8 *expected_out, size_t expected_out_nbytes)
+{
+	z_stream z;
+	int res;
+	size_t actual_out_nbytes;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	z.next_in = (void *)in;
+	z.avail_in = in_nbytes;
+	z.next_out = (void *)out;
+	z.avail_out = out_nbytes_avail;
+	res = inflate(&z, Z_FINISH);
+	ASSERT(res == Z_STREAM_END);
+	actual_out_nbytes = out_nbytes_avail - z.avail_out;
+	ASSERT(actual_out_nbytes == expected_out_nbytes);
+	ASSERT(memcmp(out, expected_out, actual_out_nbytes) == 0);
+
+	inflateEnd(&z);
+}
+
+static void
+verify_decompression(const u8 *in, size_t in_nbytes,
+		     u8 *out, size_t out_nbytes_avail,
+		     const u8 *expected_out, size_t expected_out_nbytes)
+{
+	verify_decompression_libdeflate(in, in_nbytes, out, out_nbytes_avail,
+					expected_out, expected_out_nbytes);
+	verify_decompression_zlib(in, in_nbytes, out, out_nbytes_avail,
+				  expected_out, expected_out_nbytes);
+
+}
+
+/* Test that an empty offset code is accepted. */
+static void
+test_empty_offset_code(void)
+{
+	static const u8 expected_out[] = { 'A', 'B', 'A', 'A' };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Generate a DEFLATE stream containing a "dynamic Huffman" block
+	 * containing literals, but no offsets; and having an empty offset code
+	 * (all codeword lengths set to 0).
+	 *
+	 * Litlen code:
+	 *	litlensym_A			freq=3 len=1 codeword= 0
+	 *	litlensym_B			freq=1 len=2 codeword=01
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..'A'-1]	= 0	presym_18
+	 *	['A']		= 1	presym_1
+	 *	['B']		= 2	presym_2
+	 *	['B'+1..255]	= 0	presym_18 presym_18
+	 *	[256]		= 2	presym_2
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=3 codeword=011
+	 *	presym_1	freq=1 len=3 codeword=111
+	 *	presym_2	freq=2 len=2 codeword= 01
+	 *	presym_18	freq=3 len=1 codeword=  0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_0: len=3 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 3, 3));	/* presym_1: len=3 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 54, 7));	/* presym_18, 65 zeroes */
+	ASSERT(put_bits(&os, 0x7, 3));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 89, 7));	/* presym_18, 100 zeroes */
+	ASSERT(put_bits(&os, 0x0, 1) &&
+	       put_bits(&os, 78, 7));	/* presym_18, 89 zeroes */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 3));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_B */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_A */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that a litrunlen code containing only one symbol is accepted. */
+static void
+test_singleton_litrunlen_code(void)
+{
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	/*
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..256]	= 0	presym_18 presym_18
+	 *	[256]		= 1	presym_1
+	 *	[257]		= 0	presym_0
+	 *
+	 * Precode:
+	 *	presym_0	freq=1 len=2 codeword=01
+	 *	presym_1	freq=1 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 13; i++)	/* presym_{8,...,14}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		ASSERT(put_bits(&os, 0, 1) &&	/* presym_18, 128 zeroes */
+		       put_bits(&os, 117, 7));
+	}
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_0 */
+
+	/* Litlen symbols */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_256 (end-of-block) */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out), in, 0);
+}
+
+/* Test that an offset code containing only one symbol is accepted. */
+static void
+test_singleton_offset_code(void)
+{
+	static const u8 expected_out[] = { 255, 255, 255, 255 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_255			freq=1 len=1 codeword= 0
+	 *	litlensym_256 (end-of-block)	freq=1 len=2 codeword=01
+	 *	litlensym_257 (len 3)		freq=1 len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_0 (offset 0)		freq=1 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..254] = 0	presym_{18,18}
+	 *	[255]	 = 1	presym_1
+	 *	[256]	 = 1	presym_2
+	 *	[257]	 = 1	presym_2
+	 *	[258]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=2 codeword=01
+	 *	presym_2	freq=2 len=2 codeword=11
+	 *	presym_18	freq=2 len=1 codeword= 0
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	for (i = 0; i < 12; i++)	/* presym_{0,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x0, 1) &&	/* presym_18, 127 zeroes */
+	       put_bits(&os, 116, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x3, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_1 */
+
+	/* Literal */
+	ASSERT(put_bits(&os, 0x0, 1));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_0 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+/* Test that an offset code containing only one symbol is accepted, even if that
+ * symbol is not symbol 0.  The codeword should be '0' in either case. */
+static void
+test_singleton_offset_code_notsymzero(void)
+{
+	static const u8 expected_out[] = { 254, 255, 254, 255, 254 };
+	u8 in[128];
+	u8 out[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_254			len=2 codeword=00
+	 *	litlensym_255			len=2 codeword=10
+	 *	litlensym_256 (end-of-block)	len=2 codeword=01
+	 *	litlensym_257 (len 3)		len=2 codeword=11
+	 * Offset code:
+	 *	offsetsym_1 (offset 2)		len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..253] = 0	presym_{18,18}
+	 *	[254]	 = 2	presym_2
+	 *	[255]	 = 2	presym_2
+	 *	[256]	 = 2	presym_2
+	 *	[257]	 = 2	presym_2
+	 *	[258]	 = 0	presym_0
+	 *	[259]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_0	len=2 codeword=00
+	 *	presym_1	len=2 codeword=10
+	 *	presym_2	len=2 codeword=01
+	 *	presym_18	len=2 codeword=11
+	 */
+
+	ASSERT(put_bits(&os, 1, 5));	/* num_litlen_syms: 1 + 257 */
+	ASSERT(put_bits(&os, 1, 5));	/* num_offset_syms: 1 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_18: len=2 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_0: len=2 */
+	for (i = 0; i < 11; i++)	/* presym_{8,...,13}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 2, 3));	/* presym_2: len=2 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_14: len=0 */
+	ASSERT(put_bits(&os, 2, 3));	/* presym_1: len=2 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x3, 2) &&	/* presym_18, 126 zeroes */
+	       put_bits(&os, 115, 7));
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x1, 2));	/* presym_2 */
+	ASSERT(put_bits(&os, 0x0, 2));	/* presym_0 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* presym_1 */
+
+	/* Literals */
+	ASSERT(put_bits(&os, 0x0, 2));	/* litlensym_254 */
+	ASSERT(put_bits(&os, 0x2, 2));	/* litlensym_255 */
+
+	/* Match */
+	ASSERT(put_bits(&os, 0x3, 2));	/* litlensym_257 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* offsetsym_1 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 2));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	verify_decompression(in, os.next - in, out, sizeof(out),
+			     expected_out, sizeof(expected_out));
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	begin_program(argv);
+
+	test_empty_offset_code();
+	test_singleton_litrunlen_code();
+	test_singleton_offset_code();
+	test_singleton_offset_code_notsymzero();
+
+	return 0;
+}

programs/test_litrunlen_overflow.c

@@ -0,0 +1,72 @@
+/*
+ * test_litrunlen_overflow.c
+ *
+ * Regression test for commit f2f0df727444 ("deflate_compress: fix corruption
+ * with long literal run").  Try to compress a file longer than 65535 bytes
+ * where no 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes
+ * apart, and the distribution of bytes remains constant throughout, and yet not
+ * all bytes are used so the data is still slightly compressible.  There will be
+ * no matches in this data, but the compressor should still output a compressed
+ * block, and this block should contain more than 65535 consecutive literals,
+ * which triggered the bug.
+ *
+ * Note: on random data, this situation is extremely unlikely if the compressor
+ * uses all matches it finds, since random data will on average have a 3-byte
+ * match every (256**3)/32768 = 512 bytes.
+ */
+
+#include "test_util.h"
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t data_size = 2 * 250 * 251;
+	u8 *orig_data, *compressed_data, *decompressed_data;
+	int i, stride, multiple, j = 0;
+	struct libdeflate_decompressor *d;
+	static const int levels[] = { 3, 6, 12 };
+
+	begin_program(argv);
+
+	orig_data = xmalloc(data_size);
+	compressed_data = xmalloc(data_size);
+	decompressed_data = xmalloc(data_size);
+
+	for (i = 0; i < 2; i++) {
+		for (stride = 1; stride < 251; stride++) {
+			for (multiple = 0; multiple < 251; multiple++)
+				orig_data[j++] = (stride * multiple) % 251;
+		}
+	}
+	ASSERT(j == data_size);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(levels); i++) {
+		struct libdeflate_compressor *c;
+		size_t csize;
+		enum libdeflate_result res;
+
+		c = libdeflate_alloc_compressor(levels[i]);
+		ASSERT(c != NULL);
+
+		csize = libdeflate_deflate_compress(c, orig_data, data_size,
+						    compressed_data, data_size);
+		ASSERT(csize > 0 && csize < data_size);
+
+		res = libdeflate_deflate_decompress(d, compressed_data, csize,
+						    decompressed_data,
+						    data_size, NULL);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(memcmp(orig_data, decompressed_data, data_size) == 0);
+
+		libdeflate_free_compressor(c);
+	}
+
+	libdeflate_free_decompressor(d);
+	free(orig_data);
+	free(compressed_data);
+	free(decompressed_data);
+	return 0;
+}

programs/test_slow_decompression.c

@@ -0,0 +1,472 @@
+/*
+ * test_slow_decompression.c
+ *
+ * Test how quickly libdeflate decompresses degenerate/malicious compressed data
+ * streams that start new Huffman blocks extremely frequently.
+ */
+
+#include "test_util.h"
+
+/*
+ * Generate a DEFLATE stream containing all empty "static Huffman" blocks.
+ *
+ * libdeflate used to decompress this very slowly (~1000x slower than typical
+ * data), but now it's much faster (only ~2x slower than typical data) because
+ * now it skips rebuilding the decode tables for the static Huffman codes when
+ * they're already loaded into the decompressor.
+ */
+static void
+generate_empty_static_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (put_bits(&os, 0, 1) &&	/* BFINAL: 0 */
+	       put_bits(&os, 1, 2) &&	/* BTYPE: STATIC_HUFFMAN */
+	       put_bits(&os, 0, 7))	/* litlensym_256 (end-of-block) */
+		;
+}
+
+static bool
+generate_empty_dynamic_huffman_block(struct output_bitstream *os)
+{
+	int i;
+
+	if (!put_bits(os, 0, 1))	/* BFINAL: 0 */
+		return false;
+	if (!put_bits(os, 2, 2))	/* BTYPE: DYNAMIC_HUFFMAN */
+		return false;
+
+	/*
+	 * Write a minimal Huffman code, then the end-of-block symbol.
+	 *
+	 * Litlen code:
+	 *	litlensym_256 (end-of-block)	freq=1 len=1 codeword=0
+	 * Offset code:
+	 *	offsetsym_0 (unused)		freq=0 len=1 codeword=0
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..255] = 0	presym_{18,18}
+	 *	[256]	 = 1	presym_1
+	 *	[257]	 = 1	presym_1
+	 *
+	 * Precode:
+	 *	presym_1	freq=2 len=1 codeword=0
+	 *	presym_18	freq=2 len=1 codeword=1
+	 */
+
+	if (!put_bits(os, 0, 5))	/* num_litlen_syms: 0 + 257 */
+		return false;
+	if (!put_bits(os, 0, 5))	/* num_offset_syms: 0 + 1 */
+		return false;
+	if (!put_bits(os, 14, 4))	/* num_explicit_precode_lens: 14 + 4 */
+		return false;
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++) {	/* presym_{16,17}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_18: len=1 */
+		return false;
+	for (i = 0; i < 14; i++) {	/* presym_{0,...,14}: len=0 */
+		if (!put_bits(os, 0, 3))
+			return false;
+	}
+	if (!put_bits(os, 1, 3))	/* presym_1: len=1 */
+		return false;
+
+	/* Litlen and offset codeword lengths */
+	for (i = 0; i < 2; i++) {
+		if (!put_bits(os, 1, 1) || /* presym_18, 128 zeroes */
+		    !put_bits(os, 117, 7))
+			return false;
+	}
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	if (!put_bits(os, 0, 1))	/* presym_1 */
+		return false;
+	/* Done writing the Huffman codes */
+
+	return put_bits(os, 0, 1);	/* litlensym_256 (end-of-block) */
+}
+
+/*
+ * Generate a DEFLATE stream containing all empty "dynamic Huffman" blocks.
+ *
+ * This is the worst known case currently, being ~100x slower to decompress than
+ * typical data.
+ */
+static void
+generate_empty_dynamic_huffman_blocks(u8 *p, size_t len)
+{
+	struct output_bitstream os = { .next = p, .end = p + len };
+
+	while (generate_empty_dynamic_huffman_block(&os))
+		;
+}
+
+#define NUM_ITERATIONS	100
+
+static u64
+do_test_libdeflate(const char *input_type, const u8 *in, size_t in_nbytes,
+		   u8 *out, size_t out_nbytes_avail)
+{
+	struct libdeflate_decompressor *d;
+	enum libdeflate_result res;
+	u64 t;
+	int i;
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		res = libdeflate_deflate_decompress(d, in, in_nbytes, out,
+						    out_nbytes_avail, NULL);
+		ASSERT(res == LIBDEFLATE_BAD_DATA ||
+		       res == LIBDEFLATE_INSUFFICIENT_SPACE);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, libdeflate]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	libdeflate_free_decompressor(d);
+	return t;
+}
+
+static u64
+do_test_zlib(const char *input_type, const u8 *in, size_t in_nbytes,
+	     u8 *out, size_t out_nbytes_avail)
+{
+	z_stream z;
+	int res;
+	u64 t;
+	int i;
+
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+
+	t = timer_ticks();
+	for (i = 0; i < NUM_ITERATIONS; i++) {
+		inflateReset(&z);
+		z.next_in = (void *)in;
+		z.avail_in = in_nbytes;
+		z.next_out = out;
+		z.avail_out = out_nbytes_avail;
+		res = inflate(&z, Z_FINISH);
+		ASSERT(res == Z_BUF_ERROR || res == Z_DATA_ERROR);
+	}
+	t = timer_ticks() - t;
+
+	printf("[%s, zlib      ]: %"PRIu64" KB/s\n", input_type,
+	       timer_KB_per_s((u64)in_nbytes * NUM_ITERATIONS, t));
+
+	inflateEnd(&z);
+	return t;
+}
+
+/*
+ * Test case from https://github.com/ebiggers/libdeflate/issues/33
+ * with the gzip header and footer removed to leave just the DEFLATE stream
+ */
+static const u8 orig_repro[3962] =
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48"
+	"\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11"
+	"\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea"
+	"\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00"
+	"\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x92\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a\x6a"
+	"\x6a\x6a\x6a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x92\x63\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x63\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80"
+	"\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00"
+	"\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92"
+	"\x63\x00\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04"
+	"\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x00\xea\x04\x48\x00\x20\x80\x28"
+	"\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1a\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00"
+	"\xea\x04\x48\x00\x20\x80\x28\x00\x00\x11\x00\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b"
+	"\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x1b\x92\x63\x00\x04\xea\x48\x00\x20"
+	"\x80\x28\x00\x00\x11\x1b\x1b\x1b\x1b\x92\x63\x00\xea\x04\x48\x00"
+	"\x20\x80\x28\x00\x00\x11\x00\x00\x01\x04\x00\x3f\x00\x00\x00\x00"
+	"\x28\xf7\xff\x00\xff\xff\xff\xff\x00\x00";
+
+int
+tmain(int argc, tchar *argv[])
+{
+	u8 in[4096];
+	u8 out[10000];
+	u64 t, tz;
+
+	begin_program(argv);
+
+	begin_performance_test();
+
+	/* static huffman case */
+	generate_empty_static_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("static huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("static huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is faster than zlib in this case, e.g.
+	 *	[static huffman, libdeflate]: 215861 KB/s
+	 *	[static huffman, zlib      ]: 73651 KB/s
+	 */
+	putchar('\n');
+	ASSERT(t < tz);
+
+	/* dynamic huffman case */
+	generate_empty_dynamic_huffman_blocks(in, sizeof(in));
+	t = do_test_libdeflate("dynamic huffman", in, sizeof(in),
+			       out, sizeof(out));
+	tz = do_test_zlib("dynamic huffman", in, sizeof(in), out, sizeof(out));
+	/*
+	 * libdeflate is slower than zlib in this case, though not super bad.
+	 *	[dynamic huffman, libdeflate]: 6277 KB/s
+	 *	[dynamic huffman, zlib      ]: 10419 KB/s
+	 * FIXME: make it faster.
+	 */
+	putchar('\n');
+	ASSERT(t < 4 * tz);
+
+	/* original reproducer */
+	t = do_test_libdeflate("original repro", orig_repro, sizeof(orig_repro),
+			       out, sizeof(out));
+	tz = do_test_zlib("original repro", orig_repro, sizeof(orig_repro),
+			  out, sizeof(out));
+	ASSERT(t < tz);
+
+	return 0;
+}

programs/test_trailing_bytes.c

@@ -0,0 +1,152 @@
+/*
+ * test_trailing_bytes.c
+ *
+ * Test that decompression correctly stops at the end of the first DEFLATE,
+ * zlib, or gzip stream, and doesn't process any additional trailing bytes.
+ */
+
+#include "test_util.h"
+
+static const struct {
+	size_t (LIBDEFLATEAPI *compress)(
+			struct libdeflate_compressor *compressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_out_nbytes_ret);
+	enum libdeflate_result (LIBDEFLATEAPI *decompress_ex)(
+			struct libdeflate_decompressor *decompressor,
+			const void *in, size_t in_nbytes,
+			void *out, size_t out_nbytes_avail,
+			size_t *actual_in_nbytes_ret,
+			size_t *actual_out_nbytes_ret);
+} codecs[] = {
+	{
+		.compress = libdeflate_deflate_compress,
+		.decompress = libdeflate_deflate_decompress,
+		.decompress_ex = libdeflate_deflate_decompress_ex,
+	}, {
+		.compress = libdeflate_zlib_compress,
+		.decompress = libdeflate_zlib_decompress,
+		.decompress_ex = libdeflate_zlib_decompress_ex,
+	}, {
+		.compress = libdeflate_gzip_compress,
+		.decompress = libdeflate_gzip_decompress,
+		.decompress_ex = libdeflate_gzip_decompress_ex,
+	}
+};
+
+int
+tmain(int argc, tchar *argv[])
+{
+	const size_t original_nbytes = 32768;
+	const size_t compressed_nbytes_total = 32768;
+	/*
+	 * Don't use the full buffer for compressed data, because we want to
+	 * test whether decompression can deal with additional trailing bytes.
+	 *
+	 * Note: we can't use a guarded buffer (i.e. a buffer where the byte
+	 * after compressed_nbytes is unmapped) because the decompressor may
+	 * read a few bytes beyond the end of the stream (but ultimately not
+	 * actually use those bytes) as long as they are within the buffer.
+	 */
+	const size_t compressed_nbytes_avail = 30000;
+	size_t i;
+	u8 *original;
+	u8 *compressed;
+	u8 *decompressed;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	size_t compressed_nbytes;
+	enum libdeflate_result res;
+	size_t actual_compressed_nbytes;
+	size_t actual_decompressed_nbytes;
+
+	begin_program(argv);
+
+	ASSERT(compressed_nbytes_avail < compressed_nbytes_total);
+
+	/* Prepare some dummy data to compress */
+	original = xmalloc(original_nbytes);
+	ASSERT(original != NULL);
+	for (i = 0; i < original_nbytes; i++)
+		original[i] = (i % 123) + (i % 1023);
+
+	compressed = xmalloc(compressed_nbytes_total);
+	ASSERT(compressed != NULL);
+	memset(compressed, 0, compressed_nbytes_total);
+
+	decompressed = xmalloc(original_nbytes);
+	ASSERT(decompressed != NULL);
+
+	c = libdeflate_alloc_compressor(6);
+	ASSERT(c != NULL);
+
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+
+	for (i = 0; i < ARRAY_LEN(codecs); i++) {
+		compressed_nbytes = codecs[i].compress(c, original,
+						       original_nbytes,
+						       compressed,
+						       compressed_nbytes_avail);
+		ASSERT(compressed_nbytes > 0);
+		ASSERT(compressed_nbytes <= compressed_nbytes_avail);
+
+		/* Test decompress() of stream that fills the whole buffer */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed, compressed_nbytes,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream that fills the whole buffer */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed, compressed_nbytes,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress() of stream with trailing bytes */
+		actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress(d, compressed,
+					   compressed_nbytes_total,
+					   decompressed, original_nbytes,
+					   &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+
+		/* Test decompress_ex() of stream with trailing bytes */
+		actual_compressed_nbytes = actual_decompressed_nbytes = 0;
+		memset(decompressed, 0, original_nbytes);
+		res = codecs[i].decompress_ex(d, compressed,
+					      compressed_nbytes_total,
+					      decompressed, original_nbytes,
+					      &actual_compressed_nbytes,
+					      &actual_decompressed_nbytes);
+		ASSERT(res == LIBDEFLATE_SUCCESS);
+		ASSERT(actual_compressed_nbytes == compressed_nbytes);
+		ASSERT(actual_decompressed_nbytes == original_nbytes);
+		ASSERT(memcmp(decompressed, original, original_nbytes) == 0);
+	}
+
+	free(original);
+	free(compressed);
+	free(decompressed);
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

programs/test_util.c

@@ -0,0 +1,243 @@
+/*
+ * test_util.c - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _WIN32
+/* for MAP_ANONYMOUS or MAP_ANON, which unfortunately aren't part of POSIX... */
+#  undef _POSIX_C_SOURCE
+#  ifdef __APPLE__
+#    define _DARWIN_C_SOURCE
+#  elif defined(__linux__)
+#    define _GNU_SOURCE
+#  endif
+#endif
+
+#include "test_util.h"
+
+#include <fcntl.h>
+#include <time.h>
+#ifdef _WIN32
+#  include <windows.h>
+#else
+#  include <unistd.h>
+#  include <sys/mman.h>
+#  include <sys/time.h>
+#endif
+
+#ifndef MAP_ANONYMOUS
+#  define MAP_ANONYMOUS MAP_ANON
+#endif
+
+/* Abort with an error message */
+_noreturn void
+assertion_failed(const char *expr, const char *file, int line)
+{
+	msg("Assertion failed: %s at %s:%d", expr, file, line);
+	abort();
+}
+
+void
+begin_performance_test(void)
+{
+	/* Skip performance tests by default, since they can be flaky. */
+	if (getenv("INCLUDE_PERF_TESTS") == NULL)
+		exit(0);
+}
+
+static size_t
+get_page_size(void)
+{
+#ifdef _WIN32
+	SYSTEM_INFO info;
+
+	GetSystemInfo(&info);
+	return info.dwPageSize;
+#else
+	return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+/* Allocate a buffer with guard pages */
+void
+alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret)
+{
+	const size_t pagesize = get_page_size();
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	u8 *base_addr;
+	u8 *start, *end;
+#ifdef _WIN32
+	DWORD oldProtect;
+#endif
+
+	*start_ret = NULL;
+	*end_ret = NULL;
+
+#ifdef _WIN32
+	/* Allocate buffer and guard pages with no access. */
+	base_addr = VirtualAlloc(NULL, (nr_pages + 2) * pagesize,
+				 MEM_COMMIT | MEM_RESERVE, PAGE_NOACCESS);
+	if (!base_addr) {
+		msg("Unable to allocate memory (VirtualAlloc): Windows error %u",
+		    (unsigned int)GetLastError());
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Grant read+write access to just the buffer. */
+	if (!VirtualProtect(start, end - start, PAGE_READWRITE, &oldProtect)) {
+		msg("Unable to protect memory (VirtualProtect): Windows error %u",
+		    (unsigned int)GetLastError());
+		VirtualFree(base_addr, 0, MEM_RELEASE);
+		ASSERT(0);
+	}
+#else
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	if (base_addr == (u8 *)MAP_FAILED) {
+		msg_errno("Unable to allocate memory (anonymous mmap)");
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+#endif
+	*start_ret = start;
+	*end_ret = end;
+}
+
+/* Free a buffer that was allocated by alloc_guarded_buffer() */
+void
+free_guarded_buffer(u8 *start, u8 *end)
+{
+	if (!start)
+		return;
+#ifdef _WIN32
+	VirtualFree(start - get_page_size(), 0, MEM_RELEASE);
+#else
+	munmap(start, end - start);
+#endif
+}
+
+/*
+ * Return the number of timer ticks that have elapsed since some unspecified
+ * point fixed at the start of program execution
+ */
+u64
+timer_ticks(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER count;
+
+	QueryPerformanceCounter(&count);
+	return count.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
+#else
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (1000000 * (u64)tv.tv_sec) + tv.tv_usec;
+#endif
+}
+
+/*
+ * Return the number of timer ticks per second
+ */
+static u64
+timer_frequency(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return freq.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME)
+	return 1000000000;
+#else
+	return 1000000;
+#endif
+}
+
+/*
+ * Convert a number of elapsed timer ticks to milliseconds
+ */
+u64 timer_ticks_to_ms(u64 ticks)
+{
+	return ticks * 1000 / timer_frequency();
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to MB/s
+ */
+u64 timer_MB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000000;
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to KB/s
+ */
+u64 timer_KB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000;
+}
+
+bool
+put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits)
+{
+	os->bitbuf |= bits << os->bitcount;
+	os->bitcount += num_bits;
+	while (os->bitcount >= 8) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	return true;
+}
+
+bool
+flush_bits(struct output_bitstream *os)
+{
+	while (os->bitcount > 0) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	os->bitcount = 0;
+	return true;
+}

programs/test_util.h

@@ -0,0 +1,67 @@
+/*
+ * test_util.h - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PROGRAMS_TEST_UTIL_H
+#define PROGRAMS_TEST_UTIL_H
+
+#include "prog_util.h"
+
+#include <zlib.h> /* for comparison purposes */
+
+#ifdef __GNUC__
+# define _noreturn __attribute__((noreturn))
+#else
+# define _noreturn
+#endif
+
+void _noreturn
+assertion_failed(const char *expr, const char *file, int line);
+
+#define ASSERT(expr) { if (unlikely(!(expr))) \
+	assertion_failed(#expr, __FILE__, __LINE__); }
+
+void begin_performance_test(void);
+
+void alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret);
+void free_guarded_buffer(u8 *start, u8 *end);
+
+u64 timer_ticks(void);
+u64 timer_ticks_to_ms(u64 ticks);
+u64 timer_MB_per_s(u64 bytes, u64 ticks);
+u64 timer_KB_per_s(u64 bytes, u64 ticks);
+
+struct output_bitstream {
+	machine_word_t bitbuf;
+	int bitcount;
+	u8 *next;
+	u8 *end;
+};
+
+bool put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits);
+bool flush_bits(struct output_bitstream *os);
+
+#endif /* PROGRAMS_TEST_UTIL_H */

scripts/afl-fuzz/deflate_compress/fuzz.c

@@ -7,8 +7,8 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
-	struct deflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	struct libdeflate_compressor *c;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,21 +20,21 @@ int main(int argc, char **argv)
 	ret = read(fd, in, sizeof in);
 	assert(ret == sizeof in);
 
-	c = deflate_alloc_compressor(6);
-	d = deflate_alloc_decompressor();
+	c = libdeflate_alloc_compressor(6);
+	d = libdeflate_alloc_decompressor();
 
 	char out[sizeof(in)];
 	char checkarray[sizeof(in)];
 
-	size_t csize = deflate_compress(c, in,sizeof in, out, sizeof out);
+	size_t csize = libdeflate_deflate_compress(c, in,sizeof in, out, sizeof out);
 	if (csize) {
-		enum decompress_result res;
-		res = deflate_decompress(d, out, csize, checkarray, sizeof in, NULL);
+		enum libdeflate_result res;
+		res = libdeflate_deflate_decompress(d, out, csize, checkarray, sizeof in, NULL);
 		assert(!res);
 		assert(!memcmp(in, checkarray, sizeof in));
 	}
 
-	deflate_free_compressor(c);
-	deflate_free_decompressor(d);
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/afl-fuzz/deflate_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/afl-fuzz/gzip_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/afl-fuzz/zlib_decompress/fuzz.c

@@ -6,7 +6,7 @@
 
 int main(int argc, char **argv)
 {
-	struct deflate_decompressor *d;
+	struct libdeflate_decompressor *d;
 	int ret;
 	int fd = open(argv[1], O_RDONLY);
 	struct stat stbuf;
@@ -20,9 +20,9 @@ int main(int argc, char **argv)
 
 	char out[sizeof(in) * 3];
 
-	d = deflate_alloc_decompressor();
+	d = libdeflate_alloc_decompressor();
 
-	gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	deflate_free_decompressor(d);
+	libdeflate_zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
 	return 0;
 }

scripts/android_build.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+API_LEVEL=28
+ARCH=arm64
+CFLAGS=
+ENABLE_CRC=false
+ENABLE_CRYPTO=false
+NDKDIR=$HOME/android-ndk-r21d
+
+usage() {
+	cat << EOF
+Usage: $0 [OPTION]... -- [MAKE_TARGET]...
+Build libdeflate for Android.
+
+  --api-level=LEVEL    Android API level to target (default: $API_LEVEL)
+  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
+  --enable-crc         Enable crc instructions
+  --enable-crypto      Enable crypto instructions
+  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
+EOF
+}
+if ! options=$(getopt -o '' \
+	-l 'api-level:,arch:,enable-crc,enable-crypto,help,ndkdir:' -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+
+eval set -- "$options"
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--api-level)
+		API_LEVEL="$2"
+		shift
+		;;
+	--arch)
+		ARCH="$2"
+		shift
+		;;
+	--enable-crc)
+		ENABLE_CRC=true
+		;;
+	--enable-crypto)
+		ENABLE_CRYPTO=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--ndkdir)
+		NDKDIR="$2"
+		shift
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Unknown option \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+BINDIR=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/
+
+case "$ARCH" in
+arm|arm32|aarch32)
+	CC=$BINDIR/armv7a-linux-androideabi$API_LEVEL-clang
+	if $ENABLE_CRC || $ENABLE_CRYPTO; then
+		CFLAGS="-march=armv8-a"
+		if $ENABLE_CRC; then
+			CFLAGS+=" -mcrc"
+		else
+			CFLAGS+=" -mnocrc"
+		fi
+		if $ENABLE_CRYPTO; then
+			CFLAGS+=" -mfpu=crypto-neon-fp-armv8"
+		else
+			CFLAGS+=" -mfpu=neon"
+		fi
+	fi
+	;;
+arm64|aarch64)
+	CC=$BINDIR/aarch64-linux-android$API_LEVEL-clang
+	features=""
+	if $ENABLE_CRC; then
+		features+="+crc"
+	fi
+	if $ENABLE_CRYPTO; then
+		features+="+crypto"
+	fi
+	if [ -n "$features" ]; then
+		CFLAGS="-march=armv8-a$features"
+	fi
+	;;
+*)
+	echo 1>&2 "Unknown architecture: \"$ARCH\""
+	usage 1>&2
+	exit 1
+esac
+
+cmd=(make "-j$(grep -c processor /proc/cpuinfo)" "CC=$CC" "CFLAGS=$CFLAGS" "$@")
+echo "${cmd[*]}"
+"${cmd[@]}"

scripts/android_tests.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Test libdeflate on a connected arm64 Android device.
+# Requires the Android NDK (release 19 or later) and adb.
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use NDKDIR if specified in environment, else use default value.
+: "${NDKDIR:=$HOME/android-ndk-r21d}"
+if [ ! -e "$NDKDIR" ]; then
+	cat 1>&2 << EOF
+Android NDK was not found in NDKDIR=$NDKDIR!  Set the
+environmental variable NDKDIR to the location of your Android NDK installation.
+EOF
+	exit 1
+fi
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+android_build_and_test() {
+	echo "Running Android tests with $*"
+
+	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" \
+		all test_programs > /dev/null
+	adb push "$TESTDATA" ./scripts/exec_tests.sh benchmark test_* \
+		/data/local/tmp/ > /dev/null
+
+	# Note: adb shell always returns 0, even if the shell command fails...
+	adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \
+		> "$TMPDIR/adb.out"
+	if ! grep -q "exec_tests finished successfully" "$TMPDIR/adb.out"; then
+		echo 1>&2 "Android test failure!  adb shell output:"
+		cat "$TMPDIR/adb.out"
+		exit 1
+	fi
+}
+
+for arch in arm32 arm64; do
+	android_build_and_test --arch=$arch
+	android_build_and_test --arch=$arch --enable-crc
+	android_build_and_test --arch=$arch --enable-crypto
+	android_build_and_test --arch=$arch --enable-crc --enable-crypto
+done
+echo "Android tests passed"

scripts/checksum_benchmarks.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+have_cpu_feature() {
+	local feature="$1"
+	local tag
+	case $ARCH in
+	arm*|aarch*)
+		tag="Features"
+		;;
+	*)
+		tag="flags"
+		;;
+	esac
+	grep -q "^$tag"$'[ \t]'"*:.*\<$feature\>" /proc/cpuinfo
+}
+
+make_and_test() {
+	# Build the checksum program and tests.  Set the special test support
+	# flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	make "$@" TEST_SUPPORT__DO_NOT_USE=1 checksum test_checksums > /dev/null
+
+	# Run the checksum tests, for good measure.  (This isn't actually part
+	# of the benchmarking.)
+	./test_checksums > /dev/null
+}
+
+__do_benchmark() {
+	local impl="$1" speed
+	shift
+	local flags=("$@")
+
+	speed=$(./checksum "${CKSUM_FLAGS[@]}" "${flags[@]}" -t "$FILE" | \
+		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
+	printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
+}
+
+do_benchmark() {
+	local impl="$1"
+
+	if [ "$impl" = zlib ]; then
+		__do_benchmark "$impl" "-Z"
+	else
+		make_and_test CFLAGS="${EXTRA_CFLAGS[*]}"
+		__do_benchmark "libdeflate, $impl"
+		if [ "$ARCH" = x86_64 ]; then
+			make_and_test CFLAGS="-m32 ${EXTRA_CFLAGS[*]}"
+			__do_benchmark "libdeflate, $impl, 32-bit"
+		fi
+	fi
+}
+
+sort_by_speed() {
+	awk '{print $NF, $0}' | sort -nr | cut -f2- -d' '
+}
+
+disable_cpu_feature() {
+	local name="$1"
+	shift
+	local extra_cflags=("$@")
+
+	LIBDEFLATE_DISABLE_CPU_FEATURES+=",$name"
+	EXTRA_CFLAGS+=("${extra_cflags[@]}")
+}
+
+cleanup() {
+	if $USING_TMPFILE; then
+		rm "$FILE"
+	fi
+}
+
+ARCH="$(uname -m)"
+USING_TMPFILE=false
+
+if (( $# > 1 )); then
+	echo "Usage: $0 [FILE]" 1>&2
+	exit 1
+fi
+
+trap cleanup EXIT
+
+if (( $# == 0 )); then
+	# Generate default test data file.
+	FILE=$(mktemp -t checksum_testdata.XXXXXXXXXX)
+	USING_TMPFILE=true
+	echo "Generating 100 MB test file: $FILE"
+	head -c 100000000 /dev/urandom > "$FILE"
+else
+	FILE="$1"
+fi
+
+cat << EOF
+Method                                       Speed (MB/s)
+------                                       ------------
+EOF
+
+# CRC-32
+CKSUM_NAME="CRC-32"
+CKSUM_FLAGS=()
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature pclmulqdq && have_cpu_feature avx; then
+		do_benchmark "PCLMUL/AVX"
+		disable_cpu_feature "avx" "-mno-avx"
+	fi
+	if have_cpu_feature pclmulqdq; then
+		do_benchmark "PCLMUL"
+		disable_cpu_feature "pclmul" "-mno-pclmul"
+	fi
+	;;
+arm*|aarch*)
+	if have_cpu_feature crc32; then
+		do_benchmark "ARM"
+		disable_cpu_feature "crc32" "-march=armv8-a+nocrc"
+	fi
+	if have_cpu_feature pmull; then
+		do_benchmark "PMULL"
+		disable_cpu_feature "pmull" "-march=armv8-a+nocrc+nocrypto"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed
+
+# Adler-32
+CKSUM_NAME="Adler-32"
+CKSUM_FLAGS=(-A)
+EXTRA_CFLAGS=()
+export LIBDEFLATE_DISABLE_CPU_FEATURES=""
+echo
+{
+case $ARCH in
+i386|x86_64)
+	if have_cpu_feature avx512bw; then
+		do_benchmark "AVX-512BW"
+		disable_cpu_feature "avx512bw" "-mno-avx512bw"
+	fi
+	if have_cpu_feature avx2; then
+		do_benchmark "AVX2"
+		disable_cpu_feature "avx2" "-mno-avx2"
+	fi
+	if have_cpu_feature sse2; then
+		do_benchmark "SSE2"
+		disable_cpu_feature "sse2" "-mno-sse2"
+	fi
+	;;
+arm*)
+	if have_cpu_feature neon; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-mfpu=vfpv3"
+	fi
+	;;
+aarch*)
+	if have_cpu_feature asimd; then
+		do_benchmark "NEON"
+		disable_cpu_feature "neon" "-march=armv8-a+nosimd"
+	fi
+	;;
+esac
+do_benchmark "generic"
+do_benchmark "zlib"
+} | sort_by_speed

scripts/detect.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+
+set -eu
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
+
+echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
+echo "#ifndef CONFIG_H"
+echo "#define CONFIG_H"
+
+program_compiles() {
+	echo "$1" | $CC $CFLAGS -Wno-error -x c - -o /dev/null > /dev/null 2>&1
+}
+
+check_function() {
+	funcname=$1
+	macro="HAVE_$(echo "$funcname" | tr '[:lower:]' '[:upper:]')"
+
+	echo
+	echo "/* Is the $funcname() function available? */"
+	if program_compiles "int main() { $funcname(); }"; then
+		echo "#define $macro 1"
+	else
+		echo "/* $macro is not set */"
+	fi
+}
+
+have_stat_field() {
+	program_compiles "#include <sys/types.h>
+			  #include <sys/stat.h>
+			  int main() { struct stat st; st.$1; }"
+}
+
+check_stat_nanosecond_precision() {
+	echo
+	echo "/* Does stat() provide nanosecond-precision timestamps? */"
+	if have_stat_field st_atim; then
+		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
+	elif have_stat_field st_atimespec; then
+		# Nonstandard field names used by OS X and older BSDs
+		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
+		echo "#define st_atim st_atimespec"
+		echo "#define st_mtim st_mtimespec"
+		echo "#define st_ctim st_ctimespec"
+	else
+		echo "/* HAVE_STAT_NANOSECOND_PRECISION is not set */"
+	fi
+}
+
+check_function clock_gettime
+check_function futimens
+check_function futimes
+check_function posix_fadvise
+check_function posix_madvise
+
+check_stat_nanosecond_precision
+
+echo
+echo "#endif /* CONFIG_H */"

scripts/exec_tests.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# Helper script used by run_tests.sh and android_tests.sh,
+# not intended to be run directly
+#
+
+set -eu
+
+run_cmd() {
+	echo "$WRAPPER $*"
+	$WRAPPER "$@" > /dev/null
+}
+
+for prog in ./test_*; do
+	run_cmd "$prog"
+done
+
+for format in '' '-g' '-z'; do
+	for ref_impl in '' '-Y' '-Z'; do
+		run_cmd ./benchmark $format $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9; do
+	for ref_impl in '' '-Y'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+for level in 0 1 3 7 9 12; do
+	for ref_impl in '' '-Z'; do
+		run_cmd ./benchmark -$level $ref_impl "$TESTDATA"
+	done
+done
+
+echo "exec_tests finished successfully" # Needed for 'adb shell'

scripts/gen_crc32_multipliers.c

@@ -0,0 +1,108 @@
+/*
+ * gen_crc32_multipliers.c
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <inttypes.h>
+#include <stdio.h>
+
+/* generator polynomial G(x) */
+#define CRCPOLY		0xEDB88320 /* G(x) without x^32 term */
+#define CRCPOLY_FULL	(((uint64_t)CRCPOLY << 1) | 1) /* G(x) */
+
+/* Compute x^D mod G(x) */
+static uint32_t
+compute_multiplier(int D)
+{
+	/* Start with x^0 mod G(x) */
+	uint32_t remainder = 0x80000000;
+
+	/* Each iteration, 'remainder' becomes x^i mod G(x) */
+	for (int i = 1; i <= D; i++)
+		remainder = (remainder >> 1) ^ ((remainder & 1) ? CRCPOLY : 0);
+
+	/* Now 'remainder' is x^D mod G(x) */
+	return remainder;
+}
+
+/* Compute floor(x^64 / G(x)) */
+static uint64_t
+compute_barrett_reduction_constant(void)
+{
+	uint64_t quotient = 0;
+	uint64_t dividend = 0x1;
+
+	for (int i = 0; i < 64 - 32 + 1; i++) {
+		if ((dividend >> i) & 1) {
+			quotient |= (uint64_t)1 << i;
+			dividend ^= CRCPOLY_FULL << i;
+		}
+	}
+
+	return quotient;
+}
+
+/*
+ * This program computes the constant multipliers needed for carryless
+ * multiplication accelerated CRC-32.  It assumes 128-bit vectors divided into
+ * two 64-bit halves which are multiplied separately with different 32-bit
+ * multipliers, producing two 95-bit products.  For a given number of 128-bit
+ * vectors per iteration, the program outputs a pair of multipliers, one for
+ * each 64-bit half.
+ *
+ * Careful: all polynomials are "bit-reversed", meaning that the low-order bits
+ * have the highest degree and the high-order bits have the lowest degree!
+ */
+int
+main(void)
+{
+	printf("\t/* Constants precomputed by gen_crc32_multipliers.c.  "
+	       "Do not edit! */\n");
+
+	/* High and low multipliers for each needed vector count */
+	for (int order = 2; order >= 0; order--) {
+		int vecs_per_iteration = 1 << order;
+		int right = (128 * vecs_per_iteration) + 95;
+		printf("\tconst __v2di multipliers_%d = (__v2di)"
+		       "{ 0x%08"PRIX32", 0x%08"PRIX32" };\n",
+		       vecs_per_iteration,
+		       compute_multiplier(right - 64) /* higher degree half */,
+		       compute_multiplier(right - 128) /* lower degree half */);
+	}
+
+	/* Multiplier for final 96 => 64 bit fold */
+	printf("\tconst __v2di final_multiplier = (__v2di){ 0x%08"PRIX32" };\n",
+	       compute_multiplier(63));
+
+	/* 32-bit mask */
+	printf("\tconst __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };\n");
+
+	/* Constants for final 64 => 32 bit reduction */
+	printf("\tconst __v2di barrett_reduction_constants =\n"
+	       "\t\t\t(__v2di){ 0x%016"PRIX64", 0x%016"PRIX64" };\n",
+	       compute_barrett_reduction_constant(), CRCPOLY_FULL);
+
+	return 0;
+}

scripts/gen_crc32_table.c

@@ -1,15 +1,30 @@
 /*
  * gen_crc32_table.c - a program for CRC-32 table generation
  *
- * Written in 2014-2015 by Eric Biggers <ebiggers3@gmail.com>
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
  *
- * To the extent possible under law, the author(s) have dedicated all copyright
- * and related and neighboring rights to this software to the public domain
- * worldwide. This software is distributed without any warranty.
+ * Copyright 2016 Eric Biggers
  *
- * You should have received a copy of the CC0 Public Domain Dedication along
- * with this software. If not, see
- * <http://creativecommons.org/publicdomain/zero/1.0/>.
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <stdint.h>

scripts/gzip_tests.sh

@@ -0,0 +1,490 @@
+#!/bin/bash
+#
+# Test script for libdeflate's gzip and gunzip programs.
+#
+# To run, you must set GZIP and GUNZIP in the environment to the absolute paths
+# to the gzip and gunzip programs to test.  All tests should pass regardless of
+# whether the GNU versions or the libdeflate versions, or a combination, of
+# these programs are used.
+#
+# The environmental variable TESTDATA must also be set to a file containing
+# test data.
+#
+
+set -eu -o pipefail
+
+export -n GZIP GUNZIP TESTDATA
+
+TMPDIR="$(mktemp -d)"
+CURRENT_TEST=
+
+cleanup() {
+	if [ -n "$CURRENT_TEST" ]; then
+		echo "TEST FAILED: \"$CURRENT_TEST\""
+	fi
+	rm -rf -- "$TMPDIR"
+}
+
+trap cleanup EXIT
+
+TESTDATA="$(readlink -f "$TESTDATA")"
+cd "$TMPDIR"
+
+begin_test() {
+	CURRENT_TEST="$1"
+	rm -rf -- "${TMPDIR:?}"/*
+	cp "$TESTDATA" file
+}
+
+gzip() {
+	$GZIP "$@"
+}
+
+gunzip() {
+	$GUNZIP "$@"
+}
+
+assert_status() {
+	local expected_status="$1"
+	local expected_msg="$2"
+	shift 2
+	(
+		set +e
+		{ eval "$*" > /dev/null; } 2>&1
+		local actual_status=$?
+		if [ "$actual_status" != "$expected_status" ]; then
+			echo 1>&2 "Command '$*' exited with status" \
+				"$actual_status but expected status" \
+				"$expected_status"
+			exit 1
+		fi
+		exit 0
+	) > command_output
+	if ! grep -E -q "$expected_msg" command_output; then
+		echo 1>&2 "Expected output of command '$*' to match regex" \
+			"'$expected_msg'"
+		echo 1>&2 "Actual output was:"
+		echo 1>&2 "---------------------------------------------------"
+		cat 1>&2 command_output
+		echo 1>&2 "---------------------------------------------------"
+		return 1
+	fi
+}
+
+assert_error() {
+	assert_status 1 "$@"
+}
+
+assert_warning() {
+	assert_status 2 "$@"
+}
+
+assert_skipped() {
+	assert_warning '\<(ignored|skipping|unchanged)\>' "$@"
+}
+
+assert_equals() {
+	local expected="$1"
+	local actual="$2"
+
+	if [ "$expected" != "$actual" ]; then
+		echo 1>&2 "Expected '$expected', but got '$actual'"
+		return 1
+	fi
+}
+
+# Get the filesystem type.
+FSTYPE=$(df -T . | tail -1 | awk '{print $2}')
+
+# If gzip or gunzip is the GNU version, require that it supports the '-k'
+# option.  This option was added in v1.6, released in 2013.
+check_version_prereq() {
+	local prog=$1
+
+	if ! echo | { $prog -k || true; } |& grep -q 'invalid option'; then
+		return 0
+	fi
+	if ! $prog -V |& grep -q 'Free Software Foundation'; then
+		echo 1>&2 "Unexpected case: not GNU $prog, but -k option is invalid"
+		exit 1
+	fi
+	echo "GNU $prog is too old; skipping gzip/gunzip tests"
+	exit 0
+}
+check_version_prereq gzip
+check_version_prereq gunzip
+
+begin_test 'Basic compression and decompression works'
+cp file orig
+gzip file
+[ ! -e file ] && [ -e file.gz ]
+gunzip file.gz
+[ -e file ] && [ ! -e file.gz ]
+cmp file orig
+
+
+begin_test 'gzip -d is gunzip'
+cp file orig
+gzip file
+gzip -d file.gz
+cmp file orig
+
+
+begin_test '-k (keep original file) works'
+cp file orig
+gzip -k file
+cmp file orig
+rm file
+cp file.gz orig.gz
+gunzip -k file.gz
+cmp file.gz orig.gz
+
+
+begin_test '-c (write to stdout) works'
+cp file orig
+gzip -k file
+gzip -c file > 2.gz
+cmp file orig
+cmp file.gz 2.gz
+gunzip -c 2.gz > file
+cmp file.gz 2.gz
+cmp file orig
+
+
+# Note: in some of the commands below, we intentionally use 'cat file | gzip'
+# rather than 'gzip < file', in order to test the use of a pipe.  This produces
+# a shellcheck warning about 'cat' being unnecessary.  Suppress that warning by
+# using { cat file; true; }.
+begin_test 'Reading from stdin works'
+gzip < file > 1.gz
+gzip - < file > 2.gz
+{ cat file; true; } | gzip > 3.gz
+{ cat file; true; } | gzip - > 4.gz
+cmp file <(gunzip < 1.gz)
+cmp file <(gunzip - < 2.gz)
+cmp file <({ cat 3.gz; true; } | gunzip)
+cmp file <({ cat 4.gz; true; } | gunzip -)
+
+
+begin_test '-n option is accepted'
+gzip -n file
+gunzip -n file.gz
+
+
+begin_test 'can specify multiple options'
+gzip -fk1 file
+cmp <(gzip -c -1 file) file.gz
+gunzip -kfd file.gz
+
+
+begin_test 'Compression levels'
+if [ "$GZIP" = /bin/gzip ]; then
+	assert_error '\<invalid option\>' gzip -10
+	max_level=9
+else
+	for level in 13 99999 1a; do
+		assert_error '\<Invalid compression level\>' gzip -$level
+	done
+	max_level=12
+fi
+for level in $(seq 1 $max_level); do
+	gzip -c "-$level" file > "file$level"
+	cmp file <(gunzip -c "file$level")
+done
+rm file command_output
+
+
+begin_test 'Overwriting output file requires -f'
+cp file orig
+echo -n > file.gz
+gzip -c file > 2.gz
+assert_warning 'already exists' gzip file </dev/null
+cmp file.gz /dev/null
+gzip -f file
+cmp 2.gz file.gz
+echo -n > file
+assert_warning 'already exists' gunzip file.gz </dev/null
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Nonexistent input file fails, even with -f'
+for prog in 'gzip' 'gzip -f' 'gunzip' 'gunzip -f'; do
+	assert_error 'No such file or directory' "$prog" NONEXISTENT
+done
+
+
+begin_test 'Compressing already-suffixed file requires -f or -c'
+gzip file
+gzip -c file.gz > c.gz
+gzip file.gz 2>&1 >/dev/null | grep -q 'already has .gz suffix'
+[ -e file.gz ] && [ ! -e file.gz.gz ]
+gzip -f file.gz
+[ ! -e file.gz ] && [ -e file.gz.gz ]
+cmp file.gz.gz c.gz
+
+
+begin_test 'Decompressing unsuffixed file only works with -c'
+gzip file && mv file.gz file
+assert_skipped gunzip file
+assert_skipped gunzip -f file
+gunzip -c file > orig
+mv file file.gz && gunzip file.gz && cmp file orig
+
+
+begin_test '... unless there is a corresponding suffixed file'
+cp file orig
+gzip file
+[ ! -e file ] && [ -e file.gz ]
+gunzip -c file > tmp
+cmp tmp orig
+rm tmp
+ln -s NONEXISTENT file
+gunzip -c file > tmp
+cmp tmp orig
+rm tmp file
+gunzip file
+[ -e file ] && [ ! -e file.gz ]
+cmp file orig
+
+
+begin_test 'Directory is skipped, even with -f'
+mkdir dir
+mkdir dir.gz
+for opt in '' '-f' '-c'; do
+	assert_skipped gzip $opt dir
+done
+#assert_skipped gzip dir.gz  # XXX: GNU gzip warns, libdeflate gzip no-ops
+for opt in '' '-f' '-c'; do
+	for name in dir dir.gz; do
+		assert_skipped gunzip $opt $name
+	done
+done
+
+
+begin_test '(gzip) symlink is rejected without -f or -c'
+ln -s file symlink1
+ln -s file symlink2
+assert_error 'Too many levels of symbolic links' gzip symlink1
+[ -e file ] && [ -e symlink1 ] && [ ! -e symlink1.gz ]
+gzip -f symlink1
+[ -e file ] && [ ! -e symlink1 ] && [ -e symlink1.gz ]
+gzip -c symlink2 > /dev/null
+
+
+begin_test '(gunzip) symlink is rejected without -f or -c'
+gzip file
+ln -s file.gz symlink1.gz
+ln -s file.gz symlink2.gz
+assert_error 'Too many levels of symbolic links' gunzip symlink1
+[ -e file.gz ] && [ -e symlink1.gz ] && [ ! -e symlink1 ]
+gunzip -f symlink1.gz
+[ -e file.gz ] && [ ! -e symlink1.gz ] && [ -e symlink1 ]
+gunzip -c symlink2.gz > /dev/null
+
+
+begin_test 'FIFO is skipped, even with -f'
+mkfifo foo
+mkfifo foo.gz
+assert_skipped gzip foo
+assert_skipped gzip -f foo
+#assert_skipped gzip -c foo # XXX: works with GNU gzip, not libdeflate's
+assert_skipped gunzip foo.gz
+assert_skipped gunzip -f foo.gz
+#assert_skipped gunzip -c foo.gz # XXX: works with GNU gzip, not libdeflate's
+
+
+begin_test '(gzip) overwriting symlink does not follow symlink'
+echo a > a
+echo b > b
+gzip a
+ln -s a.gz b.gz
+gzip -f b
+gunzip a.gz
+cmp <(echo a) a
+
+
+begin_test '(gunzip) overwriting symlink does not follow symlink'
+echo a > a
+echo b > b
+gzip b
+ln -s a b
+gunzip -f b.gz
+cmp <(echo a) a
+cmp <(echo b) b
+
+
+begin_test '(gzip) hard linked file skipped without -f or -c'
+cp file orig
+ln file link
+assert_equals 2 "$(stat -c %h file)"
+assert_skipped gzip file
+gzip -c file > /dev/null
+assert_equals 2 "$(stat -c %h file)"
+gzip -f file
+assert_equals 1 "$(stat -c %h link)"
+assert_equals 1 "$(stat -c %h file.gz)"
+cmp link orig
+# XXX: GNU gzip skips hard linked files with -k, libdeflate's doesn't
+
+
+begin_test '(gunzip) hard linked file skipped without -f or -c'
+gzip file
+ln file.gz link.gz
+cp file.gz orig.gz
+assert_equals 2 "$(stat -c %h file.gz)"
+assert_skipped gunzip file.gz
+gunzip -c file.gz > /dev/null
+assert_equals 2 "$(stat -c %h file.gz)"
+gunzip -f file
+assert_equals 1 "$(stat -c %h link.gz)"
+assert_equals 1 "$(stat -c %h file)"
+cmp link.gz orig.gz
+
+
+begin_test 'Multiple files'
+cp file file2
+gzip file file2
+[ ! -e file ] && [ ! -e file2 ] && [ -e file.gz ] && [ -e file2.gz ]
+gunzip file.gz file2.gz
+[ -e file ] && [ -e file2 ] && [ ! -e file.gz ] && [ ! -e file2.gz ]
+
+
+begin_test 'Multiple files, continue on warning'
+mkdir 1
+cp file 2
+assert_skipped gzip 1 2
+[ ! -e 1.gz ]
+cmp file <(gunzip -c 2.gz)
+rmdir 1
+mkdir 1.gz
+assert_skipped gunzip 1.gz 2.gz
+[ ! -e 1 ]
+cmp 2 file
+
+
+if (( $(id -u) != 0 )); then
+	begin_test 'Multiple files, continue on error'
+	cp file 1
+	cp file 2
+	chmod a-r 1
+	assert_error 'Permission denied' gzip 1 2
+	[ ! -e 1.gz ]
+	cmp file <(gunzip -c 2.gz)
+	rm -f 1
+	cp 2.gz 1.gz
+	chmod a-r 1.gz
+	assert_error 'Permission denied' gunzip 1.gz 2.gz
+	[ ! -e 1 ]
+	cmp 2 file
+fi
+
+
+begin_test 'Compressing empty file'
+echo -n > empty
+gzip empty
+gunzip empty.gz
+cmp /dev/null empty
+
+
+begin_test 'Decompressing malformed file'
+echo -n > foo.gz
+assert_error '\<(not in gzip format|unexpected end of file)\>' \
+	gunzip foo.gz
+echo 1 > foo.gz
+assert_error '\<not in gzip format\>' gunzip foo.gz
+echo abcdefgh > foo.gz
+assert_error '\<not in gzip format\>' gunzip foo.gz
+echo -ne '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4b\x4c\x4a\x4e\x49\x24\x16\x73\x01\x00\x6c\x5b\xa2\x62\x2e\x00\x00\x00' \
+	> foo.gz
+assert_error '\<(not in gzip format|crc error)\>' gunzip foo.gz
+
+
+for suf in .foo foo .blaaaaaaaaaaaaaaaargh; do
+	begin_test "Custom suffix: $suf"
+	gzip -S $suf file
+	[ ! -e file ] && [ ! -e file.gz ] && [ -e file$suf ]
+	assert_skipped gunzip file$suf
+	gunzip -S $suf file$suf
+	[ -e file ] && [ ! -e file.gz ] && [ ! -e file$suf ]
+done
+# DIFFERENCE: GNU gzip lower cases suffix, we don't
+
+
+begin_test 'Empty suffix is rejected'
+assert_error '\<invalid suffix\>' gzip -S '""' file
+assert_error '\<invalid suffix\>' gunzip -S '""' file
+
+
+begin_test 'Timestamps and mode are preserved'
+if [ "$FSTYPE" = shiftfs ]; then
+	# In Travis CI, the filesystem (shiftfs) only supports seconds precision
+	# timestamps.  Nanosecond precision still sometimes seems to work,
+	# probably due to caching, but it is unreliable.
+	format='%a;%X;%Y'
+else
+	format='%a;%x;%y'
+fi
+chmod 777 file
+orig_stat="$(stat -c "$format" file)"
+gzip file
+sleep 1
+gunzip file.gz
+assert_equals "$orig_stat" "$(stat -c "$format" file)"
+
+
+begin_test 'Decompressing multi-member gzip file'
+cat file file > orig
+gzip -c file > file.gz
+gzip -c file >> file.gz
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Decompressing multi-member gzip file (final member smaller)'
+echo 'hello world' > hello
+cat file hello > orig
+gzip -c file > file.gz
+gzip -c hello >> file.gz
+gunzip -f file.gz
+cmp file orig
+
+
+begin_test 'Help option'
+gzip -h 2>&1 | grep -q 'Usage'
+gunzip -h 2>&1 | grep -q 'Usage'
+
+
+begin_test 'Incorrect usage'
+for prog in gzip gunzip; do
+	for opt in '--invalid-option' '-0'; do
+		assert_error '\<(unrecognized|invalid) option\>' $prog $opt
+	done
+done
+
+
+begin_test '-t (test) option works'
+good_files=(
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIoAUrnJFZVKqTkp+txAQBqzFDrLQAAAA==')
+bad_files=(
+'H4sIAO1YYmAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+46ehsAAAA='
+'H4sIAO1YYmAAA3PMSVTITVTIzi85VABTIJ5jzpGZelwAX+86ehsAAAA='
+'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsBAAA='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIogUrnJFZVKqTkp+txAQBqzFDrLQAAAA=='
+'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9L')
+for contents in "${good_files[@]}"; do
+	echo "$contents" | base64 -d | gzip -t
+done
+for contents in "${bad_files[@]}"; do
+	echo "$contents" | base64 -d > file
+	assert_error '\<invalid compressed data|file corrupt|unexpected end of file|Out of memory\>' \
+		gzip -t file
+done
+
+
+begin_test 'Version information'
+gzip -V | grep -q Copyright
+gunzip -V | grep -q Copyright
+
+CURRENT_TEST=

scripts/make-windows-releases.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+for arch in 'i686' 'x86_64'; do
+	make clean
+	make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all \
+		benchmark.exe checksum.exe
+	dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin
+	rm -rf "$dir" "$dir.zip"
+	mkdir "$dir"
+	cp libdeflate.{dll,lib,def} libdeflatestatic.lib libdeflate.h ./*.exe \
+		"$dir"
+	${arch}-w64-mingw32-strip "$dir/libdeflate.dll" "$dir"/*.exe
+	for file in COPYING NEWS.md README.md; do
+		sed < $file > "$dir/${file}.txt" -e 's/$/\r/g'
+	done
+	(cd "$dir" && zip -r "../${dir}.zip" .)
+done

scripts/pgo_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Try gcc profile-guided optimizations
+
+set -eu
+
+MAKE="make -j$(grep -c processor /proc/cpuinfo)"
+DATAFILE="$HOME/data/silesia"
+
+$MAKE benchmark > /dev/null
+echo "====================="
+echo "Original performance:"
+echo "---------------------"
+./benchmark "$@" "$DATAFILE"
+
+$MAKE CFLAGS=-fprofile-generate LDFLAGS=-fprofile-generate benchmark > /dev/null
+./benchmark "$@" "$DATAFILE" > /dev/null
+$MAKE CFLAGS=-fprofile-use benchmark > /dev/null
+rm -f {lib,programs}/*.gcda
+echo "=========================="
+echo "PGO-optimized performance:"
+echo "--------------------------"
+./benchmark "$@" "$DATAFILE"

scripts/produce_gzip_benchmark_table.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-set -e
+set -eu -o pipefail
+topdir="$(dirname "$0")/.."
 
 do_benchmark() {
-	usize=$(stat -c %s "$file")
-	"$HOME/proj/libdeflate/benchmark" -g -s $usize "$@" "$file" \
-			| grep Compressed | cut -f 4 -d ' '
+	"$topdir/benchmark" -g -s "$(stat -c %s "$file")" "$@" "$file" \
+		| grep Compressed | cut -f 4 -d ' '
 }
 
 echo "File | zlib -6 | zlib -9 | libdeflate -6 | libdeflate -9 | libdeflate -12"
@@ -14,11 +14,11 @@ echo "-----|---------|---------|---------------|---------------|---------------"
 for file in "$@"; do
 	echo -n "$(basename "$file")"
 	results=()
-	results+=($(do_benchmark -Y -6))
-	results+=($(do_benchmark -Y -9))
-	results+=($(do_benchmark -6))
-	results+=($(do_benchmark -9))
-	results+=($(do_benchmark -12))
+	results+=("$(do_benchmark -Y -6)")
+	results+=("$(do_benchmark -Y -9)")
+	results+=("$(do_benchmark -6)")
+	results+=("$(do_benchmark -9)")
+	results+=("$(do_benchmark -12)")
 	best=2000000000
 	for result in "${results[@]}"; do
 		if (( result < best)); then

scripts/run_tests.sh

@@ -0,0 +1,329 @@
+#!/bin/bash
+#
+# Test script for libdeflate
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use CC if specified in environment, else default to "cc".
+: "${CC:=cc}"
+
+# Use CFLAGS if specified in environment.
+: "${CFLAGS:=}"
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+MAKE="make -j$(getconf _NPROCESSORS_ONLN)"
+
+CC_VERSION=$($CC --version | head -1)
+
+ARCH=$(uname -m)
+
+for skip in SKIP_FREESTANDING SKIP_VALGRIND SKIP_UBSAN SKIP_ASAN SKIP_CFI \
+	    SKIP_SHARED_LIB; do
+	if [ "${!skip:-}" = "1" ]; then
+		eval $skip=true
+	else
+		eval $skip=false
+	fi
+done
+
+###############################################################################
+
+INDENT=0
+
+log() {
+	echo -n "[$(date)] "
+	head -c $(( INDENT * 4 )) /dev/zero | tr '\0' ' '
+	echo "$@"
+}
+
+begin() {
+	log "$@"
+	(( INDENT++ )) || true
+}
+
+end() {
+	(( INDENT-- )) || true
+}
+
+run_cmd() {
+	log "$@"
+	"$@" > /dev/null
+}
+
+fail() {
+	echo 1>&2 "$@"
+	exit 1
+}
+
+file_count() {
+	local dir=$1
+
+	find "$dir" -type f -o -type l | wc -l
+}
+
+cflags_supported() {
+	# -Werror is needed here in order for old versions of clang to reject
+	# invalid options.
+	echo 'int main(void){ return 0; }' \
+		| $CC $CFLAGS "$@" -Werror -x c - -o /dev/null 2>/dev/null
+}
+
+valgrind_version_at_least() {
+	local want_vers=$1
+	local vers
+
+	vers=$(valgrind --version | grep -E -o '[0-9\.]+' | head -1)
+
+	[ "$want_vers" = "$(echo -e "$vers\n$want_vers" | sort -V | head -1)" ]
+}
+
+build_and_run_tests() {
+	local quick=false
+	if [ "${1:-}" = "--quick" ]; then
+		quick=true
+		shift
+	fi
+
+	begin "CC=$CC CFLAGS=\"$CFLAGS\" WRAPPER=\"$WRAPPER\" $*"
+
+	# Build libdeflate, including the test programs.  Set the special test
+	# support flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
+	$MAKE "$@" TEST_SUPPORT__DO_NOT_USE=1 all test_programs > /dev/null
+
+	# When not using -march=native, run the tests multiple times with
+	# different combinations of CPU features disabled.  This is needed to
+	# test all variants of dynamically-dispatched code.
+	#
+	# For now, we aren't super exhausive in which combinations of features
+	# we test disabling.  We just disable the features roughly in order from
+	# newest to oldest for each architecture, cumulatively.  In practice,
+	# that's good enough to cover all the code.
+	local features=('')
+	if ! [[ "$CFLAGS" =~ "-march=native" ]] && ! $quick; then
+		case "$ARCH" in
+		i386|x86_64)
+			features+=(avx512bw avx2 avx bmi2 pclmul sse2)
+			;;
+		arm*|aarch*)
+			features+=(crc32 pmull neon)
+			;;
+		esac
+	fi
+	local disable_str=""
+	local feature
+	for feature in "${features[@]}"; do
+		if [ -n "$feature" ]; then
+			if [ -n "$disable_str" ]; then
+				disable_str+=","
+			fi
+			disable_str+="$feature"
+		fi
+		log "Using LIBDEFLATE_DISABLE_CPU_FEATURES=$disable_str"
+		LIBDEFLATE_DISABLE_CPU_FEATURES="$disable_str" \
+		    sh ./scripts/exec_tests.sh > /dev/null
+	done
+	end
+}
+
+verify_freestanding_build() {
+	# It is expected that sanitizer builds link to external functions.
+	if [[ "$CFLAGS" =~ "-fsanitize" ]]; then
+		return 0
+	fi
+	log "Verifying that freestanding build is really freestanding"
+	if nm libdeflate.so | grep -q ' U '; then
+		echo 1>&2 "Freestanding lib links to external functions!:"
+		nm libdeflate.so | grep ' U '
+		return 1
+	fi
+	if ldd libdeflate.so | grep -q -v '\<statically linked\>'; then
+		echo 1>&2 "Freestanding lib links to external libraries!:"
+		ldd libdeflate.so
+		return 1
+	fi
+}
+
+gzip_tests() {
+	local gzips=("$PWD/gzip")
+	local gunzips=("$PWD/gunzip")
+	if [ "${1:-}" != "--quick" ]; then
+		gzips+=(/bin/gzip)
+		gunzips+=(/bin/gunzip)
+	fi
+	local gzip gunzip
+
+	begin "Running gzip program tests with CC=\"$CC\" CFLAGS=\"$CFLAGS\""
+	$MAKE gzip gunzip > /dev/null
+	for gzip in "${gzips[@]}"; do
+		for gunzip in "${gunzips[@]}"; do
+			log "GZIP=$gzip, GUNZIP=$gunzip"
+			GZIP="$gzip" GUNZIP="$gunzip" TESTDATA="$TESTDATA" \
+				./scripts/gzip_tests.sh
+		done
+	done
+	end
+}
+
+do_run_tests() {
+	build_and_run_tests "$@"
+	if [ "${1:-}" != "--quick" ]; then
+		if $SKIP_FREESTANDING; then
+			log "Skipping freestanding build tests due to SKIP_FREESTANDING=1"
+		else
+			build_and_run_tests FREESTANDING=1
+			verify_freestanding_build
+		fi
+	fi
+	gzip_tests "$@"
+}
+
+check_symbol_prefixes() {
+	log "Checking that all global symbols are prefixed with \"libdeflate_\""
+	$MAKE libdeflate.a > /dev/null
+	if nm libdeflate.a | grep ' T ' | grep -E -v " _?libdeflate_"; then
+		fail "Some global symbols aren't prefixed with \"libdeflate_\""
+	fi
+	log "Checking that all exported symbols are prefixed with \"libdeflate\""
+	$MAKE libdeflate.so > /dev/null
+	if nm libdeflate.so | grep ' T ' \
+			| grep -E -v " (libdeflate_|_init\>|_fini\>)"; then
+		fail "Some exported symbols aren't prefixed with \"libdeflate_\""
+	fi
+}
+
+test_use_shared_lib() {
+	if $SKIP_SHARED_LIB; then
+		log "Skipping USE_SHARED_LIB=1 tests due to SKIP_SHARED_LIB=1"
+		return
+	fi
+	log "Testing USE_SHARED_LIB=1"
+	$MAKE gzip > /dev/null
+	if ldd gzip | grep -q 'libdeflate.so'; then
+		fail "Binary should be statically linked by default"
+	fi
+	$MAKE USE_SHARED_LIB=1 all check > /dev/null
+	ldd gzip > "$TMPDIR/ldd.out"
+	if ! grep -q 'libdeflate.so' "$TMPDIR/ldd.out"; then
+		cat 1>&2 "$TMPDIR/ldd.out"
+		fail "Binary isn't dynamically linked"
+	fi
+	rm "$TMPDIR/ldd.out"
+}
+
+install_uninstall_tests() {
+	local shell
+
+	begin "Testing 'make install' and 'make uninstall'"
+	for shell in '/bin/bash' '/bin/dash'; do
+		log "Trying SHELL=$shell"
+		$MAKE SHELL=$shell clean > /dev/null
+		$MAKE SHELL=$shell DESTDIR="$TMPDIR/inst" install > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" == 0 )); then
+			fail "'make install' didn't install any files"
+		fi
+		make SHELL=$shell DESTDIR="$TMPDIR/inst" uninstall > /dev/null
+		if (( "$(file_count "$TMPDIR/inst")" != 0 )); then
+			fail "'make uninstall' didn't uninstall all files"
+		fi
+		rm -r "$TMPDIR/inst"
+	done
+	end
+}
+
+run_tests() {
+	export WRAPPER="" # no wrapper by default; overridden by valgrind tests
+	local cflags
+
+	begin "Running tests"
+	do_run_tests
+	end
+
+	cflags=("-O3")
+	if cflags_supported "${cflags[@]}" "-march=native"; then
+		cflags+=("-march=native")
+	fi
+	begin "Running tests with ${cflags[*]}"
+	CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests
+	end
+
+	# Need valgrind 3.9.0 for '--errors-for-leak-kinds=all'
+	# Need valgrind 3.12.0 for armv8 crypto and crc instructions
+	if $SKIP_VALGRIND; then
+		log "Skipping valgrind tests due to SKIP_VALGRIND=1"
+	elif valgrind_version_at_least 3.12.0; then
+		begin "Running tests with Valgrind"
+		WRAPPER="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all" \
+			do_run_tests --quick
+		end
+	fi
+
+	cflags=("-fsanitize=undefined" "-fno-sanitize-recover=undefined")
+	if $SKIP_UBSAN; then
+		log "Skipping UBSAN tests due to SKIP_UBSAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with UBSAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping UBSAN tests because compiler ($CC_VERSION) doesn't support UBSAN"
+	fi
+
+	cflags=("-fsanitize=address" "-fno-sanitize-recover=address")
+	if $SKIP_ASAN; then
+		log "Skipping ASAN tests due to SKIP_ASAN=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with ASAN"
+		CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick
+		end
+	else
+		log "Skipping ASAN tests because compiler ($CC_VERSION) doesn't support ASAN"
+	fi
+
+	cflags=("-fsanitize=cfi" "-fno-sanitize-recover=cfi" "-flto"
+		"-fvisibility=hidden")
+	if $SKIP_CFI; then
+		log "Skipping CFI tests due to SKIP_CFI=1"
+	elif cflags_supported "${cflags[@]}"; then
+		begin "Running tests with CFI"
+		CFLAGS="$CFLAGS ${cflags[*]}" AR=llvm-ar do_run_tests --quick
+		end
+	else
+		log "Skipping CFI tests because compiler ($CC_VERSION) doesn't support CFI"
+	fi
+
+	install_uninstall_tests
+	check_symbol_prefixes
+	test_use_shared_lib
+}
+
+###############################################################################
+
+log "Starting libdeflate tests"
+run_tests
+log "All tests passed!"

tools/arm_test.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-set -e
-
-NDKDIR=/opt/android-ndk
-
-make -j benchmark \
-	CC="$NDKDIR/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin/arm-linux-androideabi-gcc" \
-	CFLAGS="--sysroot=$NDKDIR/platforms/android-12/arch-arm -march=armv7-a -fPIC -pie -mfpu=neon -mfloat-abi=softfp"
-
-adb push benchmark /data/local/tmp
-adb push $HOME/data/testdata  /data/local/tmp
-adb shell /data/local/tmp/benchmark "$@" /data/local/tmp/testdata

tools/make-windows-releases

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-for arch in 'i686' 'x86_64'; do
-	make clean
-	make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all benchmark.exe
-	dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin
-	rm -rf $dir ${dir}.zip
-	mkdir $dir
-	cp libdeflate.dll libdeflate.lib libdeflate.h *.exe $dir
-	${arch}-w64-mingw32-strip ${dir}/libdeflate.dll ${dir}/*.exe
-	for file in COPYING COPYING.LIB NEWS; do
-		sed < $file > ${dir}/${file}.txt -e 's/$/\r/g'
-	done
-	for file in README.md; do
-		sed < $file > ${dir}/${file} -e 's/$/\r/g'
-	done
-	(cd ${dir} && zip -r ../${dir}.zip .)
-done

tools/mips_test.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -e
-
-TOOLCHAIN_DIR=$HOME/src/ddwrt-toolchains/toolchain-mips_34kc_gcc-5.1.0_musl-1.1.9
-
-make -j benchmark \
-	CC="$TOOLCHAIN_DIR/bin/mips-openwrt-linux-musl-gcc" \
-	CFLAGS="-DNEED_PRINTF"
-
-scp benchmark $HOME/data/test root@dd-wrt:
-ssh root@dd-wrt ./benchmark "$@" test

tools/windows_test.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-set -e
-
-make -j CC=i686-w64-mingw32-gcc
-cp -vf *.exe /j/exe/
-make -j CC=x86_64-w64-mingw32-gcc
-cp -vf *.exe /j/exe64/
-
-sudo $HOME/bin/sudo/restart-smbd