Remove references to AVX10/256 and AVX10/512

Intel cancelled these.

Also add a bit more explanation about the attempts to avoid
downclocking.

.cirrus.yml

@@ -1,8 +1,11 @@
 task:
   freebsd_instance:
     matrix:
-      - image_family: freebsd-12-3
-      - image_family: freebsd-13-0
-  install_script: pkg install -y gmake
+      - image_family: freebsd-14-2
+  install_script: pkg install -y cmake
   script:
-    - gmake check
+    - cmake -B build -DLIBDEFLATE_BUILD_TESTS=1
+    - cmake --build build
+    - ctest --test-dir build
+    # Direct compilation without official build system
+    - cc -O2 -Wall -Werror lib/*.c lib/*/*.c programs/gzip.c programs/prog_util.c programs/tgetopt.c -o libdeflate-gzip

.github/workflows/android.yml

@@ -1,104 +0,0 @@
-name: cross compile with android-ndk
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  cross-compile-for-android:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11]
-        target-abi: [armeabi-v7a, arm64-v8a, x86, x86_64]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - run:  brew install tree file
-
-      - run:  tree --version
-      - run:  file --version
-      - run:  make --version
-
-      - uses: actions/checkout@v2
-
-      - name: build for ${{ matrix.target-abi }}
-        run: |
-          COLOR_GREEN='\033[0;32m'        # Green
-          COLOR_PURPLE='\033[0;35m'       # Purple
-          COLOR_OFF='\033[0m'             # Reset
-
-          run() {
-            printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF $COLOR_GREEN$*$COLOR_OFF"
-            eval "$*"
-          }
-
-          BUILD_MACHINE_OS_TYPE=$(uname | tr A-Z a-z)
-          BUILD_MACHINE_OS_ARCH=$(uname -m)
-
-          export ANDROID_NDK_HOME="$ANDROID_NDK_LATEST_HOME"
-
-          env | sed -n '/^ANDROID_/p'
-
-          ANDROID_NDK_VERS=$(grep "Pkg.Revision" "$ANDROID_NDK_HOME/source.properties" | cut -d " " -f3)
-          ANDROID_NDK_BASE=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$BUILD_MACHINE_OS_TYPE-$BUILD_MACHINE_OS_ARCH
-          ANDROID_NDK_BIND=$ANDROID_NDK_BASE/bin
-          SYSROOT=$ANDROID_NDK_BASE/sysroot
-
-          printf '%s = %s\n' ANDROID_NDK_HOME "$ANDROID_NDK_HOME"
-          printf '%s = %s\n' ANDROID_NDK_BIND "$ANDROID_NDK_BIND"
-          printf '%s = %s\n' ANDROID_NDK_VERS "$ANDROID_NDK_VERS"
-          printf '%s\n'
-
-          export CPPFLAGS="--sysroot $SYSROOT"
-          export   CFLAGS="--sysroot $SYSROOT -fpic"
-          export  LDFLAGS="--sysroot $SYSROOT"
-
-          case ${{ matrix.target-abi }} in
-            armeabi-v7a) CC=armv7a-linux-androideabi21-clang ;;
-            arm64-v8a)   CC=aarch64-linux-android21-clang    ;;
-            x86)         CC=i686-linux-android21-clang       ;;
-            x86_64)      CC=x86_64-linux-android21-clang     ;;
-          esac
-
-          AR=llvm-ar
-
-          export PATH="$ANDROID_NDK_BIND:$PATH"
-
-          printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF ${COLOR_GREEN}show PATH$COLOR_OFF"
-          printf '%s\n' "$PATH" | tr ':' '\n'
-          printf '%s\n'
-
-          run make clean install PREFIX=$PWD/install.d CC="$CC" CPPFLAGS="'$CPPFLAGS'" CFLAGS="'$CFLAGS'" AR="$AR" V=1
-
-      - run:  tree install.d
-
-      - run:  |
-          file install.d/lib/libdeflate.so
-          case ${{ matrix.target-abi }} in
-            armeabi-v7a) file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, ARM, EABI5'   ;;
-            arm64-v8a)   file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, ARM aarch64,' ;;
-            x86)         file install.d/lib/libdeflate.so     | grep 'ELF 32-bit LSB shared object, Intel 80386,' ;;
-            x86_64)      file install.d/lib/libdeflate.so     | grep 'ELF 64-bit LSB shared object, x86-64,'      ;;
-          esac
-
-      - run:  |
-          file install.d/bin/libdeflate-gzip
-          case ${{ matrix.target-abi }} in
-            armeabi-v7a) file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
-            arm64-v8a)   file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
-            x86)         file install.d/bin/libdeflate-gzip   | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
-            x86_64)      file install.d/bin/libdeflate-gzip   | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
-          esac
-
-      - run:  |
-          file install.d/bin/libdeflate-gunzip
-          case ${{ matrix.target-abi }} in
-            armeabi-v7a) file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, ARM, EABI5'   ;;
-            arm64-v8a)   file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;;
-            x86)         file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;;
-            x86_64)      file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, x86-64,'      ;;
-          esac

.github/workflows/ci.yml

@@ -1,126 +1,291 @@
 name: CI
 on: [pull_request]
-env:
-  CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS
 
 jobs:
   x86_64-build-and-test:
     name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }})
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-18.04]
+        os: [ubuntu-24.04, ubuntu-22.04]
         compiler: [gcc, clang]
     runs-on: ${{ matrix.os }}
     env:
       CC: ${{ matrix.compiler }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y clang llvm libz-dev valgrind
     - run: scripts/run_tests.sh
+    - name: Direct compilation without official build system
+      run: $CC -O2 -Wall -Werror lib/*{,/*}.c programs/{gzip,prog_util,tgetopt}.c -o libdeflate-gzip
 
   other-arch-build-and-test:
-    name: Build and test (${{ matrix.arch }}, Debian Buster, ${{ matrix.compiler }})
+    name: Build and test (${{ matrix.arch }}, ${{ matrix.distro }}, ${{ matrix.compiler }})
     strategy:
       matrix:
-        arch: [armv6, armv7, aarch64, s390x, ppc64le]
-        compiler: [gcc, clang]
+        include:
+        # - { arch: armv6, distro: bookworm, compiler: gcc }
+        # - { arch: armv6, distro: bookworm, compiler: clang }
+        - { arch: armv7, distro: bookworm, compiler: gcc }
+        - { arch: armv7, distro: bookworm, compiler: clang }
+        - { arch: aarch64, distro: bookworm, compiler: gcc }
+        - { arch: aarch64, distro: bookworm, compiler: clang }
+        - { arch: s390x, distro: bookworm, compiler: gcc }
+        - { arch: s390x, distro: bookworm, compiler: clang }
+        - { arch: ppc64le, distro: bookworm, compiler: gcc }
+        - { arch: ppc64le, distro: bookworm, compiler: clang }
+        - { arch: riscv64, distro: ubuntu_latest, compiler: gcc }
+        - { arch: riscv64, distro: ubuntu_latest, compiler: clang }
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: uraimo/run-on-arch-action@v2.0.5
+      - uses: actions/checkout@v4
+      - uses: uraimo/run-on-arch-action@v3.0.0
         with:
           arch: ${{ matrix.arch }}
-          distro: buster
+          distro: ${{ matrix.distro }}
           githubToken: ${{ github.token }}
           install: |
             apt-get update
-            apt-get install -y build-essential clang llvm libz-dev valgrind
+            apt-get install -y build-essential cmake clang llvm libz-dev
           run: |
-            # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64
-            # here.  (It's probably something related to the QEMU user-mode
-            # emulation that run-on-arch-action uses.)
-            export SKIP_VALGRIND=1
-            export SKIP_ASAN=1
-
-            case ${{ matrix.arch }} in
-            s390x)
-              # On s390x, in freestanding builds the shared library links to an
-              # external symbol __clzdi2, even when -static-libgcc is used.
-              export SKIP_FREESTANDING=1
-              ;;
-            aarch64)
-              # "ldd: exited with unknown exit code (139)"
-              if [ ${{ matrix.compiler }} = clang ]; then
-                export SKIP_SHARED_LIB=1
-              fi
-              ;;
-            esac
-
-            export CC=${{ matrix.compiler }}
-            scripts/run_tests.sh
+            tests=(regular)
+            if [ ${{matrix.compiler}} = clang ]; then
+                tests+=(ubsan)
+            fi
+            CC=${{matrix.compiler}} scripts/run_tests.sh "${tests[@]}"
 
   macos-build-and-test:
     name: Build and test (macOS)
     runs-on: macos-latest
+    env:
+      CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS
     steps:
-    - uses: actions/checkout@v2
-    - run: make all check
+    - uses: actions/checkout@v4
+    - run: cmake -B build -DLIBDEFLATE_BUILD_TESTS=1
+    - run: cmake --build build --verbose
+    - run: DESTDIR=build/install cmake --install build --verbose
+    - run: ctest --test-dir build
+    - name: Direct compilation without official build system
+      run: cc -O2 -Wall -Werror lib/*{,/*}.c programs/{gzip,prog_util,tgetopt}.c -o libdeflate-gzip
 
   windows-msys2-build-and-test:
-    name: Build and test (Windows, MSYS2)
+    name: Build and test (Windows, MSYS2, ${{matrix.sys}})
     runs-on: windows-latest
+    strategy:
+      matrix:
+        include:
+        - { sys: mingw64, env: x86_64 }
+        - { sys: mingw32, env: i686 }
+    defaults:
+      run:
+        shell: msys2 {0}
+    env:
+      CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS
     steps:
-    - uses: actions/checkout@v2
-    - shell: bash
-      run: |
-        PATH="C:\\msys64\\mingw64\\bin:C:\\msys64\\usr\\bin:$PATH" \
-          make CC=gcc all check
+    - uses: actions/checkout@v4
+    - uses: msys2/setup-msys2@v2
+      with:
+        msystem: ${{matrix.sys}}
+        update: true
+        install: >
+          make
+          mingw-w64-${{matrix.env}}-cc
+          mingw-w64-${{matrix.env}}-cmake
+          mingw-w64-${{matrix.env}}-ninja
+          mingw-w64-${{matrix.env}}-zlib
+    # Note: as per the CMake documentation, DESTDIR is unsupported on Windows.
+    - run: cmake -B build -G Ninja -DLIBDEFLATE_BUILD_TESTS=1 -DCMAKE_INSTALL_PREFIX=build\install
+    - run: cmake --build build --verbose
+    - run: cmake --install build --verbose
+    - run: ctest --test-dir build
+    - name: Direct compilation without official build system
+      run: cc -O2 -Wall -Werror -municode lib/*{,/*}.c programs/{gzip,prog_util,tgetopt}.c -o libdeflate-gzip.exe
 
-  windows-msvc-build:
-    name: Build (Windows, Visual Studio)
+  windows-visualstudio-build-and-test:
+    name: Build and test (Windows, ${{matrix.gen}}, ${{matrix.toolset}}, ${{matrix.vs}})
+    strategy:
+      matrix:
+        include:
+        - {os: windows-2022, gen: "Visual Studio 17 2022", toolset: v143,    vs: x64,   vcpkg: x64-windows}
+        - {os: windows-2022, gen: "Visual Studio 17 2022", toolset: ClangCL, vs: x64,   vcpkg: x64-windows}
+        - {os: windows-2022, gen: "Visual Studio 17 2022", toolset: v143,    vs: Win32, vcpkg: x86-windows}
+        - {os: windows-2022, gen: "Visual Studio 17 2022", toolset: ClangCL, vs: Win32, vcpkg: x86-windows}
+        - {os: windows-2019, gen: "Visual Studio 16 2019", toolset: v142,    vs: x64,   vcpkg: x64-windows}
+        - {os: windows-2019, gen: "Visual Studio 16 2019", toolset: v142,    vs: Win32, vcpkg: x86-windows}
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: microsoft/setup-msbuild@v2
+    - run: vcpkg install zlib:${{matrix.vcpkg}}
+    - run: >
+        echo C:\vcpkg\packages\zlib_${{matrix.vcpkg}}\bin
+        | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+    # Note: as per the CMake documentation, DESTDIR is unsupported on Windows.
+    - run: >
+        cmake -B build -G "${{matrix.gen}}" -T ${{matrix.toolset}}
+        -A ${{matrix.vs}} -DLIBDEFLATE_BUILD_TESTS=1
+        -DCMAKE_C_FLAGS="/W4 /WX /DLIBDEFLATE_ENABLE_ASSERTIONS"
+        -DZLIB_LIBRARY=C:\vcpkg\packages\zlib_${{matrix.vcpkg}}\lib\zlib.lib
+        -DZLIB_INCLUDE_DIR=C:\vcpkg\packages\zlib_${{matrix.vcpkg}}\include
+        -DCMAKE_INSTALL_PREFIX=build\install
+    - run: cmake --build build --verbose --config Debug
+    - run: cmake --install build --verbose --config Debug
+    - run: ctest --test-dir build -C Debug
+
+  windows-visualstudio-build:
+    name: Build (Windows, Visual Studio ${{matrix.toolset}}, ${{matrix.platform}})
+    strategy:
+      matrix:
+        platform: [ARM64, ARM64EC]
+        toolset: [v143, ClangCL]
+        exclude: # Exclude unsupported combinations
+        - platform: ARM64EC
+          toolset: ClangCL
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: ilammy/msvc-dev-cmd@v1
-    - run: nmake /f Makefile.msc
+    - uses: actions/checkout@v4
+    - uses: microsoft/setup-msbuild@v2
+    # Note: as per the CMake documentation, DESTDIR is unsupported on Windows.
+    - run: >
+        cmake -B build -G "Visual Studio 17 2022" -T ${{matrix.toolset}}
+        -A ${{matrix.platform}} -DCMAKE_C_FLAGS="/W4 /WX"
+        -DCMAKE_INSTALL_PREFIX=build\install -DCMAKE_SYSTEM_VERSION=10
+    - run: cmake --build build --verbose --config Release
+    - run: cmake --install build --verbose --config Release
 
   run-clang-static-analyzer:
     name: Run clang static analyzer
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y clang-tools
-    - name: Run clang static analyzer
-      run: make scan-build
+    - run: scan-build cmake -B build -DLIBDEFLATE_BUILD_TESTS=1
+    - run: scan-build cmake --build build --verbose
 
   run-shellcheck:
     name: Run shellcheck
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y shellcheck
     - name: Run shellcheck
-      run: make shellcheck
+      run: shellcheck scripts/*.sh
 
   cross-compile-for-windows:
     name: Cross compile for Windows
     runs-on: ubuntu-latest
+    env:
+      CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev
+    # Unfortunately Ubuntu doesn't have {i686,x86_64}-w64-mingw32-cmake like
+    # some distros have, so we have to provide our own toolchain files here.
     - name: 32-bit build
-      run: make CC=i686-w64-mingw32-gcc all test_programs
+      run: |
+        scripts/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 \
+            -DCMAKE_TOOLCHAIN_FILE=scripts/toolchain-i686-w64-mingw32.cmake
+        cmake --build build --verbose
+        DESTDIR=build/install cmake --install build --verbose
     - name: 64-bit build
-      run: make CC=x86_64-w64-mingw32-gcc all test_programs
+      run: |
+        scripts/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 \
+            -DCMAKE_TOOLCHAIN_FILE=scripts/toolchain-x86_64-w64-mingw32.cmake
+        cmake --build build --verbose
+        DESTDIR=build/install cmake --install build --verbose
+
+  cross-compile-for-android:
+    name: Cross compile for ${{matrix.abi}} Android on ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        abi: [armeabi-v7a, arm64-v8a, x86, x86_64]
+    runs-on: ${{matrix.os}}
+    env:
+      CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS
+    steps:
+    - uses: actions/checkout@v4
+    - run: |
+        scripts/cmake-helper.sh \
+            -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK_LATEST_HOME"/build/cmake/android.toolchain.cmake \
+            -DANDROID_ABI=${{matrix.abi}} \
+            -DANDROID_PLATFORM=28 \
+            -DLIBDEFLATE_BUILD_TESTS=1
+        cmake --build build --verbose
+        DESTDIR=build/install cmake --install build --verbose
+
+  cpu-features-regression-tests:
+    name: Test building adler32.c and crc32.c with various flags
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+    - name: Compile tests
+      run: |
+        cflags="-O0 -Wall -Werror"
+        for file in lib/adler32.c lib/crc32.c; do
+          echo "arm32, default options, file=$file"
+          arm-linux-gnueabihf-gcc -c $cflags $file
+          for arch in armv4 armv4t armv5t armv5te armv5tej armv6 armv6j armv6k \
+                      armv6z armv6kz armv6zk armv6t2; do
+            echo "arm32, -march=$arch, file=$file"
+            arm-linux-gnueabihf-gcc -c -march=$arch -mfpu=vfp -marm $cflags $file
+          done
+          for arch in armv7 armv7-a armv7ve armv7-r armv7-m armv7e-m; do
+            echo "arm32, -march=$arch, file=$file"
+            arm-linux-gnueabihf-gcc -c -march=$arch -mfpu=vfp $cflags $file
+          done
+          echo "arm64, -mcpu=emag"
+          aarch64-linux-gnu-gcc -c -mcpu=emag $cflags $file
+        done
+
+  fuzz-with-libFuzzer:
+    name: Fuzz with libFuzzer (${{matrix.target}} ${{matrix.sanitizer}})
+    strategy:
+      matrix:
+        include:
+        - target: deflate_compress
+          sanitizer:
+        - target: deflate_compress
+          sanitizer: --asan
+        - target: deflate_compress
+          sanitizer: --msan
+        - target: deflate_compress
+          sanitizer: --ubsan
+        - target: deflate_decompress
+          sanitizer:
+        - target: deflate_decompress
+          sanitizer: --asan
+        - target: deflate_decompress
+          sanitizer: --msan
+        - target: deflate_decompress
+          sanitizer: --ubsan
+        - target: zlib_decompress
+          sanitizer:
+        - target: gzip_decompress
+          sanitizer:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang llvm
+    - name: Fuzz
+      run: |
+        scripts/libFuzzer/fuzz.sh --time=120 ${{matrix.sanitizer}} \
+            ${{matrix.target}}

.gitignore

@@ -1,21 +1,5 @@
-*.a
-*.def
-*.dll
-*.dllobj
-*.dylib
-*.exe
-*.exp
-*.lib
-*.o
-*.obj
-*.so
-*.so.*
-/.build-config
-/programs/config.h
-/benchmark
-/checksum
-/gzip
-/gunzip
-/test_*
-tags
+/build*
+/libdeflate-*-windows-*
+/libdeflate-*.tar.gz
 cscope*
+tags

CMakeLists.txt

@@ -0,0 +1,339 @@
+cmake_minimum_required(VERSION 3.10)
+
+# Default to a release build.
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+    message(STATUS "No build type selected; defaulting to Release")
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build" FORCE)
+endif()
+
+# With MSVC, don't automatically append /W3 to the compiler flags.
+# This makes it possible for the user to select /W4.
+if(POLICY CMP0092)
+    cmake_policy(SET CMP0092 NEW)
+endif()
+
+# Extract the version string from libdeflate.h so that it doesn't have to be
+# duplicated here.
+set(VERSION_REGEX "#define LIBDEFLATE_VERSION_STRING[ \t]+\"([0-9\\.]+)\"")
+file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/libdeflate.h VERSION_STRING REGEX ${VERSION_REGEX})
+string(REGEX REPLACE ${VERSION_REGEX} "\\1" VERSION_STRING "${VERSION_STRING}")
+
+# Declare the project.
+project(libdeflate
+        LANGUAGES C
+        VERSION ${VERSION_STRING})
+
+# Include the CMake modules required by the top-level directory.
+include(CMakePackageConfigHelpers)
+include(CheckCCompilerFlag)
+include(GNUInstallDirs)
+
+# Declare the options, which can be overridden via 'cmake -DOPTION=VALUE'.
+option(LIBDEFLATE_BUILD_STATIC_LIB "Build the static library" ON)
+option(LIBDEFLATE_BUILD_SHARED_LIB "Build the shared library" ON)
+option(LIBDEFLATE_COMPRESSION_SUPPORT "Support compression" ON)
+option(LIBDEFLATE_DECOMPRESSION_SUPPORT "Support decompression" ON)
+option(LIBDEFLATE_ZLIB_SUPPORT "Support the zlib format" ON)
+option(LIBDEFLATE_GZIP_SUPPORT "Support the gzip format" ON)
+option(LIBDEFLATE_FREESTANDING
+       "Build a freestanding library, i.e. a library that doesn't link to any
+       libc functions like malloc(), free(), and memcpy().  Library users will
+       need to provide a custom memory allocator." OFF)
+option(LIBDEFLATE_BUILD_GZIP "Build the libdeflate-gzip program" ON)
+option(LIBDEFLATE_BUILD_TESTS "Build the test programs" OFF)
+option(LIBDEFLATE_USE_SHARED_LIB
+       "Link the libdeflate-gzip and test programs to the shared library instead
+       of the static library" OFF)
+if(APPLE)
+    option(LIBDEFLATE_APPLE_FRAMEWORK "Build as Apple Framework" OFF)
+endif()
+
+if(LIBDEFLATE_BUILD_TESTS)
+    enable_testing()
+endif()
+
+# The gzip program can't be built if any library feature it needs is disabled.
+if(NOT LIBDEFLATE_COMPRESSION_SUPPORT OR NOT LIBDEFLATE_DECOMPRESSION_SUPPORT
+   OR NOT LIBDEFLATE_GZIP_SUPPORT)
+    set(LIBDEFLATE_BUILD_GZIP OFF)
+endif()
+
+# If the static library isn't being built, we have to link to the shared one.
+if(NOT LIBDEFLATE_BUILD_STATIC_LIB)
+    set(LIBDEFLATE_USE_SHARED_LIB ON)
+endif()
+
+# Set common C compiler flags for all targets (the library and the programs).
+set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
+set(CMAKE_C_STANDARD 99)
+if(NOT MSVC)
+    check_c_compiler_flag(-Wdeclaration-after-statement HAVE_WDECLARATION_AFTER_STATEMENT)
+    check_c_compiler_flag(-Wimplicit-fallthrough HAVE_WIMPLICIT_FALLTHROUGH)
+    check_c_compiler_flag(-Wmissing-field-initializers HAVE_WMISSING_FIELD_INITIALIZERS)
+    check_c_compiler_flag(-Wmissing-prototypes HAVE_WMISSING_PROTOTYPES)
+    check_c_compiler_flag(-Wpedantic HAVE_WPEDANTIC)
+    check_c_compiler_flag(-Wshadow HAVE_WSHADOW)
+    check_c_compiler_flag(-Wstrict-prototypes HAVE_WSTRICT_PROTOTYPES)
+    check_c_compiler_flag(-Wundef HAVE_WUNDEF)
+    check_c_compiler_flag(-Wvla HAVE_WVLA)
+    add_compile_options(
+        -Wall
+        $<$<BOOL:${HAVE_WDECLARATION_AFTER_STATEMENT}>:-Wdeclaration-after-statement>
+        $<$<BOOL:${HAVE_WIMPLICIT_FALLTHROUGH}>:-Wimplicit-fallthrough>
+        $<$<BOOL:${HAVE_WMISSING_FIELD_INITIALIZERS}>:-Wmissing-field-initializers>
+        $<$<BOOL:${HAVE_WMISSING_PROTOTYPES}>:-Wmissing-prototypes>
+        $<$<BOOL:${HAVE_WPEDANTIC}>:-Wpedantic>
+        $<$<BOOL:${HAVE_WSHADOW}>:-Wshadow>
+        $<$<BOOL:${HAVE_WSTRICT_PROTOTYPES}>:-Wstrict-prototypes>
+        $<$<BOOL:${HAVE_WUNDEF}>:-Wundef>
+        $<$<BOOL:${HAVE_WVLA}>:-Wvla>
+    )
+endif()
+if(LIBDEFLATE_FREESTANDING)
+    add_definitions(-DFREESTANDING)
+endif()
+
+# Check for cases where the compiler supports an instruction set extension but
+# the assembler does not, and in those cases print a warning and add an
+# appropriate -DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_* flag.  libdeflate's C
+# source files already check the compiler version before using the corresponding
+# intrinsics, but in the rare case of gcc being paired with a binutils much
+# older than itself those checks are insufficient.  There is no way to check the
+# assembler version from C.  The proper fix for too-old binutils is for the user
+# to upgrade binutils.  Unfortunately, as libdeflate has started using newer
+# instructions, binutils incompatibilities have started being seen more
+# frequently.  Hence these checks for assembler support here in CMakeLists.txt
+# to provide a fallback for users who may be unable to fix their toolchain.
+# These don't solve the problem for users not using CMake, though such users can
+# add specific -DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_* flags they need.
+function(check_assembler_support  feature assembly_code)
+    execute_process(COMMAND echo "${assembly_code}"
+                    COMMAND ${CMAKE_C_COMPILER} -c -x assembler -o /dev/null -
+                    RESULT_VARIABLE result
+                    ERROR_QUIET)
+    if(NOT ${result} EQUAL 0)
+        add_definitions(-DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_${feature})
+        message(STATUS "Your gcc supports ${feature} instructions but it is paired with an assembler that does not.  Upgrading binutils is recommended.")
+    endif()
+endfunction()
+if(UNIX AND CMAKE_C_COMPILER_ID STREQUAL "GNU")
+    execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpmachine
+                    OUTPUT_VARIABLE machine)
+    if(${machine} MATCHES "^(x86_64|i[3-6]86)")
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI if needed.
+            check_assembler_support(AVX512VNNI "vpdpbusd %zmm0, %zmm0, %zmm0")
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ if needed.
+            check_assembler_support(VPCLMULQDQ "vpclmulqdq $0, %zmm0, %zmm0, %zmm0")
+        endif()
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI if needed.
+            check_assembler_support(AVX_VNNI "{vex} vpdpbusd %ymm0, %ymm0, %ymm0")
+        endif()
+    elseif(${machine} MATCHES "^aarch64")
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD if needed.
+            check_assembler_support(DOTPROD ".arch armv8.2-a+dotprod\nudot v0.4s, v0.16b, v0.16b")
+        endif()
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 9.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3 if needed.
+            check_assembler_support(SHA3 ".arch armv8.2-a+sha3\neor3 v0.16b, v0.16b, v0.16b, v0.16b")
+        endif()
+    endif()
+endif()
+
+# Determine the list of source files and the list of compiler options that will
+# be used for both the static library and the shared library.
+
+set(LIB_SOURCES
+    common_defs.h
+    libdeflate.h
+    lib/arm/cpu_features.c
+    lib/arm/cpu_features.h
+    lib/cpu_features_common.h
+    lib/deflate_constants.h
+    lib/lib_common.h
+    lib/utils.c
+    lib/x86/cpu_features.c
+    lib/x86/cpu_features.h
+)
+if(LIBDEFLATE_COMPRESSION_SUPPORT)
+    list(APPEND LIB_SOURCES
+         lib/arm/matchfinder_impl.h
+         lib/bt_matchfinder.h
+         lib/deflate_compress.c
+         lib/deflate_compress.h
+         lib/hc_matchfinder.h
+         lib/ht_matchfinder.h
+         lib/matchfinder_common.h
+         lib/riscv/matchfinder_impl.h
+         lib/x86/matchfinder_impl.h
+    )
+endif()
+if(LIBDEFLATE_DECOMPRESSION_SUPPORT)
+    list(APPEND LIB_SOURCES
+         lib/decompress_template.h
+         lib/deflate_decompress.c
+         lib/x86/decompress_impl.h
+    )
+endif()
+if(LIBDEFLATE_ZLIB_SUPPORT)
+    list(APPEND LIB_SOURCES
+         lib/adler32.c
+         lib/arm/adler32_impl.h
+         lib/x86/adler32_impl.h
+         lib/x86/adler32_template.h
+         lib/zlib_constants.h
+    )
+    if(LIBDEFLATE_COMPRESSION_SUPPORT)
+        list(APPEND LIB_SOURCES lib/zlib_compress.c)
+    endif()
+    if(LIBDEFLATE_DECOMPRESSION_SUPPORT)
+        list(APPEND LIB_SOURCES lib/zlib_decompress.c)
+    endif()
+endif()
+if(LIBDEFLATE_GZIP_SUPPORT)
+    list(APPEND LIB_SOURCES
+         lib/arm/crc32_impl.h
+         lib/arm/crc32_pmull_helpers.h
+         lib/arm/crc32_pmull_wide.h
+         lib/crc32.c
+         lib/crc32_multipliers.h
+         lib/crc32_tables.h
+         lib/gzip_constants.h
+         lib/x86/crc32_impl.h
+         lib/x86/crc32_pclmul_template.h
+    )
+    if(LIBDEFLATE_COMPRESSION_SUPPORT)
+        list(APPEND LIB_SOURCES lib/gzip_compress.c)
+    endif()
+    if(LIBDEFLATE_DECOMPRESSION_SUPPORT)
+        list(APPEND LIB_SOURCES lib/gzip_decompress.c)
+    endif()
+endif()
+
+if(LIBDEFLATE_FREESTANDING)
+    list(APPEND LIB_COMPILE_OPTIONS -ffreestanding -nostdlib)
+    list(APPEND LIB_LINK_LIBRARIES -ffreestanding -nostdlib)
+endif()
+
+set(LIB_INCLUDE_DIRS
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_FULL_INCLUDEDIR}>)
+
+# Build the static library.
+if(LIBDEFLATE_BUILD_STATIC_LIB)
+    add_library(libdeflate_static STATIC ${LIB_SOURCES})
+
+    # This alias allows third-party usage of the library with CMake to work the
+    # same way with add_subdirectory() as with other ways.
+    add_library(libdeflate::libdeflate_static ALIAS libdeflate_static)
+
+    if(WIN32 AND NOT MINGW)
+        set(STATIC_LIB_NAME deflatestatic)
+    else()
+        set(STATIC_LIB_NAME deflate)
+    endif()
+    set_target_properties(libdeflate_static PROPERTIES
+                          OUTPUT_NAME ${STATIC_LIB_NAME}
+                          PUBLIC_HEADER libdeflate.h)
+    if(LIBDEFLATE_APPLE_FRAMEWORK)
+        set_target_properties(libdeflate_static PROPERTIES
+            FRAMEWORK TRUE
+            FRAMEWORK_VERSION "${VERSION_STRING}"
+            PRODUCT_BUNDLE_IDENTIFIER "github.com/ebiggers/libdeflate"
+            XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
+            XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
+            XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+            XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
+            MACOSX_FRAMEWORK_IDENTIFIER "github.com/ebiggers/libdeflate"
+            MACOSX_FRAMEWORK_BUNDLE_VERSION "${VERSION_STRING}"
+            MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${PROJECT_VERSION}"
+            MACOSX_RPATH TRUE)
+    endif()
+
+    target_include_directories(libdeflate_static PUBLIC ${LIB_INCLUDE_DIRS})
+    target_compile_definitions(libdeflate_static PRIVATE ${LIB_COMPILE_DEFINITIONS})
+    target_compile_options(libdeflate_static PRIVATE ${LIB_COMPILE_OPTIONS})
+    list(APPEND LIB_TARGETS libdeflate_static)
+endif()
+
+# Build the shared library.
+if(LIBDEFLATE_BUILD_SHARED_LIB)
+    add_library(libdeflate_shared SHARED ${LIB_SOURCES})
+
+    # This alias allows third-party usage of the library with CMake to work the
+    # same way with add_subdirectory() as with other ways.
+    add_library(libdeflate::libdeflate_shared ALIAS libdeflate_shared)
+
+    set_target_properties(libdeflate_shared PROPERTIES
+                          OUTPUT_NAME deflate
+                          PUBLIC_HEADER libdeflate.h
+                          C_VISIBILITY_PRESET hidden
+                          SOVERSION 0)
+    target_include_directories(libdeflate_shared PUBLIC ${LIB_INCLUDE_DIRS})
+    target_compile_definitions(libdeflate_shared PUBLIC LIBDEFLATE_DLL)
+    target_compile_definitions(libdeflate_shared PRIVATE ${LIB_COMPILE_DEFINITIONS})
+    target_compile_options(libdeflate_shared PRIVATE ${LIB_COMPILE_OPTIONS})
+    target_link_libraries(libdeflate_shared PRIVATE ${LIB_LINK_LIBRARIES})
+    list(APPEND LIB_TARGETS libdeflate_shared)
+endif()
+
+# Install the static and/or shared library.
+install(TARGETS ${LIB_TARGETS}
+        EXPORT libdeflate_exported_targets
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        FRAMEWORK DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT runtime OPTIONAL
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+        PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+# Generate and install the pkg-config file.  (Don't confuse this with the CMake
+# package config file, which is CMake-specific.)  Take care to define the
+# include and lib directories in terms of the ${prefix} and ${exec_prefix}
+# pkg-config variables when possible, since some pkg-config users expect to be
+# able to override these variables to relocate packages.
+if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
+    set(CMAKE_PKGCONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}")
+else()
+    set(CMAKE_PKGCONFIG_INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
+endif()
+if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
+    set(CMAKE_PKGCONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+else()
+    set(CMAKE_PKGCONFIG_LIBDIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
+endif()
+configure_file(libdeflate.pc.in libdeflate.pc @ONLY)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libdeflate.pc
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+
+# Generate a "libdeflate-targets.cmake" file in the build tree that can be
+# included by outside projects to import targets from the build tree.
+export(EXPORT libdeflate_exported_targets
+       NAMESPACE libdeflate::
+       FILE libdeflate-targets.cmake)
+
+# Generate and install a separate "libdeflate-targets.cmake" file that can be
+# included by outside projects to import targets from the installation tree.
+install(EXPORT libdeflate_exported_targets
+        NAMESPACE libdeflate::
+        FILE libdeflate-targets.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate)
+
+# Generate and install the CMake package version and config files.
+write_basic_package_version_file(libdeflate-config-version.cmake
+                                 VERSION ${PROJECT_VERSION}
+                                 COMPATIBILITY AnyNewerVersion)
+configure_package_config_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/libdeflate-config.cmake.in
+    libdeflate-config.cmake
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libdeflate-config.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/libdeflate-config-version.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate)
+
+# Build the programs subdirectory if needed.
+if(LIBDEFLATE_BUILD_GZIP OR LIBDEFLATE_BUILD_TESTS)
+    add_subdirectory(programs)
+endif()

COPYING

@@ -1,4 +1,5 @@
 Copyright 2016 Eric Biggers
+Copyright 2024 Google LLC
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation files

Makefile

@@ -1,424 +0,0 @@
-#
-# Use 'make help' to list available targets.
-#
-# Define V=1 to enable "verbose" mode, showing all executed commands.
-#
-# Define USE_SHARED_LIB to link the binaries to the shared library version of
-# libdeflate rather than to the static library version.
-#
-# Define DISABLE_SHARED to disable building and installing of shared library.
-# Defining this variable undefines USE_SHARED_LIB.
-#
-# Define DECOMPRESSION_ONLY to omit all compression code, building a
-# decompression-only library.  If doing this, you must also build a specific
-# library target such as 'libdeflate.a', as the programs will no longer compile.
-#
-# Define DISABLE_GZIP to disable support for the gzip wrapper format.
-#
-# Define DISABLE_ZLIB to disable support for the zlib wrapper format.
-#
-# Define PREFIX to override the installation prefix, like './configure --prefix'
-# in autotools-based projects (default: /usr/local)
-#
-# Define BINDIR to override where to install binaries, like './configure
-# --bindir' in autotools-based projects (default: PREFIX/bin)
-#
-# Define INCDIR to override where to install headers, like './configure
-# --includedir' in autotools-based projects (default: PREFIX/include)
-#
-# Define LIBDIR to override where to install libraries, like './configure
-# --libdir' in autotools-based projects (default: PREFIX/lib)
-#
-# Define DESTDIR to override the installation destination directory
-# (default: empty string)
-#
-# Define FREESTANDING to build a freestanding library, i.e. a library that
-# doesn't link to any libc functions like malloc(), free(), and memcpy().
-# All users will need to call libdeflate_set_memory_allocator().
-#
-# You can also specify a custom CC, CFLAGS, CPPFLAGS, and/or LDFLAGS.
-#
-##############################################################################
-
-VERSION = $(shell sed -n 's/\#define LIBDEFLATE_VERSION_STRING.*"\(.*\)"/\1/p' \
-	    libdeflate.h)
-
-#### Common compiler flags.  You can add additional flags by defining CFLAGS
-#### in the environment or on the 'make' command line.
-####
-#### The default optimization flags can be overridden, e.g. via CFLAGS="-O3" or
-#### CFLAGS="-O0 -fno-omit-frame-pointer".  But this usually isn't recommended;
-#### you're unlikely to get significantly better performance even with -O3.
-
-cc-option = $(shell if $(CC) $(1) -c -x c /dev/null -o /dev/null \
-	      1>&2 2>/dev/null; then echo $(1); fi)
-
-override CFLAGS :=							\
-	-O2 -fomit-frame-pointer -std=c99 -I. -Wall -Wundef		\
-	$(call cc-option,-Wpedantic)					\
-	$(call cc-option,-Wdeclaration-after-statement)			\
-	$(call cc-option,-Wmissing-prototypes)				\
-	$(call cc-option,-Wstrict-prototypes)				\
-	$(call cc-option,-Wvla)						\
-	$(call cc-option,-Wimplicit-fallthrough)			\
-	$(CFLAGS)
-
-FREESTANDING :=
-ifdef FREESTANDING
-override CPPFLAGS += -DFREESTANDING
-LIB_CFLAGS += -ffreestanding -nostdlib
-endif
-
-# Don't use this option except for testing; it isn't a stable interface.
-TEST_SUPPORT__DO_NOT_USE :=
-ifdef TEST_SUPPORT__DO_NOT_USE
-override CPPFLAGS += -DTEST_SUPPORT__DO_NOT_USE
-endif
-
-##############################################################################
-
-PREFIX ?= /usr/local
-BINDIR ?= $(PREFIX)/bin
-INCDIR ?= $(PREFIX)/include
-LIBDIR ?= $(PREFIX)/lib
-
-SOVERSION          := 0
-
-STATIC_LIB_SUFFIX  := .a
-PROG_SUFFIX        :=
-PROG_CFLAGS        :=
-HARD_LINKS         := 1
-
-TARGET_MACHINE     := $(shell $(CC) -dumpmachine 2>/dev/null)
-
-# Compiling for Windows with MinGW or LLVM?
-ifneq ($(findstring -mingw,$(TARGET_MACHINE))$(findstring -windows-gnu,$(TARGET_MACHINE)),)
-    STATIC_LIB_SUFFIX  := static.lib
-    SHARED_LIB         := libdeflate.dll
-    SHARED_LIB_SYMLINK :=
-    SHARED_LIB_CFLAGS  :=
-    SHARED_LIB_LDFLAGS := -Wl,--out-implib,libdeflate.lib \
-                          -Wl,--output-def,libdeflate.def
-    # Only if not LLVM
-    ifeq ($(findstring -windows-gnu,$(TARGET_MACHINE)),)
-        SHARED_LIB_LDFLAGS += -Wl,--add-stdcall-alias
-    endif
-
-    PROG_SUFFIX        := .exe
-    PROG_CFLAGS        := -static -municode
-    HARD_LINKS         :=
-    override CFLAGS    := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format)
-
-    # If AR was not already overridden, then derive it from $(CC).
-    # Note that CC may take different forms, e.g. "cc", "gcc", "clang",
-    # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1", or
-    # "x86_64-w64-mingw32-clang".
-    # On Windows it may also have a .exe extension.
-    ifeq ($(AR),ar)
-        AR := $(shell echo $(CC) | \
-                sed -E 's/(g?cc|clang)(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\4/')
-    endif
-
-# Compiling for macOS?
-else ifneq ($(findstring -darwin,$(TARGET_MACHINE)),)
-   SHARED_LIB         := libdeflate.$(SOVERSION).dylib
-   SHARED_LIB_SYMLINK := libdeflate.dylib
-   SHARED_LIB_CFLAGS  := -fPIC
-   SHARED_LIB_LDFLAGS := -install_name $(LIBDIR)/$(SHARED_LIB)
-
-# Compiling for Android?
-else ifneq ($(findstring -android,$(TARGET_MACHINE)),)
-   SHARED_LIB         := libdeflate.so
-   SHARED_LIB_SYMLINK :=
-   SHARED_LIB_CFLAGS  := -fPIC
-   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
-
-# Compiling for Haiku?
-else ifneq ($(findstring -haiku,$(TARGET_MACHINE)),)
-   SHARED_LIB         := libdeflate.so.$(SOVERSION)
-   SHARED_LIB_SYMLINK := libdeflate.so
-   SHARED_LIB_CFLAGS  := -fPIC
-   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
-   HARD_LINKS         :=
-
-# Compiling for Linux, FreeBSD, etc.
-else
-   SHARED_LIB         := libdeflate.so.$(SOVERSION)
-   SHARED_LIB_SYMLINK := libdeflate.so
-   SHARED_LIB_CFLAGS  := -fPIC
-   SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB)
-endif
-
-##############################################################################
-
-#### Quiet make is enabled by default.  Define V=1 to disable.
-
-ifneq ($(findstring s,$(MAKEFLAGS)),s)
-ifneq ($(V),1)
-        QUIET_CC       = @echo '  CC      ' $@;
-        QUIET_CCLD     = @echo '  CCLD    ' $@;
-        QUIET_AR       = @echo '  AR      ' $@;
-        QUIET_LN       = @echo '  LN      ' $@;
-        QUIET_CP       = @echo '  CP      ' $@;
-        QUIET_GEN      = @echo '  GEN     ' $@;
-endif
-endif
-
-##############################################################################
-
-# Rebuild if a user-specified setting that affects the build changed.
-.build-config: FORCE
-	@flags=$$(							\
-		echo 'USE_SHARED_LIB=$(USE_SHARED_LIB)';		\
-		echo 'DISABLE_SHARED=$(DISABLE_SHARED)';		\
-		echo 'DECOMPRESSION_ONLY=$(DECOMPRESSION_ONLY)';	\
-		echo 'DISABLE_GZIP=$(DISABLE_GZIP)';			\
-		echo 'DISABLE_ZLIB=$(DISABLE_ZLIB)';			\
-		echo 'FREESTANDING=$(FREESTANDING)';			\
-		echo 'CC=$(CC)';					\
-		echo 'CFLAGS=$(CFLAGS)';				\
-		echo 'CPPFLAGS=$(CPPFLAGS)';				\
-		echo 'LDFLAGS=$(LDFLAGS)';				\
-	);								\
-	if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then		\
-		[ -e $@ ] && echo "Rebuilding due to new settings";	\
-		echo "$$flags" > $@;					\
-	fi
-
-##############################################################################
-
-COMMON_HEADERS := $(wildcard common/*.h) libdeflate.h
-DEFAULT_TARGETS :=
-
-#### Library
-
-STATIC_LIB := libdeflate$(STATIC_LIB_SUFFIX)
-
-LIB_CFLAGS += $(CFLAGS) -fvisibility=hidden -D_ANSI_SOURCE
-
-LIB_HEADERS := $(wildcard lib/*.h) $(wildcard lib/*/*.h)
-
-LIB_SRC := lib/deflate_decompress.c lib/utils.c \
-	   $(wildcard lib/*/cpu_features.c)
-
-DECOMPRESSION_ONLY :=
-ifndef DECOMPRESSION_ONLY
-    LIB_SRC += lib/deflate_compress.c
-endif
-
-DISABLE_ZLIB :=
-ifndef DISABLE_ZLIB
-    LIB_SRC += lib/adler32.c lib/zlib_decompress.c
-    ifndef DECOMPRESSION_ONLY
-        LIB_SRC += lib/zlib_compress.c
-    endif
-endif
-
-DISABLE_GZIP :=
-ifndef DISABLE_GZIP
-    LIB_SRC += lib/crc32.c lib/gzip_decompress.c
-    ifndef DECOMPRESSION_ONLY
-        LIB_SRC += lib/gzip_compress.c
-    endif
-endif
-
-STATIC_LIB_OBJ := $(LIB_SRC:.c=.o)
-SHARED_LIB_OBJ := $(LIB_SRC:.c=.shlib.o)
-
-# Compile static library object files
-$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
-	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) $<
-
-# Compile shared library object files
-$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config
-	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \
-		$(SHARED_LIB_CFLAGS) -DLIBDEFLATE_DLL $<
-
-# Create static library
-$(STATIC_LIB):$(STATIC_LIB_OBJ)
-	$(QUIET_AR) $(AR) cr $@ $+
-
-DEFAULT_TARGETS += $(STATIC_LIB)
-
-# Create shared library
-$(SHARED_LIB):$(SHARED_LIB_OBJ)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) \
-		$(SHARED_LIB_LDFLAGS) -shared $+
-
-ifdef DISABLE_SHARED
-undefine SHARED_LIB_SYMLINK
-else
-DEFAULT_TARGETS += $(SHARED_LIB)
-endif
-
-ifdef SHARED_LIB_SYMLINK
-# Create the symlink libdeflate.so => libdeflate.so.$SOVERSION
-$(SHARED_LIB_SYMLINK):$(SHARED_LIB)
-	$(QUIET_LN) ln -sf $+ $@
-DEFAULT_TARGETS += $(SHARED_LIB_SYMLINK)
-endif
-
-##############################################################################
-
-#### Programs
-
-PROG_CFLAGS += $(CFLAGS)		 \
-	       -D_POSIX_C_SOURCE=200809L \
-	       -D_FILE_OFFSET_BITS=64	 \
-	       -DHAVE_CONFIG_H
-
-ALL_PROG_COMMON_HEADERS := programs/config.h \
-			   programs/prog_util.h \
-			   programs/test_util.h
-PROG_COMMON_SRC      := programs/prog_util.c \
-			programs/tgetopt.c
-NONTEST_PROG_SRC     := programs/gzip.c
-TEST_PROG_COMMON_SRC := programs/test_util.c
-TEST_PROG_SRC        := programs/benchmark.c \
-			programs/checksum.c \
-			$(filter-out $(TEST_PROG_COMMON_SRC),$(wildcard programs/test_*.c))
-
-NONTEST_PROGRAMS := $(NONTEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
-DEFAULT_TARGETS  += $(NONTEST_PROGRAMS)
-TEST_PROGRAMS    := $(TEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX))
-
-PROG_COMMON_OBJ      := $(PROG_COMMON_SRC:%.c=%.o)
-NONTEST_PROG_OBJ     := $(NONTEST_PROG_SRC:%.c=%.o)
-TEST_PROG_COMMON_OBJ := $(TEST_PROG_COMMON_SRC:%.c=%.o)
-TEST_PROG_OBJ        := $(TEST_PROG_SRC:%.c=%.o)
-
-ALL_PROG_OBJ	     := $(PROG_COMMON_OBJ) $(NONTEST_PROG_OBJ) \
-			$(TEST_PROG_COMMON_OBJ) $(TEST_PROG_OBJ)
-
-# Generate autodetected configuration header
-programs/config.h:scripts/detect.sh .build-config
-	$(QUIET_GEN) CC="$(CC)" CFLAGS="$(PROG_CFLAGS)" $< > $@
-
-# Compile program object files
-$(ALL_PROG_OBJ): %.o: %.c $(ALL_PROG_COMMON_HEADERS) $(COMMON_HEADERS) \
-			.build-config
-	$(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(PROG_CFLAGS) $<
-
-# Link the programs.
-#
-# Note: the test programs are not compiled by default.  One reason is that the
-# test programs must be linked with zlib for doing comparisons.
-
-ifdef USE_SHARED_LIB
-LIB := $(SHARED_LIB)
-else
-LIB := $(STATIC_LIB)
-endif
-
-$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(LIB)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+
-
-$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) \
-			$(TEST_PROG_COMMON_OBJ) $(LIB)
-	$(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz
-
-ifdef HARD_LINKS
-# Hard link gunzip to gzip
-gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
-	$(QUIET_LN) ln -f $< $@
-else
-# No hard links; copy gzip to gunzip
-gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX)
-	$(QUIET_CP) cp -f $< $@
-endif
-
-DEFAULT_TARGETS += gunzip$(PROG_SUFFIX)
-
-##############################################################################
-
-all:$(DEFAULT_TARGETS)
-
-# We don't generate the pkg-config file until install time, as needed
-# definitions might not exist until then.
-PKGCONFBASE := libdeflate.pc.in
-
-# Install the files.  Note: not all versions of the 'install' program have the
-# '-D' and '-t' options, so don't use them; use portable commands only.
-install:all $(PKGCONFBASE)
-	install -d $(DESTDIR)$(LIBDIR)/pkgconfig $(DESTDIR)$(INCDIR)	\
-		$(DESTDIR)$(BINDIR)
-	install -m644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR)
-	if [ -z "$(DISABLE_SHARED)" ]; then				\
-		install -m755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR);	\
-	fi
-	sed -e "s|@PREFIX@|$(PREFIX)|"					\
-	    -e "s|@LIBDIR@|$(LIBDIR)|"					\
-	    -e "s|@INCDIR@|$(INCDIR)|"					\
-	    -e "s|@VERSION@|$(VERSION)|"				\
-	    $(PKGCONFBASE) > $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
-	chmod 644 $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc
-	install -m644 libdeflate.h $(DESTDIR)$(INCDIR)
-	install -m755 gzip$(PROG_SUFFIX) \
-		$(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)
-	if [ -n "$(HARD_LINKS)" ]; then					\
-		ln -f $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)	\
-		      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
-	else								\
-		ln -sf libdeflate-gzip$(PROG_SUFFIX)			\
-		       $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \
-	fi
-	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
-		ln -sf $(SHARED_LIB)					\
-		       $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
-	fi
-
-uninstall:
-	rm -f $(DESTDIR)$(LIBDIR)/$(STATIC_LIB)				\
-	      $(DESTDIR)$(LIBDIR)/$(SHARED_LIB)				\
-	      $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc		\
-	      $(DESTDIR)$(INCDIR)/libdeflate.h				\
-	      $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX)		\
-	      $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX)
-	if [ -n "$(SHARED_LIB_SYMLINK)" ]; then				\
-		rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK);	\
-	fi
-
-test_programs:$(TEST_PROGRAMS)
-
-# A minimal 'make check' target.  This only runs some quick tests;
-# use scripts/run_tests.sh if you want to run the full tests.
-check:test_programs
-	LD_LIBRARY_PATH=. ./benchmark$(PROG_SUFFIX) < ./benchmark$(PROG_SUFFIX)
-	for prog in test_*; do		\
-		LD_LIBRARY_PATH=. ./$$prog || exit 1;	\
-	done
-
-# Run the clang static analyzer.
-scan-build:
-	scan-build --status-bugs make all test_programs
-
-# Run shellcheck on all shell scripts.
-shellcheck:
-	shellcheck scripts/*.sh
-
-help:
-	@echo "Available targets:"
-	@echo "------------------"
-	@for target in $(DEFAULT_TARGETS) $(TEST_PROGRAMS); do \
-		echo -e "$$target";		\
-	done
-
-clean:
-	rm -f *.a *.dll *.exe *.exp *.dylib *.so \
-		lib/*.o lib/*/*.o \
-		lib/*.obj lib/*/*.obj \
-		lib/*.dllobj lib/*/*.dllobj \
-		programs/*.o programs/*.obj \
-		$(DEFAULT_TARGETS) $(TEST_PROGRAMS) programs/config.h \
-		libdeflate.lib libdeflate.def libdeflatestatic.lib \
-		.build-config
-
-realclean: clean
-	rm -f tags cscope*
-
-FORCE:
-
-.PHONY: all install uninstall test_programs check scan-build shellcheck help \
-	clean realclean
-
-.DEFAULT_GOAL = all

Makefile.msc

@@ -1,65 +0,0 @@
-#
-# Makefile for the Microsoft toolchain
-#
-# Usage:
-#	nmake /f Makefile.msc
-#
-
-.SUFFIXES: .c .obj .dllobj
-
-CC = cl
-LD = link
-AR = lib
-CFLAGS = /MD /O2 -I.
-LDFLAGS =
-
-STATIC_LIB = libdeflatestatic.lib
-SHARED_LIB = libdeflate.dll
-IMPORT_LIB = libdeflate.lib
-
-STATIC_LIB_OBJ =			\
-	  lib/adler32.obj		\
-	  lib/crc32.obj			\
-	  lib/deflate_compress.obj	\
-	  lib/deflate_decompress.obj	\
-	  lib/gzip_compress.obj		\
-	  lib/gzip_decompress.obj	\
-	  lib/utils.obj			\
-	  lib/x86/cpu_features.obj	\
-	  lib/zlib_compress.obj		\
-	  lib/zlib_decompress.obj
-
-SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj)
-
-PROG_COMMON_OBJ = programs/prog_util.obj \
-		  programs/tgetopt.obj \
-		  $(STATIC_LIB)
-
-PROG_CFLAGS = $(CFLAGS) -Iprograms
-
-all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe
-
-.c.obj:
-	$(CC) -c /Fo$@ $(CFLAGS) $**
-
-.c.dllobj:
-	$(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_DLL $**
-
-$(STATIC_LIB): $(STATIC_LIB_OBJ)
-	$(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ)
-
-$(SHARED_LIB): $(SHARED_LIB_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ)
-
-$(IMPORT_LIB): $(SHARED_LIB)
-
-gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ $**
-
-gunzip.exe:gzip.exe
-	copy $** $@
-
-clean:
-	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
-		lib\*.obj lib\x86\*.obj lib\*.dllobj lib\x86\*.dllobj \
-		programs\*.obj 2>nul

NEWS.md

@@ -1,5 +1,212 @@
 # libdeflate release notes
 
+## Version 1.23
+
+* Fixed bug introduced in 1.20 where incorrect checksums could be calculated if
+  libdeflate was compiled with clang at -O0 and run on a CPU supporting AVX512.
+
+* Fixed bug introduced in 1.20 where incorrect checksums could be calculated in
+  rare cases on macOS computers that support AVX512 and are running an older
+  version of macOS that contains a bug that corrupts AVX512 registers.  This
+  could occur only if code outside libdeflate enabled AVX512 in the thread.
+
+* Fixed build error when using -mno-evex512 with clang 18+ or gcc 14+.
+
+* Increased the minimum CMake version to 3.10.
+
+* Further optimized the x86 CRC code.
+
+## Version 1.22
+
+* The CMake-based build system now implements a workaround for gcc being paired
+  with a too-old binutils version.  This can prevent build errors.
+
+## Version 1.21
+
+* Fixed build error on x86 with gcc 8.1 and gcc 8.2.
+
+* Fixed build error on x86 when gcc 11 is paired with a binutils version that
+  doesn't support AVX-VNNI, e.g. as it is on RHEL 9.
+
+* Fixed build error on arm64 with gcc 6.
+
+* Fixed build error on arm64 with gcc 13.1 and later with some -mcpu options.
+
+* Enabled detection of dotprod support in Windows ARM64 builds.
+
+## Version 1.20
+
+* Improved CRC-32 performance on recent x86 CPUs by adding
+  VPCLMULQDQ-accelerated implementations using 256-bit and 512-bit vectors.
+
+* Improved Adler-32 performance on recent x86 CPUs by adding
+  VNNI-accelerated implementations using 256-bit and 512-bit vectors.
+
+* Improved CRC-32 and Adler-32 performance on short inputs.
+
+* Optimized the portable implementation of Adler-32.
+
+* Added some basic optimizations for RISC-V.
+
+* Dropped support for gcc versions older than v4.9 (released in 2014)
+  and clang versions older than v3.9 (released in 2016).
+
+* Dropped support for CRC-32 acceleration on 32-bit ARM using the ARMv8 pmull or
+  crc32 instructions.  This code only worked on CPUs that also have a 64-bit
+  mode, and it was already disabled on many compiler versions due to compiler
+  limitations.  CRC-32 acceleration remains fully supported on 64-bit ARM.
+
+## Version 1.19
+
+* Added new functions `libdeflate_alloc_compressor_ex()` and
+  `libdeflate_alloc_decompressor_ex()`.  These functions allow specifying a
+  custom memory allocator on a per-compressor basis.
+
+* libdeflate now always generates Huffman codes with at least 2 codewords.  This
+  fixes a compatibility issue where Windows Explorer's ZIP unpacker could not
+  decompress DEFLATE streams created by libdeflate.  libdeflate's behavior was
+  allowed by the DEFLATE RFC, but not all software was okay with it.  In rare
+  cases, compression ratios can be slightly reduced by this change.
+
+* Disabled the use of some compiler intrinsics on MSVC versions where they don't
+  work correctly.
+
+* libdeflate can now compress up to the exact size of the output buffer.
+
+* Slightly improved compression performance at levels 1-9.
+
+* Improved the compression ratio of very short inputs.
+
+## Version 1.18
+
+* Fixed a bug where the build type didn't default to "Release" when using
+  CMake 3.10 or earlier.
+
+* Fixed a bug where some optimized code wasn't used when building with
+  Clang 15 or later (x86), or with Clang 16 or later (aarch64).
+
+* Fixed build errors with some architecture and compiler combos:
+  * aarch64 with Clang 16
+  * armv6kz or armv7e-m with gcc
+  * armhf with gcc (on Debian only)
+
+## Version 1.17
+
+(Apologies for another release so soon after v1.16, but the bug fix listed below
+needed to go out.)
+
+* Fixed a bug introduced in v1.16 where compression at levels 10-12 would
+  sometimes produce an output larger than the size that was returned by the
+  corresponding `libdeflate_*_compress_bound()` function.
+
+* Converted the fuzzing scripts to use LLVM's libFuzzer and added them to the
+  GitHub Actions workflow.  (This would have detected the above bug.)
+
+* Further improved the support for direct compilation without using the official
+  build system.  The top-level source directory no longer needs to be added to
+  the include path, and building the programs no longer requires that
+  `_FILE_OFFSET_BITS` and `_POSIX_C_SOURCE` be defined on the command line.
+
+## Version 1.16
+
+* Improved the compression ratio at levels 10-12 slightly, mainly levels 11-12.
+  Some inputs (such as certain PNG files) see much improved compression ratios.
+  As a trade-off, compressing at levels 11-12 is now about 5-20% slower.
+
+* For consistency with zlib, the decompressor now returns an error on some
+  invalid inputs that were accepted before.
+
+* Fixed a build error on arm64 with gcc with certain target CPUs.  (Fixes v1.12)
+
+* Fixed a build error on arm32 with gcc 10.1-10.3 and 11.1-11.2.  (Fixes v1.15)
+
+* Fixed a build error on arm32 with gcc in soft float mode.  (Fixes v1.15)
+
+* Fixed a build error in programs/gzip.c with uClibc.  (Fixes v1.15)
+
+* Fixed the install target on Windows.  (Fixes v1.15)
+
+## Version 1.15
+
+* libdeflate now uses CMake instead of a plain Makefile.
+
+* Improved MSVC support.  Enabled most architecture-specific code with MSVC,
+  fixed building with clang in MSVC compatibility mode, and other improvements.
+
+* When libdeflate is built with MinGW, the static library and import library are
+  now named using the MinGW convention (`*.a` and `*.dll.a`) instead of the
+  Visual Studio convention.  This affects the official Windows binaries.
+
+## Version 1.14
+
+Significantly improved decompression performance on all platforms.  Examples
+include (measuring DEFLATE only):
+
+| Platform                           | Speedup over v1.13 |
+|------------------------------------|--------------------|
+| x86_64 (Intel Comet Lake), gcc     | 1.287x             |
+| x86_64 (Intel Comet Lake), clang   | 1.437x             |
+| x86_64 (Intel Ice Lake), gcc       | 1.332x             |
+| x86_64 (Intel Ice Lake), clang     | 1.296x             |
+| x86_64 (Intel Sandy Bridge), gcc   | 1.162x             |
+| x86_64 (Intel Sandy Bridge), clang | 1.092x             |
+| x86_64 (AMD Zen 2), gcc            | 1.263x             |
+| x86_64 (AMD Zen 2), clang          | 1.259x             |
+| i386 (Intel Comet Lake), gcc       | 1.570x             |
+| i386 (Intel Comet Lake), clang     | 1.344x             |
+| arm64 (Apple M1), clang            | 1.306x             |
+| arm64 (Cortex-A76), clang          | 1.355x             |
+| arm64 (Cortex-A55), clang          | 1.190x             |
+| arm32 (Cortex-A76), clang          | 1.665x             |
+| arm32 (Cortex-A55), clang          | 1.283x             |
+
+Thanks to Dougall Johnson (https://dougallj.wordpress.com/) for ideas for many
+of the improvements.
+
+## Version 1.13
+
+* Changed the 32-bit Windows build of the library to use the default calling
+  convention (cdecl) instead of stdcall, reverting a change from libdeflate 1.4.
+
+* Fixed a couple macOS compatibility issues with the gzip program.
+
+## Version 1.12
+
+This release focuses on improving the performance of the CRC-32 and Adler-32
+checksum algorithms on x86 and ARM (both 32-bit and 64-bit).
+
+* Build updates:
+
+  * Fixed building libdeflate on Apple platforms.
+
+  * For Visual Studio builds, Visual Studio 2015 or later is now required.
+
+* CRC-32 algorithm updates:
+
+  * Improved CRC-32 performance on short inputs on x86 and ARM.
+
+  * Improved CRC-32 performance on Apple Silicon Macs by using a 12-way pmull
+    implementation.   Performance on large inputs on M1 is now about 67 GB/s,
+    compared to 8 GB/s before, or 31 GB/s with the Apple-provided zlib.
+
+  * Improved CRC-32 performance on some other ARM CPUs by reworking the code so
+    that multiple crc32 instructions can be issued in parallel.
+
+  * Improved CRC-32 performance on some x86 CPUs by increasing the stride length
+    of the pclmul implementation.
+
+* Adler-32 algorithm updates:
+
+  * Improved Adler-32 performance on some x86 CPUs by optimizing the AVX-2
+    implementation.  E.g., performance on Zen 1 improved from 19 to 30 GB/s, and
+    on Ice Lake from 35 to 41 GB/s (if the AVX-512 implementation is excluded).
+
+  * Removed the AVX-512 implementation of Adler-32 to avoid CPU frequency
+    downclocking, and because the AVX-2 implementation was made faster.
+
+  * Improved Adler-32 performance on some ARM CPUs by optimizing the NEON
+    implementation.  E.g., Apple M1 improved from about 36 to 52 GB/s.
+
 ## Version 1.11
 
 * Library updates:

README.md

@@ -10,28 +10,34 @@ The supported formats are:
 - gzip (a.k.a. DEFLATE with a gzip wrapper)
 
 libdeflate is heavily optimized.  It is significantly faster than the zlib
-library, both for compression and decompression, and especially on x86
+library, both for compression and decompression, and especially on x86 and ARM
 processors.  In addition, libdeflate provides optional high compression modes
 that provide a better compression ratio than the zlib's "level 9".
 
-libdeflate itself is a library, but the following command-line programs which
-use this library are also provided:
+libdeflate itself is a library.  The following command-line programs which use
+this library are also included:
 
-* gzip (or gunzip), a program which mostly behaves like the standard equivalent,
-  except that it does not yet have good streaming support and therefore does not
-  yet support very large files
-* benchmark, a program for benchmarking in-memory compression and decompression
+* `libdeflate-gzip`, a program which can be a drop-in replacement for standard
+  `gzip` under some circumstances.  Note that `libdeflate-gzip` has some
+  limitations; it is provided for convenience and is **not** meant to be the
+  main use case of libdeflate.  It needs a lot of memory to process large files,
+  and it omits support for some infrequently-used options of GNU gzip.
+
+* `benchmark`, a test program that does round-trip compression and decompression
+  of the provided data, and measures the compression and decompression speed.
+  It can use libdeflate, zlib, or a combination of the two.
+
+* `checksum`, a test program that checksums the provided data with Adler-32 or
+  CRC-32, and optionally measures the speed.  It can use libdeflate or zlib.
 
 For the release notes, see the [NEWS file](NEWS.md).
 
 ## Table of Contents
 
 - [Building](#building)
-  - [For UNIX](#for-unix)
-  - [For macOS](#for-macos)
-  - [For Windows](#for-windows)
-    - [Using Cygwin](#using-cygwin)
-    - [Using MSYS2](#using-msys2)
+  - [Using CMake](#using-cmake)
+  - [Directly integrating the library sources](#directly-integrating-the-library-sources)
+  - [Supported compilers](#supported-compilers)
 - [API](#api)
 - [Bindings for other programming languages](#bindings-for-other-programming-languages)
 - [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip)
@@ -39,127 +45,73 @@ For the release notes, see the [NEWS file](NEWS.md).
 - [Motivation](#motivation)
 - [License](#license)
 
-
 # Building
 
-## For UNIX
+## Using CMake
 
-Just run `make`, then (if desired) `make install`.  You need GNU Make and either
-GCC or Clang.  GCC is recommended because it builds slightly faster binaries.
+libdeflate uses [CMake](https://cmake.org/).  It can be built just like any
+other CMake project, e.g. with:
 
-By default, the following targets are built: the static library `libdeflate.a`,
-the shared library `libdeflate.so`, the `gzip` program, and the `gunzip` program
-(which is actually just a hard link to `gzip`).  Benchmarking and test programs
-such as `benchmark` are not built by default.  You can run `make help` to
-display the available build targets.
+    cmake -B build && cmake --build build
 
-There are also many options which can be set on the `make` command line, e.g. to
-omit library features or to customize the directories into which `make install`
-installs files.  See the Makefile for details.
+By default the following targets are built:
 
-## For macOS
+- The static library (normally called `libdeflate.a`)
+- The shared library (normally called `libdeflate.so`)
+- The `libdeflate-gzip` program, including its alias `libdeflate-gunzip`
 
-Prebuilt macOS binaries can be installed with [Homebrew](https://brew.sh):
-
-    brew install libdeflate
-
-But if you need to build the binaries yourself, see the section for UNIX above.
-
-## For Windows
+Besides the standard CMake build and installation options, there are some
+libdeflate-specific build options.  See `CMakeLists.txt` for the list of these
+options.  To set an option, add `-DOPTION=VALUE` to the `cmake` command.
 
 Prebuilt Windows binaries can be downloaded from
-https://github.com/ebiggers/libdeflate/releases.  But if you need to build the
-binaries yourself, MinGW (gcc) is the recommended compiler to use.  If you're
-performing the build *on* Windows (as opposed to cross-compiling for Windows on
-Linux, for example), you'll need to follow the directions in **one** of the two
-sections below to set up a minimal UNIX-compatible environment using either
-Cygwin or MSYS2, then do the build.  (Other MinGW distributions may not work, as
-they often omit basic UNIX tools such as `sh`.)
+https://github.com/ebiggers/libdeflate/releases.
 
-Alternatively, libdeflate may be built using the Visual Studio toolchain by
-running `nmake /f Makefile.msc`.  However, while this is supported in the sense
-that it will produce working binaries, it is not recommended because the
-binaries built with MinGW will be significantly faster.
+## Directly integrating the library sources
 
-Also note that 64-bit binaries are faster than 32-bit binaries and should be
-preferred whenever possible.
+Although the official build system is CMake, care has been taken to keep the
+library source files compilable directly, without a prerequisite configuration
+step.  Therefore, it is also fine to just add the library source files directly
+to your application, without using CMake.
 
-### Using Cygwin
+You should compile both `lib/*.c` and `lib/*/*.c`.  You don't need to worry
+about excluding irrelevant architecture-specific code, as this is already
+handled in the source files themselves using `#ifdef`s.
 
-Run the Cygwin installer, available from https://cygwin.com/setup-x86_64.exe.
-When you get to the package selection screen, choose the following additional
-packages from category "Devel":
+If you are doing a freestanding build with `-ffreestanding`, you must add
+`-DFREESTANDING` as well (matching what the `CMakeLists.txt` does).
 
-- git
-- make
-- mingw64-i686-binutils
-- mingw64-i686-gcc-g++
-- mingw64-x86_64-binutils
-- mingw64-x86_64-gcc-g++
+## Supported compilers
 
-(You may skip the mingw64-i686 packages if you don't need to build 32-bit
-binaries.)
+- gcc: v4.9 and later
+- clang: v3.9 and later (upstream), Xcode 8 and later (Apple)
+- MSVC: Visual Studio 2015 and later
+- Other compilers: any other C99-compatible compiler should work, though if your
+  compiler pretends to be gcc, clang, or MSVC, it needs to be sufficiently
+  compatible with the compiler it pretends to be.
 
-After the installation finishes, open a Cygwin terminal.  Then download
-libdeflate's source code (if you haven't already) and `cd` into its directory:
+The above are the minimums, but using a newer compiler allows more of the
+architecture-optimized code to be built.  libdeflate is most heavily optimized
+for gcc and clang, but MSVC is supported fairly well now too.
 
-    git clone https://github.com/ebiggers/libdeflate
-    cd libdeflate
+The recommended optimization flag is `-O2`, and the `CMakeLists.txt` sets this
+for release builds.  `-O3` is fine too, but often `-O2` actually gives better
+results.  It's unnecessary to add flags such as `-mavx2` or `/arch:AVX2`, though
+you can do so if you want to.  Most of the relevant optimized functions are
+built regardless of such flags, and appropriate ones are selected at runtime.
+For the same reason, flags like `-mno-avx2` do *not* cause all code using the
+corresponding instruction set extension to be omitted from the binary; this is
+working as intended due to the use of runtime CPU feature detection.
 
-(Note that it's not required to use `git`; an alternative is to extract a .zip
-or .tar.gz archive of the source code downloaded from the releases page.
-Also, in case you need to find it in the file browser, note that your home
-directory in Cygwin is usually located at `C:\cygwin64\home\<your username>`.)
-
-Then, to build 64-bit binaries:
-
-    make CC=x86_64-w64-mingw32-gcc
-
-or to build 32-bit binaries:
-
-    make CC=i686-w64-mingw32-gcc
-
-### Using MSYS2
-
-Run the MSYS2 installer, available from http://www.msys2.org/.  After
-installing, open an MSYS2 shell and run:
-
-    pacman -Syu
-
-Say `y`, then when it's finished, close the shell window and open a new one.
-Then run the same command again:
-
-    pacman -Syu
-
-Then, install the packages needed to build libdeflate:
-
-    pacman -S git \
-              make \
-              mingw-w64-i686-binutils \
-              mingw-w64-i686-gcc \
-              mingw-w64-x86_64-binutils \
-              mingw-w64-x86_64-gcc
-
-(You may skip the mingw-w64-i686 packages if you don't need to build 32-bit
-binaries.)
-
-Then download libdeflate's source code (if you haven't already):
-
-    git clone https://github.com/ebiggers/libdeflate
-
-(Note that it's not required to use `git`; an alternative is to extract a .zip
-or .tar.gz archive of the source code downloaded from the releases page.
-Also, in case you need to find it in the file browser, note that your home
-directory in MSYS2 is usually located at `C:\msys64\home\<your username>`.)
-
-Then, to build 64-bit binaries, open "MSYS2 MinGW 64-bit" from the Start menu
-and run the following commands:
-
-    cd libdeflate
-    make clean
-    make
-
-Or to build 32-bit binaries, do the same but use "MSYS2 MinGW 32-bit" instead.
+If using gcc, your gcc should always be paired with a binutils version that is
+not much older than itself, to avoid problems where the compiler generates
+instructions the assembler cannot assemble.  Usually systems have their gcc and
+binutils paired properly, but rarely a mismatch can arise in cases such as the
+user installing a newer gcc version without a proper binutils alongside it.
+Since libdeflate v1.22, the CMake-based build system will detect incompatible
+binutils versions and disable some optimized code accordingly.  In older
+versions of libdeflate, or if CMake is not being used, a too-old binutils can
+cause build errors like "no such instruction" from the assembler.
 
 # API
 
@@ -183,10 +135,7 @@ guessing.  However, libdeflate's decompression routines do optionally provide
 the actual number of output bytes in case you need it.
 
 Windows developers: note that the calling convention of libdeflate.dll is
-"stdcall" -- the same as the Win32 API.  If you call into libdeflate.dll using a
-non-C/C++ language, or dynamically using LoadLibrary(), make sure to use the
-stdcall convention.  Using the wrong convention may crash your application.
-(Note: older versions of libdeflate used the "cdecl" convention instead.)
+"cdecl".  (libdeflate v1.4 through v1.12 used "stdcall" instead.)
 
 # Bindings for other programming languages
 
@@ -195,9 +144,13 @@ libdeflate from a programming language other than C or C++, consider using the
 following bindings:
 
 * C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET)
+* Delphi: [libdeflate-pas](https://github.com/zedxxx/libdeflate-pas)
 * Go: [go-libdeflate](https://github.com/4kills/go-libdeflate)
 * Java: [libdeflate-java](https://github.com/astei/libdeflate-java)
 * Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl)
+* Nim: [libdeflate-nim](https://github.com/gemesa/libdeflate-nim)
+* Perl: [Gzip::Libdeflate](https://github.com/benkasminbullock/gzip-libdeflate)
+* PHP: [ext-libdeflate](https://github.com/pmmp/ext-libdeflate)
 * Python: [deflate](https://github.com/dcwatson/deflate)
 * Ruby: [libdeflate-ruby](https://github.com/kaorimatz/libdeflate-ruby)
 * Rust: [libdeflater](https://github.com/adamkewley/libdeflater)
@@ -206,6 +159,11 @@ Note: these are third-party projects which haven't necessarily been vetted by
 the authors of libdeflate.  Please direct all questions, bugs, and improvements
 for these bindings to their authors.
 
+Also, unfortunately many of these bindings bundle or pin an old version of
+libdeflate.  To avoid known issues in old versions and to improve performance,
+before using any of these bindings please ensure that the bundled or pinned
+version of libdeflate has been upgraded to the latest release.
+
 # DEFLATE vs. zlib vs. gzip
 
 The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib

common/common_defs.h

@@ -1,334 +0,0 @@
-/*
- * common_defs.h
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef COMMON_COMMON_DEFS_H
-#define COMMON_COMMON_DEFS_H
-
-#ifdef __GNUC__
-#  include "compiler_gcc.h"
-#elif defined(_MSC_VER)
-#  include "compiler_msc.h"
-#else
-#  pragma message("Unrecognized compiler.  Please add a header file for your compiler.  Compilation will proceed, but performance may suffer!")
-#endif
-
-/* ========================================================================== */
-/*                              Type definitions                              */
-/* ========================================================================== */
-
-#include <stddef.h> /* size_t */
-
-#ifndef __bool_true_false_are_defined
-#  include <stdbool.h> /* bool */
-#endif
-
-/* Fixed-width integer types */
-#include <stdint.h>
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-typedef int8_t s8;
-typedef int16_t s16;
-typedef int32_t s32;
-typedef int64_t s64;
-
-/*
- * Word type of the target architecture.  Use 'size_t' instead of 'unsigned
- * long' to account for platforms such as Windows that use 32-bit 'unsigned
- * long' on 64-bit architectures.
- */
-typedef size_t machine_word_t;
-
-/* Number of bytes in a word */
-#define WORDBYTES	((int)sizeof(machine_word_t))
-
-/* Number of bits in a word */
-#define WORDBITS	(8 * WORDBYTES)
-
-/* ========================================================================== */
-/*                         Optional compiler features                         */
-/* ========================================================================== */
-
-/* LIBEXPORT - export a function from a shared library */
-#ifndef LIBEXPORT
-#  define LIBEXPORT
-#endif
-
-/* inline - suggest that a function be inlined */
-#ifndef inline
-#  define inline
-#endif
-
-/* forceinline - force a function to be inlined, if possible */
-#ifndef forceinline
-#  define forceinline inline
-#endif
-
-/* restrict - annotate a non-aliased pointer */
-#ifndef restrict
-#  define restrict
-#endif
-
-/* likely(expr) - hint that an expression is usually true */
-#ifndef likely
-#  define likely(expr)		(expr)
-#endif
-
-/* unlikely(expr) - hint that an expression is usually false */
-#ifndef unlikely
-#  define unlikely(expr)	(expr)
-#endif
-
-/* prefetchr(addr) - prefetch into L1 cache for read */
-#ifndef prefetchr
-#  define prefetchr(addr)
-#endif
-
-/* prefetchw(addr) - prefetch into L1 cache for write */
-#ifndef prefetchw
-#  define prefetchw(addr)
-#endif
-
-/* Does the compiler support the 'target' function attribute? */
-#ifndef COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-#  define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
-#endif
-
-/* Which targets are supported with the 'target' function attribute? */
-#ifndef COMPILER_SUPPORTS_BMI2_TARGET
-#  define COMPILER_SUPPORTS_BMI2_TARGET 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX_TARGET
-#  define COMPILER_SUPPORTS_AVX_TARGET 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET
-#  define COMPILER_SUPPORTS_AVX512BW_TARGET 0
-#endif
-
-/*
- * Which targets are supported with the 'target' function attribute and have
- * intrinsics that work within 'target'-ed functions?
- */
-#ifndef COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0
-#endif
-#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0
-#endif
-
-/* _aligned_attribute(n) - declare that the annotated variable, or variables of
- * the annotated type, are to be aligned on n-byte boundaries */
-#ifndef _aligned_attribute
-#endif
-
-/* ========================================================================== */
-/*                          Miscellaneous macros                              */
-/* ========================================================================== */
-
-#define ARRAY_LEN(A)		(sizeof(A) / sizeof((A)[0]))
-#define MIN(a, b)		((a) <= (b) ? (a) : (b))
-#define MAX(a, b)		((a) >= (b) ? (a) : (b))
-#define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
-#define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
-#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
-
-/* ========================================================================== */
-/*                           Endianness handling                              */
-/* ========================================================================== */
-
-/*
- * CPU_IS_LITTLE_ENDIAN() - a macro which evaluates to 1 if the CPU is little
- * endian or 0 if it is big endian.  The macro should be defined in a way such
- * that the compiler can evaluate it at compilation time.  If not defined, a
- * fallback is used.
- */
-#ifndef CPU_IS_LITTLE_ENDIAN
-static forceinline int CPU_IS_LITTLE_ENDIAN(void)
-{
-	union {
-		unsigned int v;
-		unsigned char b;
-	} u;
-	u.v = 1;
-	return u.b;
-}
-#endif
-
-/* bswap16(n) - swap the bytes of a 16-bit integer */
-#ifndef bswap16
-static forceinline u16 bswap16(u16 n)
-{
-	return (n << 8) | (n >> 8);
-}
-#endif
-
-/* bswap32(n) - swap the bytes of a 32-bit integer */
-#ifndef bswap32
-static forceinline u32 bswap32(u32 n)
-{
-	return ((n & 0x000000FF) << 24) |
-	       ((n & 0x0000FF00) << 8) |
-	       ((n & 0x00FF0000) >> 8) |
-	       ((n & 0xFF000000) >> 24);
-}
-#endif
-
-/* bswap64(n) - swap the bytes of a 64-bit integer */
-#ifndef bswap64
-static forceinline u64 bswap64(u64 n)
-{
-	return ((n & 0x00000000000000FF) << 56) |
-	       ((n & 0x000000000000FF00) << 40) |
-	       ((n & 0x0000000000FF0000) << 24) |
-	       ((n & 0x00000000FF000000) << 8) |
-	       ((n & 0x000000FF00000000) >> 8) |
-	       ((n & 0x0000FF0000000000) >> 24) |
-	       ((n & 0x00FF000000000000) >> 40) |
-	       ((n & 0xFF00000000000000) >> 56);
-}
-#endif
-
-#define le16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap16(n))
-#define le32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap32(n))
-#define le64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap64(n))
-#define be16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap16(n) : (n))
-#define be32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap32(n) : (n))
-#define be64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap64(n) : (n))
-
-/* ========================================================================== */
-/*                          Unaligned memory accesses                         */
-/* ========================================================================== */
-
-/*
- * UNALIGNED_ACCESS_IS_FAST should be defined to 1 if unaligned memory accesses
- * can be performed efficiently on the target platform.
- */
-#ifndef UNALIGNED_ACCESS_IS_FAST
-#  define UNALIGNED_ACCESS_IS_FAST 0
-#endif
-
-/* ========================================================================== */
-/*                             Bit scan functions                             */
-/* ========================================================================== */
-
-/*
- * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
- * significant end) of the *most* significant 1 bit in the input value.  The
- * input value must be nonzero!
- */
-
-#ifndef bsr32
-static forceinline unsigned
-bsr32(u32 n)
-{
-	unsigned i = 0;
-	while ((n >>= 1) != 0)
-		i++;
-	return i;
-}
-#endif
-
-#ifndef bsr64
-static forceinline unsigned
-bsr64(u64 n)
-{
-	unsigned i = 0;
-	while ((n >>= 1) != 0)
-		i++;
-	return i;
-}
-#endif
-
-static forceinline unsigned
-bsrw(machine_word_t n)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		return bsr32(n);
-	else
-		return bsr64(n);
-}
-
-/*
- * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
- * significant end) of the *least* significant 1 bit in the input value.  The
- * input value must be nonzero!
- */
-
-#ifndef bsf32
-static forceinline unsigned
-bsf32(u32 n)
-{
-	unsigned i = 0;
-	while ((n & 1) == 0) {
-		i++;
-		n >>= 1;
-	}
-	return i;
-}
-#endif
-
-#ifndef bsf64
-static forceinline unsigned
-bsf64(u64 n)
-{
-	unsigned i = 0;
-	while ((n & 1) == 0) {
-		i++;
-		n >>= 1;
-	}
-	return i;
-}
-#endif
-
-static forceinline unsigned
-bsfw(machine_word_t n)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		return bsf32(n);
-	else
-		return bsf64(n);
-}
-
-#endif /* COMMON_COMMON_DEFS_H */

common/compiler_gcc.h

@@ -1,235 +0,0 @@
-/*
- * compiler_gcc.h - definitions for the GNU C Compiler.  This also handles clang
- * and the Intel C Compiler (icc).
- *
- * TODO: icc is not well tested, so some things are currently disabled even
- * though they maybe can be enabled on some icc versions.
- */
-
-#if !defined(__clang__) && !defined(__INTEL_COMPILER)
-#  define GCC_PREREQ(major, minor)		\
-	(__GNUC__ > (major) ||			\
-	 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
-#else
-#  define GCC_PREREQ(major, minor)	0
-#endif
-
-/* Note: only check the clang version when absolutely necessary!
- * "Vendors" such as Apple can use different version numbers. */
-#ifdef __clang__
-#  ifdef __apple_build_version__
-#    define CLANG_PREREQ(major, minor, apple_version)	\
-	(__apple_build_version__ >= (apple_version))
-#  else
-#    define CLANG_PREREQ(major, minor, apple_version)	\
-	(__clang_major__ > (major) ||			\
-	 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
-#  endif
-#else
-#  define CLANG_PREREQ(major, minor, apple_version)	0
-#endif
-
-#ifndef __has_attribute
-#  define __has_attribute(attribute)	0
-#endif
-#ifndef __has_feature
-#  define __has_feature(feature)	0
-#endif
-#ifndef __has_builtin
-#  define __has_builtin(builtin)	0
-#endif
-
-#ifdef _WIN32
-#  define LIBEXPORT __declspec(dllexport)
-#else
-#  define LIBEXPORT __attribute__((visibility("default")))
-#endif
-
-#define inline			inline
-#define forceinline		inline __attribute__((always_inline))
-#define restrict		__restrict__
-#define likely(expr)		__builtin_expect(!!(expr), 1)
-#define unlikely(expr)		__builtin_expect(!!(expr), 0)
-#define prefetchr(addr)		__builtin_prefetch((addr), 0)
-#define prefetchw(addr)		__builtin_prefetch((addr), 1)
-#define _aligned_attribute(n)	__attribute__((aligned(n)))
-
-#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE	\
-	(GCC_PREREQ(4, 4) || __has_attribute(target))
-
-#if COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-
-#  if defined(__i386__) || defined(__x86_64__)
-
-#    define COMPILER_SUPPORTS_PCLMUL_TARGET	\
-	(GCC_PREREQ(4, 4) || __has_builtin(__builtin_ia32_pclmulqdq128))
-
-#    define COMPILER_SUPPORTS_AVX_TARGET	\
-	(GCC_PREREQ(4, 6) || __has_builtin(__builtin_ia32_maxps256))
-
-#    define COMPILER_SUPPORTS_BMI2_TARGET	\
-	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di))
-
-#    define COMPILER_SUPPORTS_AVX2_TARGET	\
-	(GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256))
-
-#    define COMPILER_SUPPORTS_AVX512BW_TARGET	\
-	(GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512))
-
-	/*
-	 * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics
-	 * not available in the main target could not be used in 'target'
-	 * attribute functions.  Unfortunately clang has no feature test macro
-	 * for this so we have to check its version.
-	 */
-#    if GCC_PREREQ(4, 9) || CLANG_PREREQ(3, 8, 7030000)
-#      define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS	1
-#      define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS	\
-		COMPILER_SUPPORTS_PCLMUL_TARGET
-#      define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS	\
-		COMPILER_SUPPORTS_AVX2_TARGET
-#      define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS	\
-		COMPILER_SUPPORTS_AVX512BW_TARGET
-#    endif
-
-#  elif defined(__arm__) || defined(__aarch64__)
-
-    /*
-     * Determine whether NEON and crypto intrinsics are supported.
-     *
-     * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither
-     * was available unless enabled in the main target.
-     *
-     * But even after that, to include <arm_neon.h> (which contains both the
-     * basic NEON intrinsics and the crypto intrinsics) the main target still
-     * needs to have:
-     *   - gcc: hardware floating point support
-     *   - clang: NEON support (but not necessarily crypto support)
-     */
-#    if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \
-        (defined(__clang__) && defined(__ARM_NEON))
-#      define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1
-       /*
-        * The crypto intrinsics are broken on arm32 with clang, even when using
-        * -mfpu=crypto-neon-fp-armv8, because clang's <arm_neon.h> puts them
-        * behind __aarch64__.  Undefine __ARM_FEATURE_CRYPTO in that case...
-        */
-#      if defined(__clang__) && defined(__arm__)
-#        undef __ARM_FEATURE_CRYPTO
-#      elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__)
-#        define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1
-#      endif
-#    endif
-
-     /*
-      * Determine whether ARM CRC32 intrinsics are supported.
-      *
-      * This support has been affected by several gcc bugs, which we must avoid
-      * by only allowing gcc versions that have the corresponding fixes.  First,
-      * gcc commit 943766d37ae4 ("[arm] Fix use of CRC32 intrinsics with Armv8-a
-      * and hard-float"), i.e. gcc 8.4+, 9.3+, 10.1+, or 11+, is needed.
-      * Second, gcc commit c1cdabe3aab8 ("arm: reorder assembler architecture
-      * directives [PR101723]"), i.e. gcc 9.5+, 10.4+, 11.3+, or 12+, is needed
-      * when binutils is 2.34 or later, due to
-      * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104439.  We use the second
-      * set of prerequisites, as they are stricter and we have no way to detect
-      * the binutils version in C source without requiring a configure script.
-      *
-      * Yet another gcc bug makes arm_acle.h sometimes not define the crc
-      * functions even when the corresponding builtins are available.  However,
-      * we work around this later when including arm_acle.h.
-      *
-      * Things are a bit easier with clang -- we can just check whether the
-      * crc builtins are available.  However, clang's arm_acle.h is broken in
-      * the same way as gcc's, which we work around later in the same way.
-      */
-#    if GCC_PREREQ(11, 3) || \
-        (GCC_PREREQ(10, 4) && !GCC_PREREQ(11, 0)) || \
-        (GCC_PREREQ(9, 5) && !GCC_PREREQ(10, 0)) || \
-        (defined(__clang__) && __has_builtin(__builtin_arm_crc32b))
-#      define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1
-#    endif
-
-#  endif /* __arm__ || __aarch64__ */
-
-#endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */
-
-/*
- * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed
- * integers (e.g. __v4si), not vectors of unsigned integers (e.g.  __v4su).  But
- * we need the unsigned ones in order to avoid signed integer overflow, which is
- * undefined behavior.  Add the missing definitions for the unsigned ones if
- * needed.
- */
-#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \
-    (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) || \
-    defined(__INTEL_COMPILER)
-typedef unsigned long long  __v2du __attribute__((__vector_size__(16)));
-typedef unsigned int        __v4su __attribute__((__vector_size__(16)));
-typedef unsigned short      __v8hu __attribute__((__vector_size__(16)));
-typedef unsigned char      __v16qu __attribute__((__vector_size__(16)));
-typedef unsigned long long  __v4du __attribute__((__vector_size__(32)));
-typedef unsigned int        __v8su __attribute__((__vector_size__(32)));
-typedef unsigned short     __v16hu __attribute__((__vector_size__(32)));
-typedef unsigned char      __v32qu __attribute__((__vector_size__(32)));
-#endif
-
-#ifdef __INTEL_COMPILER
-typedef int   __v16si __attribute__((__vector_size__(64)));
-typedef short __v32hi __attribute__((__vector_size__(64)));
-typedef char  __v64qi __attribute__((__vector_size__(64)));
-#endif
-
-/* Newer gcc supports __BYTE_ORDER__.  Older gcc doesn't. */
-#ifdef __BYTE_ORDER__
-#  define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#endif
-
-#if GCC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)
-#  define bswap16	__builtin_bswap16
-#endif
-
-#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap32)
-#  define bswap32	__builtin_bswap32
-#endif
-
-#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap64)
-#  define bswap64	__builtin_bswap64
-#endif
-
-#if defined(__x86_64__) || defined(__i386__) || \
-    defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
-    /*
-     * For all compilation purposes, WebAssembly behaves like any other CPU
-     * instruction set. Even though WebAssembly engine might be running on top
-     * of different actual CPU architectures, the WebAssembly spec itself
-     * permits unaligned access and it will be fast on most of those platforms,
-     * and simulated at the engine level on others, so it's worth treating it
-     * as a CPU architecture with fast unaligned access.
-    */ defined(__wasm__)
-#  define UNALIGNED_ACCESS_IS_FAST 1
-#endif
-
-#define bsr32(n)	(31 - __builtin_clz(n))
-#define bsr64(n)	(63 - __builtin_clzll(n))
-#define bsf32(n)	__builtin_ctz(n)
-#define bsf64(n)	__builtin_ctzll(n)
-
-#if defined(__arm__) && \
-	(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
-static forceinline unsigned int
-rbit32(unsigned int v)
-{
-	__asm__("rbit %0, %1\n" : "=r" (v) : "r" (v));
-	return v;
-}
-#define rbit32 rbit32
-#elif defined(__aarch64__)
-static forceinline unsigned int
-rbit32(unsigned int v)
-{
-	__asm__("rbit %w0, %w1\n" : "=r" (v) : "r" (v));
-	return v;
-}
-#define rbit32 rbit32
-#endif /* __aarch64__ */

common/compiler_msc.h

@@ -1,80 +0,0 @@
-/*
- * compiler_msc.h - definitions for the Microsoft C Compiler
- */
-
-#include <stdint.h>
-#include <stdlib.h> /* for _byteswap_*() */
-
-#define LIBEXPORT	__declspec(dllexport)
-
-/*
- * Old versions (e.g. VS2010) of MSC don't have the C99 header stdbool.h.
- * Beware: the below replacement isn't fully standard, since normally any value
- * != 0 should be implicitly cast to a bool with value 1... but that doesn't
- * happen if bool is really just an 'int'.
- */
-typedef int bool;
-#define true 1
-#define false 0
-#define __bool_true_false_are_defined 1
-
-/* Define ssize_t */
-#ifdef _WIN64
-typedef long long ssize_t;
-#else
-typedef int ssize_t;
-#endif
-
-/* Assume a little endian architecture with fast unaligned access */
-#define CPU_IS_LITTLE_ENDIAN()		1
-#define UNALIGNED_ACCESS_IS_FAST	1
-
-/* __restrict has nonstandard behavior; don't use it */
-#define restrict
-
-/* ... but we can use __inline and __forceinline */
-#define inline		__inline
-#define forceinline	__forceinline
-
-/* Byte swap functions */
-#define bswap16	_byteswap_ushort
-#define bswap32	_byteswap_ulong
-#define bswap64	_byteswap_uint64
-
-/* Bit scan functions (32-bit) */
-
-static forceinline unsigned
-bsr32(uint32_t n)
-{
-	_BitScanReverse(&n, n);
-	return n;
-}
-#define bsr32 bsr32
-
-static forceinline unsigned
-bsf32(uint32_t n)
-{
-	_BitScanForward(&n, n);
-	return n;
-}
-#define bsf32 bsf32
-
-#ifdef _M_X64 /* Bit scan functions (64-bit) */
-
-static forceinline unsigned
-bsr64(uint64_t n)
-{
-	_BitScanReverse64(&n, n);
-	return n;
-}
-#define bsr64 bsr64
-
-static forceinline unsigned
-bsf64(uint64_t n)
-{
-	_BitScanForward64(&n, n);
-	return n;
-}
-#define bsf64 bsf64
-
-#endif /* _M_X64 */

common_defs.h

@@ -0,0 +1,748 @@
+/*
+ * common_defs.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COMMON_DEFS_H
+#define COMMON_DEFS_H
+
+#include "libdeflate.h"
+
+#include <stdbool.h>
+#include <stddef.h>	/* for size_t */
+#include <stdint.h>
+#ifdef _MSC_VER
+#  include <intrin.h>	/* for _BitScan*() and other intrinsics */
+#  include <stdlib.h>	/* for _byteswap_*() */
+   /* Disable MSVC warnings that are expected. */
+   /* /W2 */
+#  pragma warning(disable : 4146) /* unary minus on unsigned type */
+   /* /W3 */
+#  pragma warning(disable : 4018) /* signed/unsigned mismatch */
+#  pragma warning(disable : 4244) /* possible loss of data */
+#  pragma warning(disable : 4267) /* possible loss of precision */
+#  pragma warning(disable : 4310) /* cast truncates constant value */
+   /* /W4 */
+#  pragma warning(disable : 4100) /* unreferenced formal parameter */
+#  pragma warning(disable : 4127) /* conditional expression is constant */
+#  pragma warning(disable : 4189) /* local variable initialized but not referenced */
+#  pragma warning(disable : 4232) /* nonstandard extension used */
+#  pragma warning(disable : 4245) /* conversion from 'int' to 'unsigned int' */
+#  pragma warning(disable : 4295) /* array too small to include terminating null */
+#endif
+#ifndef FREESTANDING
+#  include <string.h>	/* for memcpy() */
+#endif
+
+/* ========================================================================== */
+/*                             Target architecture                            */
+/* ========================================================================== */
+
+/* If possible, define a compiler-independent ARCH_* macro. */
+#undef ARCH_X86_64
+#undef ARCH_X86_32
+#undef ARCH_ARM64
+#undef ARCH_ARM32
+#undef ARCH_RISCV
+#ifdef _MSC_VER
+   /* Way too many things are broken in ARM64EC to pretend that it is x86_64. */
+#  if defined(_M_X64) && !defined(_M_ARM64EC)
+#    define ARCH_X86_64
+#  elif defined(_M_IX86)
+#    define ARCH_X86_32
+#  elif defined(_M_ARM64)
+#    define ARCH_ARM64
+#  elif defined(_M_ARM)
+#    define ARCH_ARM32
+#  endif
+#else
+#  if defined(__x86_64__)
+#    define ARCH_X86_64
+#  elif defined(__i386__)
+#    define ARCH_X86_32
+#  elif defined(__aarch64__)
+#    define ARCH_ARM64
+#  elif defined(__arm__)
+#    define ARCH_ARM32
+#  elif defined(__riscv)
+#    define ARCH_RISCV
+#  endif
+#endif
+
+/* ========================================================================== */
+/*                              Type definitions                              */
+/* ========================================================================== */
+
+/* Fixed-width integer types */
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
+
+/* ssize_t, if not available in <sys/types.h> */
+#ifdef _MSC_VER
+#  ifdef _WIN64
+     typedef long long ssize_t;
+#  else
+     typedef long ssize_t;
+#  endif
+#endif
+
+/*
+ * Word type of the target architecture.  Use 'size_t' instead of
+ * 'unsigned long' to account for platforms such as Windows that use 32-bit
+ * 'unsigned long' on 64-bit architectures.
+ */
+typedef size_t machine_word_t;
+
+/* Number of bytes in a word */
+#define WORDBYTES	((int)sizeof(machine_word_t))
+
+/* Number of bits in a word */
+#define WORDBITS	(8 * WORDBYTES)
+
+/* ========================================================================== */
+/*                         Optional compiler features                         */
+/* ========================================================================== */
+
+/* Compiler version checks.  Only use when absolutely necessary. */
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
+#  define GCC_PREREQ(major, minor)		\
+	(__GNUC__ > (major) ||			\
+	 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#  if !GCC_PREREQ(4, 9)
+#    error "gcc versions older than 4.9 are no longer supported"
+#  endif
+#else
+#  define GCC_PREREQ(major, minor)	0
+#endif
+#ifdef __clang__
+#  ifdef __apple_build_version__
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__apple_build_version__ >= (apple_version))
+#  else
+#    define CLANG_PREREQ(major, minor, apple_version)	\
+	(__clang_major__ > (major) ||			\
+	 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+#  endif
+#  if !CLANG_PREREQ(3, 9, 8000000)
+#    error "clang versions older than 3.9 are no longer supported"
+#  endif
+#else
+#  define CLANG_PREREQ(major, minor, apple_version)	0
+#endif
+#ifdef _MSC_VER
+#  define MSVC_PREREQ(version)	(_MSC_VER >= (version))
+#  if !MSVC_PREREQ(1900)
+#    error "MSVC versions older than Visual Studio 2015 are no longer supported"
+#  endif
+#else
+#  define MSVC_PREREQ(version)	0
+#endif
+
+/*
+ * __has_attribute(attribute) - check whether the compiler supports the given
+ * attribute (and also supports doing the check in the first place).  Mostly
+ * useful just for clang, since gcc didn't add this macro until gcc 5.
+ */
+#ifndef __has_attribute
+#  define __has_attribute(attribute)	0
+#endif
+
+/*
+ * __has_builtin(builtin) - check whether the compiler supports the given
+ * builtin (and also supports doing the check in the first place).  Mostly
+ * useful just for clang, since gcc didn't add this macro until gcc 10.
+ */
+#ifndef __has_builtin
+#  define __has_builtin(builtin)	0
+#endif
+
+/* inline - suggest that a function be inlined */
+#ifdef _MSC_VER
+#  define inline		__inline
+#endif /* else assume 'inline' is usable as-is */
+
+/* forceinline - force a function to be inlined, if possible */
+#if defined(__GNUC__) || __has_attribute(always_inline)
+#  define forceinline		inline __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#  define forceinline		__forceinline
+#else
+#  define forceinline		inline
+#endif
+
+/* MAYBE_UNUSED - mark a function or variable as maybe unused */
+#if defined(__GNUC__) || __has_attribute(unused)
+#  define MAYBE_UNUSED		__attribute__((unused))
+#else
+#  define MAYBE_UNUSED
+#endif
+
+/* NORETURN - mark a function as never returning, e.g. due to calling abort() */
+#if defined(__GNUC__) || __has_attribute(noreturn)
+#  define NORETURN		__attribute__((noreturn))
+#else
+#  define NORETURN
+#endif
+
+/*
+ * restrict - hint that writes only occur through the given pointer.
+ *
+ * Don't use MSVC's __restrict, since it has nonstandard behavior.
+ * Standard restrict is okay, if it is supported.
+ */
+#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L)
+#  if defined(__GNUC__) || defined(__clang__)
+#    define restrict		__restrict__
+#  else
+#    define restrict
+#  endif
+#endif /* else assume 'restrict' is usable as-is */
+
+/* likely(expr) - hint that an expression is usually true */
+#if defined(__GNUC__) || __has_builtin(__builtin_expect)
+#  define likely(expr)		__builtin_expect(!!(expr), 1)
+#else
+#  define likely(expr)		(expr)
+#endif
+
+/* unlikely(expr) - hint that an expression is usually false */
+#if defined(__GNUC__) || __has_builtin(__builtin_expect)
+#  define unlikely(expr)	__builtin_expect(!!(expr), 0)
+#else
+#  define unlikely(expr)	(expr)
+#endif
+
+/* prefetchr(addr) - prefetch into L1 cache for read */
+#undef prefetchr
+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
+#  define prefetchr(addr)	__builtin_prefetch((addr), 0)
+#elif defined(_MSC_VER)
+#  if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+#    define prefetchr(addr)	_mm_prefetch((addr), _MM_HINT_T0)
+#  elif defined(ARCH_ARM64)
+#    define prefetchr(addr)	__prefetch2((addr), 0x00 /* prfop=PLDL1KEEP */)
+#  elif defined(ARCH_ARM32)
+#    define prefetchr(addr)	__prefetch(addr)
+#  endif
+#endif
+#ifndef prefetchr
+#  define prefetchr(addr)
+#endif
+
+/* prefetchw(addr) - prefetch into L1 cache for write */
+#undef prefetchw
+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
+#  define prefetchw(addr)	__builtin_prefetch((addr), 1)
+#elif defined(_MSC_VER)
+#  if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+#    define prefetchw(addr)	_m_prefetchw(addr)
+#  elif defined(ARCH_ARM64)
+#    define prefetchw(addr)	__prefetch2((addr), 0x10 /* prfop=PSTL1KEEP */)
+#  elif defined(ARCH_ARM32)
+#    define prefetchw(addr)	__prefetchw(addr)
+#  endif
+#endif
+#ifndef prefetchw
+#  define prefetchw(addr)
+#endif
+
+/*
+ * _aligned_attribute(n) - declare that the annotated variable, or variables of
+ * the annotated type, must be aligned on n-byte boundaries.
+ */
+#undef _aligned_attribute
+#if defined(__GNUC__) || __has_attribute(aligned)
+#  define _aligned_attribute(n)	__attribute__((aligned(n)))
+#elif defined(_MSC_VER)
+#  define _aligned_attribute(n)	__declspec(align(n))
+#endif
+
+/*
+ * _target_attribute(attrs) - override the compilation target for a function.
+ *
+ * This accepts one or more comma-separated suffixes to the -m prefix jointly
+ * forming the name of a machine-dependent option.  On gcc-like compilers, this
+ * enables codegen for the given targets, including arbitrary compiler-generated
+ * code as well as the corresponding intrinsics.  On other compilers this macro
+ * expands to nothing, though MSVC allows intrinsics to be used anywhere anyway.
+ */
+#if defined(__GNUC__) || __has_attribute(target)
+#  define _target_attribute(attrs)	__attribute__((target(attrs)))
+#else
+#  define _target_attribute(attrs)
+#endif
+
+/* ========================================================================== */
+/*                          Miscellaneous macros                              */
+/* ========================================================================== */
+
+#define ARRAY_LEN(A)		(sizeof(A) / sizeof((A)[0]))
+#define MIN(a, b)		((a) <= (b) ? (a) : (b))
+#define MAX(a, b)		((a) >= (b) ? (a) : (b))
+#define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
+#define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
+#define ROUND_UP(n, d)		((d) * DIV_ROUND_UP((n), (d)))
+
+/* ========================================================================== */
+/*                           Endianness handling                              */
+/* ========================================================================== */
+
+/*
+ * CPU_IS_LITTLE_ENDIAN() - 1 if the CPU is little endian, or 0 if it is big
+ * endian.  When possible this is a compile-time macro that can be used in
+ * preprocessor conditionals.  As a fallback, a generic method is used that
+ * can't be used in preprocessor conditionals but should still be optimized out.
+ */
+#if defined(__BYTE_ORDER__) /* gcc v4.6+ and clang */
+#  define CPU_IS_LITTLE_ENDIAN()  (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#elif defined(_MSC_VER)
+#  define CPU_IS_LITTLE_ENDIAN()  true
+#else
+static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
+{
+	union {
+		u32 w;
+		u8 b;
+	} u;
+
+	u.w = 1;
+	return u.b;
+}
+#endif
+
+/* bswap16(v) - swap the bytes of a 16-bit integer */
+static forceinline u16 bswap16(u16 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
+	return __builtin_bswap16(v);
+#elif defined(_MSC_VER)
+	return _byteswap_ushort(v);
+#else
+	return (v << 8) | (v >> 8);
+#endif
+}
+
+/* bswap32(v) - swap the bytes of a 32-bit integer */
+static forceinline u32 bswap32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
+	return __builtin_bswap32(v);
+#elif defined(_MSC_VER)
+	return _byteswap_ulong(v);
+#else
+	return ((v & 0x000000FF) << 24) |
+	       ((v & 0x0000FF00) << 8) |
+	       ((v & 0x00FF0000) >> 8) |
+	       ((v & 0xFF000000) >> 24);
+#endif
+}
+
+/* bswap64(v) - swap the bytes of a 64-bit integer */
+static forceinline u64 bswap64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
+	return __builtin_bswap64(v);
+#elif defined(_MSC_VER)
+	return _byteswap_uint64(v);
+#else
+	return ((v & 0x00000000000000FF) << 56) |
+	       ((v & 0x000000000000FF00) << 40) |
+	       ((v & 0x0000000000FF0000) << 24) |
+	       ((v & 0x00000000FF000000) << 8) |
+	       ((v & 0x000000FF00000000) >> 8) |
+	       ((v & 0x0000FF0000000000) >> 24) |
+	       ((v & 0x00FF000000000000) >> 40) |
+	       ((v & 0xFF00000000000000) >> 56);
+#endif
+}
+
+#define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
+#define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
+#define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
+#define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
+#define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
+#define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
+
+/* ========================================================================== */
+/*                          Unaligned memory accesses                         */
+/* ========================================================================== */
+
+/*
+ * UNALIGNED_ACCESS_IS_FAST() - 1 if unaligned memory accesses can be performed
+ * efficiently on the target platform, otherwise 0.
+ */
+#if (defined(__GNUC__) || defined(__clang__)) && \
+	(defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
+	 defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
+	 defined(__riscv_misaligned_fast) || \
+	 /*
+	  * For all compilation purposes, WebAssembly behaves like any other CPU
+	  * instruction set. Even though WebAssembly engine might be running on
+	  * top of different actual CPU architectures, the WebAssembly spec
+	  * itself permits unaligned access and it will be fast on most of those
+	  * platforms, and simulated at the engine level on others, so it's
+	  * worth treating it as a CPU architecture with fast unaligned access.
+	  */ defined(__wasm__))
+#  define UNALIGNED_ACCESS_IS_FAST	1
+#elif defined(_MSC_VER)
+#  define UNALIGNED_ACCESS_IS_FAST	1
+#else
+#  define UNALIGNED_ACCESS_IS_FAST	0
+#endif
+
+/*
+ * Implementing unaligned memory accesses using memcpy() is portable, and it
+ * usually gets optimized appropriately by modern compilers.  I.e., each
+ * memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled to a load or store
+ * instruction, not to an actual function call.
+ *
+ * We no longer use the "packed struct" approach to unaligned accesses, as that
+ * is nonstandard, has unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366).  However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
+ */
+
+#ifdef FREESTANDING
+#  define MEMCOPY	__builtin_memcpy
+#else
+#  define MEMCOPY	memcpy
+#endif
+
+/* Unaligned loads and stores without endianness conversion */
+
+#define DEFINE_UNALIGNED_TYPE(type)				\
+static forceinline type						\
+load_##type##_unaligned(const void *p)				\
+{								\
+	type v;							\
+								\
+	MEMCOPY(&v, p, sizeof(v));				\
+	return v;						\
+}								\
+								\
+static forceinline void						\
+store_##type##_unaligned(type v, void *p)			\
+{								\
+	MEMCOPY(p, &v, sizeof(v));				\
+}
+
+DEFINE_UNALIGNED_TYPE(u16)
+DEFINE_UNALIGNED_TYPE(u32)
+DEFINE_UNALIGNED_TYPE(u64)
+DEFINE_UNALIGNED_TYPE(machine_word_t)
+
+#undef MEMCOPY
+
+#define load_word_unaligned	load_machine_word_t_unaligned
+#define store_word_unaligned	store_machine_word_t_unaligned
+
+/* Unaligned loads with endianness conversion */
+
+static forceinline u16
+get_unaligned_le16(const u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST)
+		return le16_bswap(load_u16_unaligned(p));
+	else
+		return ((u16)p[1] << 8) | p[0];
+}
+
+static forceinline u16
+get_unaligned_be16(const u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST)
+		return be16_bswap(load_u16_unaligned(p));
+	else
+		return ((u16)p[0] << 8) | p[1];
+}
+
+static forceinline u32
+get_unaligned_le32(const u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST)
+		return le32_bswap(load_u32_unaligned(p));
+	else
+		return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
+			((u32)p[1] << 8) | p[0];
+}
+
+static forceinline u32
+get_unaligned_be32(const u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST)
+		return be32_bswap(load_u32_unaligned(p));
+	else
+		return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
+			((u32)p[2] << 8) | p[3];
+}
+
+static forceinline u64
+get_unaligned_le64(const u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST)
+		return le64_bswap(load_u64_unaligned(p));
+	else
+		return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
+			((u64)p[5] << 40) | ((u64)p[4] << 32) |
+			((u64)p[3] << 24) | ((u64)p[2] << 16) |
+			((u64)p[1] << 8) | p[0];
+}
+
+static forceinline machine_word_t
+get_unaligned_leword(const u8 *p)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		return get_unaligned_le32(p);
+	else
+		return get_unaligned_le64(p);
+}
+
+/* Unaligned stores with endianness conversion */
+
+static forceinline void
+put_unaligned_le16(u16 v, u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST) {
+		store_u16_unaligned(le16_bswap(v), p);
+	} else {
+		p[0] = (u8)(v >> 0);
+		p[1] = (u8)(v >> 8);
+	}
+}
+
+static forceinline void
+put_unaligned_be16(u16 v, u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST) {
+		store_u16_unaligned(be16_bswap(v), p);
+	} else {
+		p[0] = (u8)(v >> 8);
+		p[1] = (u8)(v >> 0);
+	}
+}
+
+static forceinline void
+put_unaligned_le32(u32 v, u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST) {
+		store_u32_unaligned(le32_bswap(v), p);
+	} else {
+		p[0] = (u8)(v >> 0);
+		p[1] = (u8)(v >> 8);
+		p[2] = (u8)(v >> 16);
+		p[3] = (u8)(v >> 24);
+	}
+}
+
+static forceinline void
+put_unaligned_be32(u32 v, u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST) {
+		store_u32_unaligned(be32_bswap(v), p);
+	} else {
+		p[0] = (u8)(v >> 24);
+		p[1] = (u8)(v >> 16);
+		p[2] = (u8)(v >> 8);
+		p[3] = (u8)(v >> 0);
+	}
+}
+
+static forceinline void
+put_unaligned_le64(u64 v, u8 *p)
+{
+	if (UNALIGNED_ACCESS_IS_FAST) {
+		store_u64_unaligned(le64_bswap(v), p);
+	} else {
+		p[0] = (u8)(v >> 0);
+		p[1] = (u8)(v >> 8);
+		p[2] = (u8)(v >> 16);
+		p[3] = (u8)(v >> 24);
+		p[4] = (u8)(v >> 32);
+		p[5] = (u8)(v >> 40);
+		p[6] = (u8)(v >> 48);
+		p[7] = (u8)(v >> 56);
+	}
+}
+
+static forceinline void
+put_unaligned_leword(machine_word_t v, u8 *p)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		put_unaligned_le32(v, p);
+	else
+		put_unaligned_le64(v, p);
+}
+
+/* ========================================================================== */
+/*                         Bit manipulation functions                         */
+/* ========================================================================== */
+
+/*
+ * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
+ * significant end) of the *most* significant 1 bit in the input value.  The
+ * input value must be nonzero!
+ */
+
+static forceinline unsigned
+bsr32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_clz)
+	return 31 - __builtin_clz(v);
+#elif defined(_MSC_VER)
+	unsigned long i;
+
+	_BitScanReverse(&i, v);
+	return i;
+#else
+	unsigned i = 0;
+
+	while ((v >>= 1) != 0)
+		i++;
+	return i;
+#endif
+}
+
+static forceinline unsigned
+bsr64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_clzll)
+	return 63 - __builtin_clzll(v);
+#elif defined(_MSC_VER) && defined(_WIN64)
+	unsigned long i;
+
+	_BitScanReverse64(&i, v);
+	return i;
+#else
+	unsigned i = 0;
+
+	while ((v >>= 1) != 0)
+		i++;
+	return i;
+#endif
+}
+
+static forceinline unsigned
+bsrw(machine_word_t v)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		return bsr32(v);
+	else
+		return bsr64(v);
+}
+
+/*
+ * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
+ * significant end) of the *least* significant 1 bit in the input value.  The
+ * input value must be nonzero!
+ */
+
+static forceinline unsigned
+bsf32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_ctz)
+	return __builtin_ctz(v);
+#elif defined(_MSC_VER)
+	unsigned long i;
+
+	_BitScanForward(&i, v);
+	return i;
+#else
+	unsigned i = 0;
+
+	for (; (v & 1) == 0; v >>= 1)
+		i++;
+	return i;
+#endif
+}
+
+static forceinline unsigned
+bsf64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
+	return __builtin_ctzll(v);
+#elif defined(_MSC_VER) && defined(_WIN64)
+	unsigned long i;
+
+	_BitScanForward64(&i, v);
+	return i;
+#else
+	unsigned i = 0;
+
+	for (; (v & 1) == 0; v >>= 1)
+		i++;
+	return i;
+#endif
+}
+
+static forceinline unsigned
+bsfw(machine_word_t v)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		return bsf32(v);
+	else
+		return bsf64(v);
+}
+
+/*
+ * rbit32(v): reverse the bits in a 32-bit integer.  This doesn't have a
+ * fallback implementation; use '#ifdef rbit32' to check if this is available.
+ */
+#undef rbit32
+#if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
+	(__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
+static forceinline u32
+rbit32(u32 v)
+{
+	__asm__("rbit %0, %1" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
+static forceinline u32
+rbit32(u32 v)
+{
+	__asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
+	return v;
+}
+#define rbit32 rbit32
+#endif
+
+#endif /* COMMON_DEFS_H */

lib/adler32.c

@@ -26,15 +26,14 @@
  */
 
 #include "lib_common.h"
-#include "libdeflate.h"
 
-/* The Adler-32 divisor, or "base", value. */
+/* The Adler-32 divisor, or "base", value */
 #define DIVISOR 65521
 
 /*
- * MAX_CHUNK_SIZE is the most bytes that can be processed without the
- * possibility of s2 overflowing when it is represented as an unsigned 32-bit
- * integer.  This value was computed using the following Python script:
+ * MAX_CHUNK_LEN is the most bytes that can be processed without the possibility
+ * of s2 overflowing when it is represented as an unsigned 32-bit integer.  This
+ * value was computed using the following Python script:
  *
  *	divisor = 65521
  *	count = 0
@@ -52,62 +51,94 @@
  * has value 0xFF and that s1 and s2 started with the highest possible values
  * modulo the divisor.
  */
-#define MAX_CHUNK_SIZE	5552
+#define MAX_CHUNK_LEN	5552
 
-typedef u32 (*adler32_func_t)(u32, const u8 *, size_t);
+/*
+ * Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,
+ * update n to 0, and reduce s1 and s2 mod DIVISOR.  It is assumed that neither
+ * s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes
+ * already processed after the last reduction must not exceed MAX_CHUNK_LEN.
+ *
+ * This uses only portable C code.  This is used as a fallback when a vectorized
+ * implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.
+ *
+ * Some of the vectorized implementations also use this to handle the end of the
+ * data when the data isn't evenly divisible by the length the vectorized code
+ * works on.  To avoid compiler errors about target-specific option mismatches
+ * when this is used in that way, this is a macro rather than a function.
+ *
+ * Although this is unvectorized, this does include an optimization where the
+ * main loop processes four bytes at a time using a strategy similar to that
+ * used by vectorized implementations.  This provides increased instruction-
+ * level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.
+ */
+#define ADLER32_CHUNK(s1, s2, p, n)					\
+do {									\
+	if (n >= 4) {							\
+		u32 s1_sum = 0;						\
+		u32 byte_0_sum = 0;					\
+		u32 byte_1_sum = 0;					\
+		u32 byte_2_sum = 0;					\
+		u32 byte_3_sum = 0;					\
+									\
+		do {							\
+			s1_sum += s1;					\
+			s1 += p[0] + p[1] + p[2] + p[3];		\
+			byte_0_sum += p[0];				\
+			byte_1_sum += p[1];				\
+			byte_2_sum += p[2];				\
+			byte_3_sum += p[3];				\
+			p += 4;						\
+			n -= 4;						\
+		} while (n >= 4);					\
+		s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) +	\
+		      (2 * byte_2_sum) + byte_3_sum;			\
+	}								\
+	for (; n; n--, p++) {						\
+		s1 += *p;						\
+		s2 += s1;						\
+	}								\
+	s1 %= DIVISOR;							\
+	s2 %= DIVISOR;							\
+} while (0)
 
-/* Include architecture-specific implementations if available */
-#undef DEFAULT_IMPL
-#undef DISPATCH
-#if defined(__arm__) || defined(__aarch64__)
-#  include "arm/adler32_impl.h"
-#elif defined(__i386__) || defined(__x86_64__)
-#  include "x86/adler32_impl.h"
-#endif
-
-/* Define a generic implementation if needed */
-#ifndef DEFAULT_IMPL
-#define DEFAULT_IMPL adler32_generic
-static u32 adler32_generic(u32 adler, const u8 *p, size_t size)
+static u32 MAYBE_UNUSED
+adler32_generic(u32 adler, const u8 *p, size_t len)
 {
 	u32 s1 = adler & 0xFFFF;
 	u32 s2 = adler >> 16;
-	const u8 * const end = p + size;
 
-	while (p != end) {
-		size_t chunk_size = MIN(end - p, MAX_CHUNK_SIZE);
-		const u8 *chunk_end = p + chunk_size;
-		size_t num_unrolled_iterations = chunk_size / 4;
+	while (len) {
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
 
-		while (num_unrolled_iterations--) {
-			s1 += *p++;
-			s2 += s1;
-			s1 += *p++;
-			s2 += s1;
-			s1 += *p++;
-			s2 += s1;
-			s1 += *p++;
-			s2 += s1;
-		}
-		while (p != chunk_end) {
-			s1 += *p++;
-			s2 += s1;
-		}
-		s1 %= DIVISOR;
-		s2 %= DIVISOR;
+		len -= n;
+		ADLER32_CHUNK(s1, s2, p, n);
 	}
 
 	return (s2 << 16) | s1;
 }
-#endif /* !DEFAULT_IMPL */
 
-#ifdef DISPATCH
-static u32 dispatch(u32, const u8 *, size_t);
+/* Include architecture-specific implementation(s) if available. */
+#undef DEFAULT_IMPL
+#undef arch_select_adler32_func
+typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+#  include "arm/adler32_impl.h"
+#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
+#  include "x86/adler32_impl.h"
+#endif
 
-static volatile adler32_func_t adler32_impl = dispatch;
+#ifndef DEFAULT_IMPL
+#  define DEFAULT_IMPL adler32_generic
+#endif
 
-/* Choose the fastest implementation at runtime */
-static u32 dispatch(u32 adler, const u8 *buffer, size_t size)
+#ifdef arch_select_adler32_func
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);
+
+static volatile adler32_func_t adler32_impl = dispatch_adler32;
+
+/* Choose the best implementation at runtime. */
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)
 {
 	adler32_func_t f = arch_select_adler32_func();
 
@@ -115,16 +146,17 @@ static u32 dispatch(u32 adler, const u8 *buffer, size_t size)
 		f = DEFAULT_IMPL;
 
 	adler32_impl = f;
-	return adler32_impl(adler, buffer, size);
+	return f(adler, p, len);
 }
 #else
-#  define adler32_impl DEFAULT_IMPL /* only one implementation, use it */
+/* The best implementation is statically known, so call it directly. */
+#define adler32_impl DEFAULT_IMPL
 #endif
 
-LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
-libdeflate_adler32(u32 adler, const void *buffer, size_t size)
+LIBDEFLATEAPI u32
+libdeflate_adler32(u32 adler, const void *buffer, size_t len)
 {
-	if (buffer == NULL) /* return initial value */
+	if (buffer == NULL) /* Return initial value. */
 		return 1;
-	return adler32_impl(adler, buffer, size);
+	return adler32_impl(adler, buffer, len);
 }

lib/adler32_vec_template.h

@@ -1,124 +0,0 @@
-/*
- * adler32_vec_template.h - template for vectorized Adler-32 implementations
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * This file contains a template for vectorized Adler-32 implementations.
- *
- * The inner loop between reductions modulo 65521 of an unvectorized Adler-32
- * implementation looks something like this:
- *
- *	do {
- *		s1 += *p;
- *		s2 += s1;
- *	} while (++p != chunk_end);
- *
- * For vectorized calculation of s1, we only need to sum the input bytes.  They
- * can be accumulated into multiple counters which are eventually summed
- * together.
- *
- * For vectorized calculation of s2, the basic idea is that for each iteration
- * that processes N bytes, we can perform the following vectorizable
- * calculation:
- *
- *	s2 += N*byte_1 + (N-1)*byte_2 + (N-2)*byte_3 + ... + 1*byte_N
- *
- * Or, equivalently, we can sum the byte_1...byte_N for each iteration into N
- * separate counters, then do the multiplications by N...1 just once at the end
- * rather than once per iteration.
- *
- * Also, we must account for how previous bytes will affect s2 by doing the
- * following at beginning of each iteration:
- *
- *	s2 += s1 * N
- *
- * Furthermore, like s1, "s2" can actually be multiple counters which are
- * eventually summed together.
- */
-
-static u32 ATTRIBUTES
-FUNCNAME(u32 adler, const u8 *p, size_t size)
-{
-	u32 s1 = adler & 0xFFFF;
-	u32 s2 = adler >> 16;
-	const u8 * const end = p + size;
-	const u8 *vend;
-	const size_t max_chunk_size =
-		MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) -
-		(MIN(MAX_CHUNK_SIZE, IMPL_MAX_CHUNK_SIZE) %
-		 IMPL_SEGMENT_SIZE);
-
-	/* Process a byte at a time until the needed alignment is reached */
-	if (p != end && (uintptr_t)p % IMPL_ALIGNMENT) {
-		do {
-			s1 += *p++;
-			s2 += s1;
-		} while (p != end && (uintptr_t)p % IMPL_ALIGNMENT);
-		s1 %= DIVISOR;
-		s2 %= DIVISOR;
-	}
-
-	/*
-	 * Process "chunks" of bytes using vector instructions.  Chunk sizes are
-	 * limited to MAX_CHUNK_SIZE, which guarantees that s1 and s2 never
-	 * overflow before being reduced modulo DIVISOR.  For vector processing,
-	 * chunk sizes are also made evenly divisible by IMPL_SEGMENT_SIZE and
-	 * may be further limited to IMPL_MAX_CHUNK_SIZE.
-	 */
-	STATIC_ASSERT(IMPL_SEGMENT_SIZE % IMPL_ALIGNMENT == 0);
-	vend = end - ((size_t)(end - p) % IMPL_SEGMENT_SIZE);
-	while (p != vend) {
-		size_t chunk_size = MIN((size_t)(vend - p), max_chunk_size);
-
-		s2 += s1 * chunk_size;
-
-		FUNCNAME_CHUNK((const void *)p, (const void *)(p + chunk_size),
-			       &s1, &s2);
-
-		p += chunk_size;
-		s1 %= DIVISOR;
-		s2 %= DIVISOR;
-	}
-
-	/* Process any remaining bytes */
-	if (p != end) {
-		do {
-			s1 += *p++;
-			s2 += s1;
-		} while (p != end);
-		s1 %= DIVISOR;
-		s2 %= DIVISOR;
-	}
-
-	return (s2 << 16) | s1;
-}
-
-#undef FUNCNAME
-#undef FUNCNAME_CHUNK
-#undef ATTRIBUTES
-#undef IMPL_ALIGNMENT
-#undef IMPL_SEGMENT_SIZE
-#undef IMPL_MAX_CHUNK_SIZE

lib/arm/adler32_impl.h

@@ -30,96 +30,336 @@
 
 #include "cpu_features.h"
 
-/* NEON implementation */
-#undef DISPATCH_NEON
-#if !defined(DEFAULT_IMPL) &&	\
-	(defined(__ARM_NEON) || (ARM_CPU_FEATURES_ENABLED &&	\
-				 COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS))
-#  define FUNCNAME		adler32_neon
-#  define FUNCNAME_CHUNK	adler32_neon_chunk
-#  define IMPL_ALIGNMENT	16
-#  define IMPL_SEGMENT_SIZE	32
-/* Prevent unsigned overflow of the 16-bit precision byte counters */
-#  define IMPL_MAX_CHUNK_SIZE	(32 * (0xFFFF / 0xFF))
-#  ifdef __ARM_NEON
+/* Regular NEON implementation */
+#if HAVE_NEON_INTRIN && CPU_IS_LITTLE_ENDIAN()
+#  define adler32_arm_neon	adler32_arm_neon
+#  if HAVE_NEON_NATIVE
+     /*
+      * Use no attributes if none are needed, to support old versions of clang
+      * that don't accept the simd target attribute.
+      */
 #    define ATTRIBUTES
-#    define DEFAULT_IMPL	adler32_neon
+#  elif defined(ARCH_ARM32)
+#    define ATTRIBUTES	_target_attribute("fpu=neon")
+#  elif defined(__clang__)
+#    define ATTRIBUTES	_target_attribute("simd")
 #  else
-#    ifdef __arm__
-#      define ATTRIBUTES	__attribute__((target("fpu=neon")))
-#    else
-#      define ATTRIBUTES	__attribute__((target("+simd")))
-#    endif
-#    define DISPATCH		1
-#    define DISPATCH_NEON	1
+#    define ATTRIBUTES	_target_attribute("+simd")
 #  endif
-#  include <arm_neon.h>
-static forceinline ATTRIBUTES void
-adler32_neon_chunk(const uint8x16_t *p, const uint8x16_t * const end,
-		   u32 *s1, u32 *s2)
+static ATTRIBUTES MAYBE_UNUSED u32
+adler32_arm_neon(u32 adler, const u8 *p, size_t len)
 {
-	uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 };
-	uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, 0 };
-	uint16x8_t v_byte_sums_a = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
-	uint16x8_t v_byte_sums_b = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
-	uint16x8_t v_byte_sums_c = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
-	uint16x8_t v_byte_sums_d = (uint16x8_t) { 0, 0, 0, 0, 0, 0, 0, 0 };
+	static const u16 _aligned_attribute(16) mults[64] = {
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const uint16x8_t mults_a = vld1q_u16(&mults[0]);
+	const uint16x8_t mults_b = vld1q_u16(&mults[8]);
+	const uint16x8_t mults_c = vld1q_u16(&mults[16]);
+	const uint16x8_t mults_d = vld1q_u16(&mults[24]);
+	const uint16x8_t mults_e = vld1q_u16(&mults[32]);
+	const uint16x8_t mults_f = vld1q_u16(&mults[40]);
+	const uint16x8_t mults_g = vld1q_u16(&mults[48]);
+	const uint16x8_t mults_h = vld1q_u16(&mults[56]);
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
 
-	do {
-		const uint8x16_t bytes1 = *p++;
-		const uint8x16_t bytes2 = *p++;
-		uint16x8_t tmp;
+	/*
+	 * If the length is large and the pointer is misaligned, align it.
+	 * For smaller lengths, just take the misaligned load penalty.
+	 */
+	if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+			len--;
+		} while ((uintptr_t)p & 15);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
 
-		v_s2 += v_s1;
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX.
+		 */
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
 
-		/* Vector Pairwise Add Long (u8 => u16) */
-		tmp = vpaddlq_u8(bytes1);
+		len -= n;
 
-		/* Vector Pairwise Add and Accumulate Long (u8 => u16) */
-		tmp = vpadalq_u8(tmp, bytes2);
+		if (n >= 64) {
+			uint32x4_t v_s1 = vdupq_n_u32(0);
+			uint32x4_t v_s2 = vdupq_n_u32(0);
+			/*
+			 * v_byte_sums_* contain the sum of the bytes at index i
+			 * across all 64-byte segments, for each index 0..63.
+			 */
+			uint16x8_t v_byte_sums_a = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_b = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_c = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_d = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_e = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_f = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_g = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_h = vdupq_n_u16(0);
 
-		/* Vector Pairwise Add and Accumulate Long (u16 => u32) */
-		v_s1 = vpadalq_u16(v_s1, tmp);
+			s2 += s1 * (n & ~63);
 
-		/* Vector Add Wide (u8 => u16) */
-		v_byte_sums_a = vaddw_u8(v_byte_sums_a, vget_low_u8(bytes1));
-		v_byte_sums_b = vaddw_u8(v_byte_sums_b, vget_high_u8(bytes1));
-		v_byte_sums_c = vaddw_u8(v_byte_sums_c, vget_low_u8(bytes2));
-		v_byte_sums_d = vaddw_u8(v_byte_sums_d, vget_high_u8(bytes2));
+			do {
+				/* Load the next 64 data bytes. */
+				const uint8x16_t data_a = vld1q_u8(p + 0);
+				const uint8x16_t data_b = vld1q_u8(p + 16);
+				const uint8x16_t data_c = vld1q_u8(p + 32);
+				const uint8x16_t data_d = vld1q_u8(p + 48);
+				uint16x8_t tmp;
 
-	} while (p != end);
+				/*
+				 * Accumulate the previous s1 counters into the
+				 * s2 counters.  The needed multiplication by 64
+				 * is delayed to later.
+				 */
+				v_s2 = vaddq_u32(v_s2, v_s1);
 
-	/* Vector Shift Left (u32) */
-	v_s2 = vqshlq_n_u32(v_s2, 5);
+				/*
+				 * Add the 64 data bytes to their v_byte_sums
+				 * counters, while also accumulating the sums of
+				 * each adjacent set of 4 bytes into v_s1.
+				 */
+				tmp = vpaddlq_u8(data_a);
+				v_byte_sums_a = vaddw_u8(v_byte_sums_a,
+							 vget_low_u8(data_a));
+				v_byte_sums_b = vaddw_u8(v_byte_sums_b,
+							 vget_high_u8(data_a));
+				tmp = vpadalq_u8(tmp, data_b);
+				v_byte_sums_c = vaddw_u8(v_byte_sums_c,
+							 vget_low_u8(data_b));
+				v_byte_sums_d = vaddw_u8(v_byte_sums_d,
+							 vget_high_u8(data_b));
+				tmp = vpadalq_u8(tmp, data_c);
+				v_byte_sums_e = vaddw_u8(v_byte_sums_e,
+							 vget_low_u8(data_c));
+				v_byte_sums_f = vaddw_u8(v_byte_sums_f,
+							 vget_high_u8(data_c));
+				tmp = vpadalq_u8(tmp, data_d);
+				v_byte_sums_g = vaddw_u8(v_byte_sums_g,
+							 vget_low_u8(data_d));
+				v_byte_sums_h = vaddw_u8(v_byte_sums_h,
+							 vget_high_u8(data_d));
+				v_s1 = vpadalq_u16(v_s1, tmp);
 
-	/* Vector Multiply Accumulate Long (u16 => u32) */
-	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),  (uint16x4_t) { 32, 31, 30, 29 });
-	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_a), (uint16x4_t) { 28, 27, 26, 25 });
-	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),  (uint16x4_t) { 24, 23, 22, 21 });
-	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_b), (uint16x4_t) { 20, 19, 18, 17 });
-	v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),  (uint16x4_t) { 16, 15, 14, 13 });
-	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_c), (uint16x4_t) { 12, 11, 10,  9 });
-	v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_byte_sums_d), (uint16x4_t) {  8,  7,  6,  5 });
-	v_s2 = vmlal_u16(v_s2, vget_high_u16(v_byte_sums_d), (uint16x4_t) {  4,  3,  2,  1 });
+				p += 64;
+				n -= 64;
+			} while (n >= 64);
 
-	*s1 += v_s1[0] + v_s1[1] + v_s1[2] + v_s1[3];
-	*s2 += v_s2[0] + v_s2[1] + v_s2[2] + v_s2[3];
+			/* s2 = 64*s2 + (64*bytesum0 + 63*bytesum1 + ... + 1*bytesum63) */
+		#ifdef ARCH_ARM32
+		#  define umlal2(a, b, c)  vmlal_u16((a), vget_high_u16(b), vget_high_u16(c))
+		#else
+		#  define umlal2	   vmlal_high_u16
+		#endif
+			v_s2 = vqshlq_n_u32(v_s2, 6);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),
+					 vget_low_u16(mults_a));
+			v_s2 = umlal2(v_s2, v_byte_sums_a, mults_a);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),
+					 vget_low_u16(mults_b));
+			v_s2 = umlal2(v_s2, v_byte_sums_b, mults_b);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),
+					 vget_low_u16(mults_c));
+			v_s2 = umlal2(v_s2, v_byte_sums_c, mults_c);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_d),
+					 vget_low_u16(mults_d));
+			v_s2 = umlal2(v_s2, v_byte_sums_d, mults_d);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_e),
+					 vget_low_u16(mults_e));
+			v_s2 = umlal2(v_s2, v_byte_sums_e, mults_e);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_f),
+					 vget_low_u16(mults_f));
+			v_s2 = umlal2(v_s2, v_byte_sums_f, mults_f);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_g),
+					 vget_low_u16(mults_g));
+			v_s2 = umlal2(v_s2, v_byte_sums_g, mults_g);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_h),
+					 vget_low_u16(mults_h));
+			v_s2 = umlal2(v_s2, v_byte_sums_h, mults_h);
+		#undef umlal2
+
+			/* Horizontal sum to finish up */
+		#ifdef ARCH_ARM32
+			s1 += vgetq_lane_u32(v_s1, 0) + vgetq_lane_u32(v_s1, 1) +
+			      vgetq_lane_u32(v_s1, 2) + vgetq_lane_u32(v_s1, 3);
+			s2 += vgetq_lane_u32(v_s2, 0) + vgetq_lane_u32(v_s2, 1) +
+			      vgetq_lane_u32(v_s2, 2) + vgetq_lane_u32(v_s2, 3);
+		#else
+			s1 += vaddvq_u32(v_s1);
+			s2 += vaddvq_u32(v_s2);
+		#endif
+		}
+		/*
+		 * Process the last 0 <= n < 64 bytes of the chunk using
+		 * scalar instructions and reduce s1 and s2 mod DIVISOR.
+		 */
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+	return (s2 << 16) | s1;
 }
-#  include "../adler32_vec_template.h"
-#endif /* NEON implementation */
+#undef ATTRIBUTES
+#endif /* Regular NEON implementation */
 
-#ifdef DISPATCH
+/* NEON+dotprod implementation */
+#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN() && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD)
+#  define adler32_arm_neon_dotprod	adler32_arm_neon_dotprod
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("dotprod")
+   /*
+    * Both gcc and binutils originally considered dotprod to depend on
+    * arch=armv8.2-a or later.  This was fixed in gcc 13.2 by commit
+    * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+    * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+    * dependencies from features").  Unfortunately, always using arch=armv8.2-a
+    * causes build errors with some compiler options because it may reduce the
+    * arch rather than increase it.  Therefore we try to omit the arch whenever
+    * possible.  If gcc is 14 or later, then both gcc and binutils are probably
+    * fixed, so we omit the arch.  We also omit the arch if a feature that
+    * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
+    */
+#  elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+			  || defined(__ARM_FEATURE_DOTPROD)
+#    define ATTRIBUTES	_target_attribute("+dotprod")
+#  else
+#    define ATTRIBUTES	_target_attribute("arch=armv8.2-a+dotprod")
+#  endif
+static ATTRIBUTES u32
+adler32_arm_neon_dotprod(u32 adler, const u8 *p, size_t len)
+{
+	static const u8 _aligned_attribute(16) mults[64] = {
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const uint8x16_t mults_a = vld1q_u8(&mults[0]);
+	const uint8x16_t mults_b = vld1q_u8(&mults[16]);
+	const uint8x16_t mults_c = vld1q_u8(&mults[32]);
+	const uint8x16_t mults_d = vld1q_u8(&mults[48]);
+	const uint8x16_t ones = vdupq_n_u8(1);
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+
+	/*
+	 * If the length is large and the pointer is misaligned, align it.
+	 * For smaller lengths, just take the misaligned load penalty.
+	 */
+	if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+			len--;
+		} while ((uintptr_t)p & 15);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX.
+		 */
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
+
+		len -= n;
+
+		if (n >= 64) {
+			uint32x4_t v_s1_a = vdupq_n_u32(0);
+			uint32x4_t v_s1_b = vdupq_n_u32(0);
+			uint32x4_t v_s1_c = vdupq_n_u32(0);
+			uint32x4_t v_s1_d = vdupq_n_u32(0);
+			uint32x4_t v_s2_a = vdupq_n_u32(0);
+			uint32x4_t v_s2_b = vdupq_n_u32(0);
+			uint32x4_t v_s2_c = vdupq_n_u32(0);
+			uint32x4_t v_s2_d = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_a = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_b = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_c = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_d = vdupq_n_u32(0);
+			uint32x4_t v_s1;
+			uint32x4_t v_s2;
+			uint32x4_t v_s1_sums;
+
+			s2 += s1 * (n & ~63);
+
+			do {
+				uint8x16_t data_a = vld1q_u8(p + 0);
+				uint8x16_t data_b = vld1q_u8(p + 16);
+				uint8x16_t data_c = vld1q_u8(p + 32);
+				uint8x16_t data_d = vld1q_u8(p + 48);
+
+				v_s1_sums_a = vaddq_u32(v_s1_sums_a, v_s1_a);
+				v_s1_a = vdotq_u32(v_s1_a, data_a, ones);
+				v_s2_a = vdotq_u32(v_s2_a, data_a, mults_a);
+
+				v_s1_sums_b = vaddq_u32(v_s1_sums_b, v_s1_b);
+				v_s1_b = vdotq_u32(v_s1_b, data_b, ones);
+				v_s2_b = vdotq_u32(v_s2_b, data_b, mults_b);
+
+				v_s1_sums_c = vaddq_u32(v_s1_sums_c, v_s1_c);
+				v_s1_c = vdotq_u32(v_s1_c, data_c, ones);
+				v_s2_c = vdotq_u32(v_s2_c, data_c, mults_c);
+
+				v_s1_sums_d = vaddq_u32(v_s1_sums_d, v_s1_d);
+				v_s1_d = vdotq_u32(v_s1_d, data_d, ones);
+				v_s2_d = vdotq_u32(v_s2_d, data_d, mults_d);
+
+				p += 64;
+				n -= 64;
+			} while (n >= 64);
+
+			v_s1 = vaddq_u32(vaddq_u32(v_s1_a, v_s1_b),
+					 vaddq_u32(v_s1_c, v_s1_d));
+			v_s2 = vaddq_u32(vaddq_u32(v_s2_a, v_s2_b),
+					 vaddq_u32(v_s2_c, v_s2_d));
+			v_s1_sums = vaddq_u32(vaddq_u32(v_s1_sums_a,
+							v_s1_sums_b),
+					      vaddq_u32(v_s1_sums_c,
+							v_s1_sums_d));
+			v_s2 = vaddq_u32(v_s2, vqshlq_n_u32(v_s1_sums, 6));
+
+			s1 += vaddvq_u32(v_s1);
+			s2 += vaddvq_u32(v_s2);
+		}
+		/*
+		 * Process the last 0 <= n < 64 bytes of the chunk using
+		 * scalar instructions and reduce s1 and s2 mod DIVISOR.
+		 */
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+	return (s2 << 16) | s1;
+}
+#undef ATTRIBUTES
+#endif /* NEON+dotprod implementation */
+
+#if defined(adler32_arm_neon_dotprod) && defined(__ARM_FEATURE_DOTPROD)
+#define DEFAULT_IMPL	adler32_arm_neon_dotprod
+#else
 static inline adler32_func_t
 arch_select_adler32_func(void)
 {
-	u32 features = get_cpu_features();
+	const u32 features MAYBE_UNUSED = get_arm_cpu_features();
 
-#ifdef DISPATCH_NEON
-	if (features & ARM_CPU_FEATURE_NEON)
-		return adler32_neon;
+#ifdef adler32_arm_neon_dotprod
+	if (HAVE_NEON(features) && HAVE_DOTPROD(features))
+		return adler32_arm_neon_dotprod;
+#endif
+#ifdef adler32_arm_neon
+	if (HAVE_NEON(features))
+		return adler32_arm_neon;
 #endif
 	return NULL;
 }
-#endif /* DISPATCH */
+#define arch_select_adler32_func	arch_select_adler32_func
+#endif
 
 #endif /* LIB_ARM_ADLER32_IMPL_H */

lib/arm/cpu_features.c

@@ -1,5 +1,5 @@
 /*
- * arm/cpu_features.c - feature detection for ARM processors
+ * arm/cpu_features.c - feature detection for ARM CPUs
  *
  * Copyright 2018 Eric Biggers
  *
@@ -30,10 +30,17 @@
  * features.  But an OS-specific way can be used when available.
  */
 
+#ifdef __APPLE__
+#  undef _ANSI_SOURCE
+#  undef _DARWIN_C_SOURCE
+#  define _DARWIN_C_SOURCE /* for sysctlbyname() */
+#endif
+
 #include "../cpu_features_common.h" /* must be included first */
 #include "cpu_features.h"
 
-#if ARM_CPU_FEATURES_ENABLED
+#ifdef ARM_CPU_FEATURES_KNOWN
+/* Runtime ARM CPU feature detection is supported. */
 
 #ifdef __linux__
 /*
@@ -95,7 +102,7 @@ out:
 	close(fd);
 }
 
-static u32 get_arm_cpu_features(void)
+static u32 query_arm_cpu_features(void)
 {
 	u32 features = 0;
 	unsigned long hwcap = 0;
@@ -103,14 +110,10 @@ static u32 get_arm_cpu_features(void)
 
 	scan_auxv(&hwcap, &hwcap2);
 
-#ifdef __arm__
+#ifdef ARCH_ARM32
 	STATIC_ASSERT(sizeof(long) == 4);
 	if (hwcap & (1 << 12))	/* HWCAP_NEON */
 		features |= ARM_CPU_FEATURE_NEON;
-	if (hwcap2 & (1 << 1))	/* HWCAP2_PMULL */
-		features |= ARM_CPU_FEATURE_PMULL;
-	if (hwcap2 & (1 << 4))	/* HWCAP2_CRC32 */
-		features |= ARM_CPU_FEATURE_CRC32;
 #else
 	STATIC_ASSERT(sizeof(long) == 8);
 	if (hwcap & (1 << 1))	/* HWCAP_ASIMD */
@@ -119,6 +122,10 @@ static u32 get_arm_cpu_features(void)
 		features |= ARM_CPU_FEATURE_PMULL;
 	if (hwcap & (1 << 7))	/* HWCAP_CRC32 */
 		features |= ARM_CPU_FEATURE_CRC32;
+	if (hwcap & (1 << 17))	/* HWCAP_SHA3 */
+		features |= ARM_CPU_FEATURE_SHA3;
+	if (hwcap & (1 << 20))	/* HWCAP_ASIMDDP */
+		features |= ARM_CPU_FEATURE_DOTPROD;
 #endif
 	return features;
 }
@@ -128,18 +135,22 @@ static u32 get_arm_cpu_features(void)
 
 #include <sys/types.h>
 #include <sys/sysctl.h>
+#include <TargetConditionals.h>
 
 static const struct {
 	const char *name;
 	u32 feature;
 } feature_sysctls[] = {
-	{ "hw.optional.neon",		ARM_CPU_FEATURE_NEON },
-	{ "hw.optional.AdvSIMD",	ARM_CPU_FEATURE_NEON },
-	{ "hw.optional.arm.FEAT_PMULL",	ARM_CPU_FEATURE_PMULL },
-	{ "hw.optional.armv8_crc32",	ARM_CPU_FEATURE_CRC32 },
+	{ "hw.optional.neon",		  ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.AdvSIMD",	  ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.arm.FEAT_PMULL",	  ARM_CPU_FEATURE_PMULL },
+	{ "hw.optional.armv8_crc32",	  ARM_CPU_FEATURE_CRC32 },
+	{ "hw.optional.armv8_2_sha3",	  ARM_CPU_FEATURE_SHA3 },
+	{ "hw.optional.arm.FEAT_SHA3",	  ARM_CPU_FEATURE_SHA3 },
+	{ "hw.optional.arm.FEAT_DotProd", ARM_CPU_FEATURE_DOTPROD },
 };
 
-static u32 get_arm_cpu_features(void)
+static u32 query_arm_cpu_features(void)
 {
 	u32 features = 0;
 	size_t i;
@@ -155,6 +166,29 @@ static u32 get_arm_cpu_features(void)
 	}
 	return features;
 }
+#elif defined(_WIN32)
+
+#include <windows.h>
+
+#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE /* added in Windows SDK 20348 */
+#  define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
+#endif
+
+static u32 query_arm_cpu_features(void)
+{
+	u32 features = ARM_CPU_FEATURE_NEON;
+
+	if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_CRC32;
+	if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_DOTPROD;
+
+	/* FIXME: detect SHA3 support too. */
+
+	return features;
+}
 #else
 #error "unhandled case"
 #endif
@@ -162,19 +196,35 @@ static u32 get_arm_cpu_features(void)
 static const struct cpu_feature arm_cpu_feature_table[] = {
 	{ARM_CPU_FEATURE_NEON,		"neon"},
 	{ARM_CPU_FEATURE_PMULL,		"pmull"},
+	{ARM_CPU_FEATURE_PREFER_PMULL,  "prefer_pmull"},
 	{ARM_CPU_FEATURE_CRC32,		"crc32"},
+	{ARM_CPU_FEATURE_SHA3,		"sha3"},
+	{ARM_CPU_FEATURE_DOTPROD,	"dotprod"},
 };
 
-volatile u32 _cpu_features = 0;
+volatile u32 libdeflate_arm_cpu_features = 0;
 
-void setup_cpu_features(void)
+void libdeflate_init_arm_cpu_features(void)
 {
-	u32 features = get_arm_cpu_features();
+	u32 features = query_arm_cpu_features();
+
+	/*
+	 * On the Apple M1 processor, crc32 instructions max out at about 25.5
+	 * GB/s in the best case of using a 3-way or greater interleaved chunked
+	 * implementation, whereas a pmull-based implementation achieves 68 GB/s
+	 * provided that the stride length is large enough (about 10+ vectors
+	 * with eor3, or 12+ without).
+	 *
+	 * Assume that crc32 instructions are preferable in other cases.
+	 */
+#if (defined(__APPLE__) && TARGET_OS_OSX) || defined(TEST_SUPPORT__DO_NOT_USE)
+	features |= ARM_CPU_FEATURE_PREFER_PMULL;
+#endif
 
 	disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
 					 ARRAY_LEN(arm_cpu_feature_table));
 
-	_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
+	libdeflate_arm_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
 }
 
-#endif /* ARM_CPU_FEATURES_ENABLED */
+#endif /* ARM_CPU_FEATURES_KNOWN */

lib/arm/cpu_features.h

@@ -1,5 +1,28 @@
 /*
- * arm/cpu_features.h - feature detection for ARM processors
+ * arm/cpu_features.h - feature detection for ARM CPUs
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #ifndef LIB_ARM_CPU_FEATURES_H
@@ -7,35 +30,185 @@
 
 #include "../lib_common.h"
 
-#if (defined(__arm__) || defined(__aarch64__)) && \
-	(defined(__linux__) || \
-	 (defined(__aarch64__) && defined(__APPLE__))) && \
-	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE && \
-	!defined(FREESTANDING)
-#  define ARM_CPU_FEATURES_ENABLED 1
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+
+#define ARM_CPU_FEATURE_NEON		(1 << 0)
+#define ARM_CPU_FEATURE_PMULL		(1 << 1)
+/*
+ * PREFER_PMULL indicates that the CPU has very high pmull throughput, and so
+ * the 12x wide pmull-based CRC-32 implementation is likely to be faster than an
+ * implementation based on the crc32 instructions.
+ */
+#define ARM_CPU_FEATURE_PREFER_PMULL	(1 << 2)
+#define ARM_CPU_FEATURE_CRC32		(1 << 3)
+#define ARM_CPU_FEATURE_SHA3		(1 << 4)
+#define ARM_CPU_FEATURE_DOTPROD		(1 << 5)
+
+#if !defined(FREESTANDING) && \
+    (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+    (defined(__linux__) || \
+     (defined(__APPLE__) && defined(ARCH_ARM64)) || \
+     (defined(_WIN32) && defined(ARCH_ARM64)))
+/* Runtime ARM CPU feature detection is supported. */
+#  define ARM_CPU_FEATURES_KNOWN	(1U << 31)
+extern volatile u32 libdeflate_arm_cpu_features;
+
+void libdeflate_init_arm_cpu_features(void);
+
+static inline u32 get_arm_cpu_features(void)
+{
+	if (libdeflate_arm_cpu_features == 0)
+		libdeflate_init_arm_cpu_features();
+	return libdeflate_arm_cpu_features;
+}
 #else
-#  define ARM_CPU_FEATURES_ENABLED 0
+static inline u32 get_arm_cpu_features(void) { return 0; }
 #endif
 
-#if ARM_CPU_FEATURES_ENABLED
+/* NEON */
+#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
+#  define HAVE_NEON(features)	1
+#  define HAVE_NEON_NATIVE	1
+#else
+#  define HAVE_NEON(features)	((features) & ARM_CPU_FEATURE_NEON)
+#  define HAVE_NEON_NATIVE	0
+#endif
+/*
+ * With both gcc and clang, NEON intrinsics require that the main target has
+ * NEON enabled already.  Exception: with gcc 6.1 and later (r230411 for arm32,
+ * r226563 for arm64), hardware floating point support is sufficient.
+ */
+#if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+	(HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
+#  define HAVE_NEON_INTRIN	1
+#  include <arm_neon.h>
+#else
+#  define HAVE_NEON_INTRIN	0
+#endif
 
-#define ARM_CPU_FEATURE_NEON		0x00000001
-#define ARM_CPU_FEATURE_PMULL		0x00000002
-#define ARM_CPU_FEATURE_CRC32		0x00000004
+/* PMULL */
+#ifdef __ARM_FEATURE_CRYPTO
+#  define HAVE_PMULL(features)	1
+#else
+#  define HAVE_PMULL(features)	((features) & ARM_CPU_FEATURE_PMULL)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
+	CPU_IS_LITTLE_ENDIAN() /* untested on big endian */
+#  define HAVE_PMULL_INTRIN	1
+   /* Work around MSVC's vmull_p64() taking poly64x1_t instead of poly64_t */
+#  ifdef _MSC_VER
+#    define compat_vmull_p64(a, b)  vmull_p64(vcreate_p64(a), vcreate_p64(b))
+#  else
+#    define compat_vmull_p64(a, b)  vmull_p64((a), (b))
+#  endif
+#else
+#  define HAVE_PMULL_INTRIN	0
+#endif
 
-#define ARM_CPU_FEATURES_KNOWN		0x80000000
+/* CRC32 */
+#ifdef __ARM_FEATURE_CRC32
+#  define HAVE_CRC32(features)	1
+#else
+#  define HAVE_CRC32(features)	((features) & ARM_CPU_FEATURE_CRC32)
+#endif
+#if defined(ARCH_ARM64) && \
+	(defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
+#  define HAVE_CRC32_INTRIN	1
+#  if defined(__GNUC__) || defined(__clang__)
+#    include <arm_acle.h>
+#  endif
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the crc32 intrinsics when crc32 is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_CRC32)
+#    undef __crc32b
+#    define __crc32b(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32b %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32h
+#    define __crc32h(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32h %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32w
+#    define __crc32w(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32w %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32d
+#    define __crc32d(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32x %w0, %w1, %2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_CRC32_INTRIN	0
+#endif
 
-extern volatile u32 _cpu_features;
+/* SHA3 (needed for the eor3 instruction) */
+#ifdef __ARM_FEATURE_SHA3
+#  define HAVE_SHA3(features)	1
+#else
+#  define HAVE_SHA3(features)	((features) & ARM_CPU_FEATURE_SHA3)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(9, 1) /* r268049 */ || \
+	 CLANG_PREREQ(7, 0, 10010463) /* r338010 */)
+#  define HAVE_SHA3_INTRIN	1
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the sha3 intrinsics when sha3 is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_SHA3)
+#    undef veor3q_u8
+#    define veor3q_u8(a, b, c)					\
+	({ uint8x16_t res;					\
+	   __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b"	\
+		   : "=w" (res) : "w" (a), "w" (b), "w" (c));	\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_SHA3_INTRIN	0
+#endif
 
-void setup_cpu_features(void);
+/* dotprod */
+#ifdef __ARM_FEATURE_DOTPROD
+#  define HAVE_DOTPROD(features)	1
+#else
+#  define HAVE_DOTPROD(features)	((features) & ARM_CPU_FEATURE_DOTPROD)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
+#  define HAVE_DOTPROD_INTRIN	1
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the dotprod intrinsics when dotprod is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_DOTPROD)
+#    undef vdotq_u32
+#    define vdotq_u32(a, b, c)					\
+	({ uint32x4_t res = (a);				\
+	   __asm__("udot %0.4s, %1.16b, %2.16b"			\
+		   : "+w" (res) : "w" (b), "w" (c));		\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_DOTPROD_INTRIN	0
+#endif
 
-static inline u32 get_cpu_features(void)
-{
-	if (_cpu_features == 0)
-		setup_cpu_features();
-	return _cpu_features;
-}
-
-#endif /* ARM_CPU_FEATURES_ENABLED */
+#endif /* ARCH_ARM32 || ARCH_ARM64 */
 
 #endif /* LIB_ARM_CPU_FEATURES_H */

lib/arm/crc32_impl.h

@@ -1,8 +1,7 @@
 /*
- * arm/crc32_impl.h
+ * arm/crc32_impl.h - ARM implementations of the gzip CRC-32 algorithm
  *
- * Copyright 2017 Jun He <jun.he@linaro.org>
- * Copyright 2018 Eric Biggers
+ * Copyright 2022 Eric Biggers
  *
  * Permission is hereby granted, free of charge, to any person
  * obtaining a copy of this software and associated documentation
@@ -31,221 +30,565 @@
 
 #include "cpu_features.h"
 
-/* Implementation using ARM CRC32 instructions */
-#undef DISPATCH_ARM
-#if !defined(DEFAULT_IMPL) && \
-    (defined(__ARM_FEATURE_CRC32) || \
-     (ARM_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS))
-#  ifdef __ARM_FEATURE_CRC32
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	crc32_arm
+/*
+ * crc32_arm_crc() - implementation using crc32 instructions (only)
+ *
+ * In general this implementation is straightforward.  However, naive use of the
+ * crc32 instructions is serial: one of the two inputs to each crc32 instruction
+ * is the output of the previous one.  To take advantage of CPUs that can
+ * execute multiple crc32 instructions in parallel, when possible we interleave
+ * the checksumming of several adjacent chunks, then combine their CRCs.
+ *
+ * However, without pmull, combining CRCs is fairly slow.  So in this pmull-less
+ * version, we only use a large chunk length, and thus we only do chunked
+ * processing if there is a lot of data to checksum.  This also means that a
+ * variable chunk length wouldn't help much, so we just support a fixed length.
+ */
+#if HAVE_CRC32_INTRIN
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("crc")
 #  else
-#    ifdef __arm__
-#      ifdef __clang__
-#        define ATTRIBUTES	__attribute__((target("armv8-a,crc")))
-#      else
-#        define ATTRIBUTES	__attribute__((target("arch=armv8-a+crc")))
-#      endif
-#    else
-#      ifdef __clang__
-#        define ATTRIBUTES	__attribute__((target("crc")))
-#      else
-#        define ATTRIBUTES	__attribute__((target("+crc")))
-#      endif
-#    endif
-#    define DISPATCH		1
-#    define DISPATCH_ARM	1
+#    define ATTRIBUTES	_target_attribute("+crc")
 #  endif
 
 /*
- * gcc's (as of 10.1) version of arm_acle.h for arm32, and clang's (as of
- * 10.0.1) version of arm_acle.h for both arm32 and arm64, have a bug where they
- * only define the CRC32 functions like __crc32b() when __ARM_FEATURE_CRC32 is
- * defined.  That prevents them from being used via __attribute__((target)) when
- * the main target doesn't have CRC32 support enabled.  The actual built-ins
- * like __builtin_arm_crc32b() are available and work, however; it's just the
- * wrappers in arm_acle.h like __crc32b() that erroneously don't get defined.
- * Work around this by manually defining __ARM_FEATURE_CRC32.
+ * Combine the CRCs for 4 adjacent chunks of length L = CRC32_FIXED_CHUNK_LEN
+ * bytes each by computing:
+ *
+ *	[ crc0*x^(3*8*L) + crc1*x^(2*8*L) + crc2*x^(1*8*L) + crc3 ] mod G(x)
+ *
+ * This has been optimized in several ways:
+ *
+ *    - The needed multipliers (x to some power, reduced mod G(x)) were
+ *	precomputed.
+ *
+ *    - The 3 multiplications are interleaved.
+ *
+ *    - The reduction mod G(x) is delayed to the end and done using __crc32d.
+ *	Note that the use of __crc32d introduces an extra factor of x^32.  To
+ *	cancel that out along with the extra factor of x^1 that gets introduced
+ *	because of how the 63-bit products are aligned in their 64-bit integers,
+ *	the multipliers are actually x^(j*8*L - 33) instead of x^(j*8*L).
  */
-#ifndef __ARM_FEATURE_CRC32
-#  define __ARM_FEATURE_CRC32	1
-#endif
-#include <arm_acle.h>
-
-static u32 ATTRIBUTES
-crc32_arm(u32 remainder, const u8 *p, size_t size)
+static forceinline ATTRIBUTES u32
+combine_crcs_slow(u32 crc0, u32 crc1, u32 crc2, u32 crc3)
 {
-	while (size != 0 && (uintptr_t)p & 7) {
-		remainder = __crc32b(remainder, *p++);
-		size--;
-	}
+	u64 res0 = 0, res1 = 0, res2 = 0;
+	int i;
 
-	while (size >= 64) {
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 0)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 1)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 2)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 3)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 4)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 5)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 6)));
-		remainder = __crc32d(remainder, le64_bswap(*((u64 *)p + 7)));
-		p += 64;
-		size -= 64;
+	/* Multiply crc{0,1,2} by CRC32_FIXED_CHUNK_MULT_{3,2,1}. */
+	for (i = 0; i < 32; i++) {
+		if (CRC32_FIXED_CHUNK_MULT_3 & (1U << i))
+			res0 ^= (u64)crc0 << i;
+		if (CRC32_FIXED_CHUNK_MULT_2 & (1U << i))
+			res1 ^= (u64)crc1 << i;
+		if (CRC32_FIXED_CHUNK_MULT_1 & (1U << i))
+			res2 ^= (u64)crc2 << i;
 	}
+	/* Add the different parts and reduce mod G(x). */
+	return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
+}
 
-	while (size >= 8) {
-		remainder = __crc32d(remainder, le64_bswap(*(u64 *)p));
+#define crc32_arm_crc	crc32_arm_crc
+static ATTRIBUTES u32
+crc32_arm_crc(u32 crc, const u8 *p, size_t len)
+{
+	if (len >= 64) {
+		const size_t align = -(uintptr_t)p & 7;
+
+		/* Align p to the next 8-byte boundary. */
+		if (align) {
+			if (align & 1)
+				crc = __crc32b(crc, *p++);
+			if (align & 2) {
+				crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+				p += 2;
+			}
+			if (align & 4) {
+				crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+				p += 4;
+			}
+			len -= align;
+		}
+		/*
+		 * Interleave the processing of multiple adjacent data chunks to
+		 * take advantage of instruction-level parallelism.
+		 *
+		 * Some CPUs don't prefetch the data if it's being fetched in
+		 * multiple interleaved streams, so do explicit prefetching.
+		 */
+		while (len >= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN) {
+			const u64 *wp0 = (const u64 *)p;
+			const u64 * const wp0_end =
+				(const u64 *)(p + CRC32_FIXED_CHUNK_LEN);
+			u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+			STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+			STATIC_ASSERT(CRC32_FIXED_CHUNK_LEN % (4 * 8) == 0);
+			do {
+				prefetchr(&wp0[64 + 0*CRC32_FIXED_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 1*CRC32_FIXED_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 2*CRC32_FIXED_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 3*CRC32_FIXED_CHUNK_LEN/8]);
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+				wp0++;
+			} while (wp0 != wp0_end);
+			crc = combine_crcs_slow(crc, crc1, crc2, crc3);
+			p += CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
+			len -= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
+		}
+		/*
+		 * Due to the large fixed chunk length used above, there might
+		 * still be a lot of data left.  So use a 64-byte loop here,
+		 * instead of a loop that is less unrolled.
+		 */
+		while (len >= 64) {
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 32)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 40)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 48)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 56)));
+			p += 64;
+			len -= 64;
+		}
+	}
+	if (len & 32) {
+		crc = __crc32d(crc, get_unaligned_le64(p + 0));
+		crc = __crc32d(crc, get_unaligned_le64(p + 8));
+		crc = __crc32d(crc, get_unaligned_le64(p + 16));
+		crc = __crc32d(crc, get_unaligned_le64(p + 24));
+		p += 32;
+	}
+	if (len & 16) {
+		crc = __crc32d(crc, get_unaligned_le64(p + 0));
+		crc = __crc32d(crc, get_unaligned_le64(p + 8));
+		p += 16;
+	}
+	if (len & 8) {
+		crc = __crc32d(crc, get_unaligned_le64(p));
 		p += 8;
-		size -= 8;
 	}
-
-	while (size != 0) {
-		remainder = __crc32b(remainder, *p++);
-		size--;
+	if (len & 4) {
+		crc = __crc32w(crc, get_unaligned_le32(p));
+		p += 4;
 	}
-
-	return remainder;
+	if (len & 2) {
+		crc = __crc32h(crc, get_unaligned_le16(p));
+		p += 2;
+	}
+	if (len & 1)
+		crc = __crc32b(crc, *p);
+	return crc;
 }
 #undef ATTRIBUTES
-#endif /* Implementation using ARM CRC32 instructions */
+#endif /* crc32_arm_crc() */
 
 /*
- * CRC-32 folding with ARM Crypto extension-PMULL
+ * crc32_arm_crc_pmullcombine() - implementation using crc32 instructions, plus
+ *	pmull instructions for CRC combining
  *
- * This works the same way as the x86 PCLMUL version.
- * See x86/crc32_pclmul_template.h for an explanation.
+ * This is similar to crc32_arm_crc(), but it enables the use of pmull
+ * (carryless multiplication) instructions for the steps where the CRCs of
+ * adjacent data chunks are combined.  As this greatly speeds up CRC
+ * combination, this implementation also differs from crc32_arm_crc() in that it
+ * uses a variable chunk length which can get fairly small.  The precomputed
+ * multipliers needed for the selected chunk length are loaded from a table.
+ *
+ * Note that pmull is used here only for combining the CRCs of separately
+ * checksummed chunks, not for folding the data itself.  See crc32_arm_pmull*()
+ * for implementations that use pmull for folding the data itself.
  */
-#undef DISPATCH_PMULL
-#if !defined(DEFAULT_IMPL) && \
-    (defined(__ARM_FEATURE_CRYPTO) ||	\
-     (ARM_CPU_FEATURES_ENABLED &&	\
-      COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS)) && \
-      /* not yet tested on big endian, probably needs changes to work there */ \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#  define FUNCNAME		crc32_pmull
-#  define FUNCNAME_ALIGNED	crc32_pmull_aligned
-#  ifdef __ARM_FEATURE_CRYPTO
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	crc32_pmull
+#if HAVE_CRC32_INTRIN && HAVE_PMULL_INTRIN
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("crc,aes")
 #  else
-#    ifdef __arm__
-#      define ATTRIBUTES	__attribute__((target("fpu=crypto-neon-fp-armv8")))
-#    else
-#      ifdef __clang__
-#        define ATTRIBUTES	__attribute__((target("crypto")))
-#      else
-#        define ATTRIBUTES	__attribute__((target("+crypto")))
-#      endif
-#    endif
-#    define DISPATCH		1
-#    define DISPATCH_PMULL	1
+#    define ATTRIBUTES	_target_attribute("+crc,+crypto")
 #  endif
 
-#include <arm_neon.h>
-
-static forceinline ATTRIBUTES uint8x16_t
-clmul_00(uint8x16_t a, uint8x16_t b)
+/* Do carryless multiplication of two 32-bit values. */
+static forceinline ATTRIBUTES u64
+clmul_u32(u32 a, u32 b)
 {
-	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
-				     (poly64_t)vget_low_u8(b));
-}
+	uint64x2_t res = vreinterpretq_u64_p128(
+				compat_vmull_p64((poly64_t)a, (poly64_t)b));
 
-static forceinline ATTRIBUTES uint8x16_t
-clmul_10(uint8x16_t a, uint8x16_t b)
-{
-	return (uint8x16_t)vmull_p64((poly64_t)vget_low_u8(a),
-				     (poly64_t)vget_high_u8(b));
-}
-
-static forceinline ATTRIBUTES uint8x16_t
-clmul_11(uint8x16_t a, uint8x16_t b)
-{
-	return (uint8x16_t)vmull_high_p64((poly64x2_t)a, (poly64x2_t)b);
-}
-
-static forceinline ATTRIBUTES uint8x16_t
-fold_128b(uint8x16_t dst, uint8x16_t src, uint8x16_t multipliers)
-{
-	return dst ^ clmul_00(src, multipliers) ^ clmul_11(src, multipliers);
+	return vgetq_lane_u64(res, 0);
 }
 
+/*
+ * Like combine_crcs_slow(), but uses vmull_p64 to do the multiplications more
+ * quickly, and supports a variable chunk length.  The chunk length is
+ * 'i * CRC32_MIN_VARIABLE_CHUNK_LEN'
+ * where 1 <= i < ARRAY_LEN(crc32_mults_for_chunklen).
+ */
 static forceinline ATTRIBUTES u32
-crc32_pmull_aligned(u32 remainder, const uint8x16_t *p, size_t nr_segs)
+combine_crcs_fast(u32 crc0, u32 crc1, u32 crc2, u32 crc3, size_t i)
 {
-	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
-	const uint8x16_t multipliers_4 =
-		(uint8x16_t)(uint64x2_t){ 0x8F352D95, 0x1D9513D7 };
-	const uint8x16_t multipliers_1 =
-		(uint8x16_t)(uint64x2_t){ 0xAE689191, 0xCCAA009E };
-	const uint8x16_t final_multiplier =
-		(uint8x16_t)(uint64x2_t){ 0xB8BC6765 };
-	const uint8x16_t mask32 = (uint8x16_t)(uint32x4_t){ 0xFFFFFFFF };
-	const uint8x16_t barrett_reduction_constants =
-			(uint8x16_t)(uint64x2_t){ 0x00000001F7011641,
-						  0x00000001DB710641 };
-	const uint8x16_t zeroes = (uint8x16_t){ 0 };
+	u64 res0 = clmul_u32(crc0, crc32_mults_for_chunklen[i][0]);
+	u64 res1 = clmul_u32(crc1, crc32_mults_for_chunklen[i][1]);
+	u64 res2 = clmul_u32(crc2, crc32_mults_for_chunklen[i][2]);
 
-	const uint8x16_t * const end = p + nr_segs;
-	const uint8x16_t * const end512 = p + (nr_segs & ~3);
-	uint8x16_t x0, x1, x2, x3;
+	return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
+}
 
-	x0 = *p++ ^ (uint8x16_t)(uint32x4_t){ remainder };
-	if (nr_segs >= 4) {
-		x1 = *p++;
-		x2 = *p++;
-		x3 = *p++;
+#define crc32_arm_crc_pmullcombine	crc32_arm_crc_pmullcombine
+static ATTRIBUTES u32
+crc32_arm_crc_pmullcombine(u32 crc, const u8 *p, size_t len)
+{
+	const size_t align = -(uintptr_t)p & 7;
 
-		/* Fold 512 bits at a time */
-		while (p != end512) {
-			x0 = fold_128b(*p++, x0, multipliers_4);
-			x1 = fold_128b(*p++, x1, multipliers_4);
-			x2 = fold_128b(*p++, x2, multipliers_4);
-			x3 = fold_128b(*p++, x3, multipliers_4);
+	if (len >= align + CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
+		/* Align p to the next 8-byte boundary. */
+		if (align) {
+			if (align & 1)
+				crc = __crc32b(crc, *p++);
+			if (align & 2) {
+				crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+				p += 2;
+			}
+			if (align & 4) {
+				crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+				p += 4;
+			}
+			len -= align;
+		}
+		/*
+		 * Handle CRC32_MAX_VARIABLE_CHUNK_LEN specially, so that better
+		 * code is generated for it.
+		 */
+		while (len >= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN) {
+			const u64 *wp0 = (const u64 *)p;
+			const u64 * const wp0_end =
+				(const u64 *)(p + CRC32_MAX_VARIABLE_CHUNK_LEN);
+			u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+			STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+			STATIC_ASSERT(CRC32_MAX_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
+			do {
+				prefetchr(&wp0[64 + 0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+				prefetchr(&wp0[64 + 3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				wp0++;
+				crc  = __crc32d(crc,  le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+				wp0++;
+			} while (wp0 != wp0_end);
+			crc = combine_crcs_fast(crc, crc1, crc2, crc3,
+						ARRAY_LEN(crc32_mults_for_chunklen) - 1);
+			p += CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
+			len -= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
+		}
+		/* Handle up to one variable-length chunk. */
+		if (len >= CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
+			const size_t i = len / (CRC32_NUM_CHUNKS *
+						CRC32_MIN_VARIABLE_CHUNK_LEN);
+			const size_t chunk_len =
+				i * CRC32_MIN_VARIABLE_CHUNK_LEN;
+			const u64 *wp0 = (const u64 *)(p + 0*chunk_len);
+			const u64 *wp1 = (const u64 *)(p + 1*chunk_len);
+			const u64 *wp2 = (const u64 *)(p + 2*chunk_len);
+			const u64 *wp3 = (const u64 *)(p + 3*chunk_len);
+			const u64 * const wp0_end = wp1;
+			u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+			STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+			STATIC_ASSERT(CRC32_MIN_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
+			do {
+				prefetchr(wp0 + 64);
+				prefetchr(wp1 + 64);
+				prefetchr(wp2 + 64);
+				prefetchr(wp3 + 64);
+				crc  = __crc32d(crc,  le64_bswap(*wp0++));
+				crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+				crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+				crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+				crc  = __crc32d(crc,  le64_bswap(*wp0++));
+				crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+				crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+				crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+				crc  = __crc32d(crc,  le64_bswap(*wp0++));
+				crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+				crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+				crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+				crc  = __crc32d(crc,  le64_bswap(*wp0++));
+				crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+				crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+				crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+			} while (wp0 != wp0_end);
+			crc = combine_crcs_fast(crc, crc1, crc2, crc3, i);
+			p += CRC32_NUM_CHUNKS * chunk_len;
+			len -= CRC32_NUM_CHUNKS * chunk_len;
 		}
 
-		/* Fold 512 bits => 128 bits */
-		x1 = fold_128b(x1, x0, multipliers_1);
-		x2 = fold_128b(x2, x1, multipliers_1);
-		x0 = fold_128b(x3, x2, multipliers_1);
+		while (len >= 32) {
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
+			crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
+			p += 32;
+			len -= 32;
+		}
+	} else {
+		while (len >= 32) {
+			crc = __crc32d(crc, get_unaligned_le64(p + 0));
+			crc = __crc32d(crc, get_unaligned_le64(p + 8));
+			crc = __crc32d(crc, get_unaligned_le64(p + 16));
+			crc = __crc32d(crc, get_unaligned_le64(p + 24));
+			p += 32;
+			len -= 32;
+		}
+	}
+	if (len & 16) {
+		crc = __crc32d(crc, get_unaligned_le64(p + 0));
+		crc = __crc32d(crc, get_unaligned_le64(p + 8));
+		p += 16;
+	}
+	if (len & 8) {
+		crc = __crc32d(crc, get_unaligned_le64(p));
+		p += 8;
+	}
+	if (len & 4) {
+		crc = __crc32w(crc, get_unaligned_le32(p));
+		p += 4;
+	}
+	if (len & 2) {
+		crc = __crc32h(crc, get_unaligned_le16(p));
+		p += 2;
+	}
+	if (len & 1)
+		crc = __crc32b(crc, *p);
+	return crc;
+}
+#undef ATTRIBUTES
+#endif /* crc32_arm_crc_pmullcombine() */
+
+/*
+ * crc32_arm_pmullx4() - implementation using "folding" with pmull instructions
+ *
+ * This implementation is intended for CPUs that support pmull instructions but
+ * not crc32 instructions.
+ */
+#if HAVE_PMULL_INTRIN
+#  define crc32_arm_pmullx4	crc32_arm_pmullx4
+#  define SUFFIX			 _pmullx4
+#  ifdef __clang__
+     /*
+      * This used to use "crypto", but that stopped working with clang 16.
+      * Now only "aes" works.  "aes" works with older versions too, so use
+      * that.  No "+" prefix; clang 15 and earlier doesn't accept that.
+      */
+#    define ATTRIBUTES	_target_attribute("aes")
+#  else
+     /*
+      * With gcc, only "+crypto" works.  Both the "+" prefix and the
+      * "crypto" (not "aes") are essential...
+      */
+#    define ATTRIBUTES	_target_attribute("+crypto")
+#  endif
+#  define ENABLE_EOR3		0
+#  include "crc32_pmull_helpers.h"
+
+static ATTRIBUTES u32
+crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
+{
+	static const u64 _aligned_attribute(16) mults[3][2] = {
+		{ CRC32_X159_MODG, CRC32_X95_MODG },  /* 1 vecs */
+		{ CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */
+		{ CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */
+	};
+	static const u64 _aligned_attribute(16) barrett_consts[3][2] = {
+		{ CRC32_X95_MODG, },
+		{ CRC32_BARRETT_CONSTANT_1, },
+		{ CRC32_BARRETT_CONSTANT_2, },
+	};
+	const poly64x2_t multipliers_1 = load_multipliers(mults[0]);
+	uint8x16_t v0, v1, v2, v3;
+
+	if (len < 64 + 15) {
+		if (len < 16)
+			return crc32_slice1(crc, p, len);
+		v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
+		p += 16;
+		len -= 16;
+		while (len >= 16) {
+			v0 = fold_vec(v0, vld1q_u8(p), multipliers_1);
+			p += 16;
+			len -= 16;
+		}
+	} else {
+		const poly64x2_t multipliers_4 = load_multipliers(mults[1]);
+		const poly64x2_t multipliers_2 = load_multipliers(mults[2]);
+		const size_t align = -(uintptr_t)p & 15;
+		const uint8x16_t *vp;
+
+		v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
+		p += 16;
+		/* Align p to the next 16-byte boundary. */
+		if (align) {
+			v0 = fold_partial_vec(v0, p, align, multipliers_1);
+			p += align;
+			len -= align;
+		}
+		vp = (const uint8x16_t *)p;
+		v1 = *vp++;
+		v2 = *vp++;
+		v3 = *vp++;
+		while (len >= 64 + 64) {
+			v0 = fold_vec(v0, *vp++, multipliers_4);
+			v1 = fold_vec(v1, *vp++, multipliers_4);
+			v2 = fold_vec(v2, *vp++, multipliers_4);
+			v3 = fold_vec(v3, *vp++, multipliers_4);
+			len -= 64;
+		}
+		v0 = fold_vec(v0, v2, multipliers_2);
+		v1 = fold_vec(v1, v3, multipliers_2);
+		if (len & 32) {
+			v0 = fold_vec(v0, *vp++, multipliers_2);
+			v1 = fold_vec(v1, *vp++, multipliers_2);
+		}
+		v0 = fold_vec(v0, v1, multipliers_1);
+		if (len & 16)
+			v0 = fold_vec(v0, *vp++, multipliers_1);
+		p = (const u8 *)vp;
+		len &= 15;
 	}
 
-	/* Fold 128 bits at a time */
-	while (p != end)
-		x0 = fold_128b(*p++, x0, multipliers_1);
+	/* Handle any remaining partial block now before reducing to 32 bits. */
+	if (len)
+		v0 = fold_partial_vec(v0, p, len, multipliers_1);
 
-	/* Fold 128 => 96 bits, implicitly appending 32 zeroes */
-	x0 = vextq_u8(x0, zeroes, 8) ^ clmul_10(x0, multipliers_1);
-
-	/* Fold 96 => 64 bits */
-	x0 = vextq_u8(x0, zeroes, 4) ^ clmul_00(x0 & mask32, final_multiplier);
-
-	/* Reduce 64 => 32 bits using Barrett reduction */
-	x1 = x0;
-	x0 = clmul_00(x0 & mask32, barrett_reduction_constants);
-	x0 = clmul_10(x0 & mask32, barrett_reduction_constants);
-	return vgetq_lane_u32((uint32x4_t)(x0 ^ x1), 1);
+	/* Reduce to 32 bits, following lib/x86/crc32_pclmul_template.h */
+	v0 = veorq_u8(clmul_low(v0, load_multipliers(barrett_consts[0])),
+		      vextq_u8(v0, vdupq_n_u8(0), 8));
+	v1 = clmul_low(v0, load_multipliers(barrett_consts[1]));
+	v1 = clmul_low(v1, load_multipliers(barrett_consts[2]));
+	v0 = veorq_u8(v0, v1);
+	return vgetq_lane_u32(vreinterpretq_u32_u8(v0), 2);
 }
-#define IMPL_ALIGNMENT		16
-#define IMPL_SEGMENT_SIZE	16
-#include "../crc32_vec_template.h"
-#endif /* PMULL implementation */
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef ENABLE_EOR3
+#endif /* crc32_arm_pmullx4() */
+
+/*
+ * crc32_arm_pmullx12_crc() - large-stride implementation using "folding" with
+ *	pmull instructions, where crc32 instructions are also available
+ *
+ * See crc32_pmull_wide.h for explanation.
+ */
+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN
+#  define crc32_arm_pmullx12_crc	crc32_arm_pmullx12_crc
+#  define SUFFIX				 _pmullx12_crc
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("aes,crc")
+#  else
+#    define ATTRIBUTES	_target_attribute("+crypto,+crc")
+#  endif
+#  define ENABLE_EOR3	0
+#  include "crc32_pmull_wide.h"
+#endif
+
+/*
+ * crc32_arm_pmullx12_crc_eor3()
+ *
+ * This like crc32_arm_pmullx12_crc(), but it adds the eor3 instruction (from
+ * the sha3 extension) for even better performance.
+ */
+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3)
+#  define crc32_arm_pmullx12_crc_eor3	crc32_arm_pmullx12_crc_eor3
+#  define SUFFIX				 _pmullx12_crc_eor3
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("aes,crc,sha3")
+   /*
+    * Both gcc and binutils originally considered sha3 to depend on
+    * arch=armv8.2-a or later.  This was fixed in gcc 13.2 by commit
+    * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+    * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+    * dependencies from features").  Unfortunately, always using arch=armv8.2-a
+    * causes build errors with some compiler options because it may reduce the
+    * arch rather than increase it.  Therefore we try to omit the arch whenever
+    * possible.  If gcc is 14 or later, then both gcc and binutils are probably
+    * fixed, so we omit the arch.  We also omit the arch if a feature that
+    * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
+    */
+#  elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+			  || defined(__ARM_FEATURE_DOTPROD)
+#    define ATTRIBUTES	_target_attribute("+crypto,+crc,+sha3")
+#  else
+#    define ATTRIBUTES	_target_attribute("arch=armv8.2-a+crypto+crc+sha3")
+#  endif
+#  define ENABLE_EOR3	1
+#  include "crc32_pmull_wide.h"
+#endif
 
-#ifdef DISPATCH
 static inline crc32_func_t
 arch_select_crc32_func(void)
 {
-	u32 features = get_cpu_features();
+	const u32 features MAYBE_UNUSED = get_arm_cpu_features();
 
-#ifdef DISPATCH_ARM
-	if (features & ARM_CPU_FEATURE_CRC32)
-		return crc32_arm;
+#ifdef crc32_arm_pmullx12_crc_eor3
+	if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
+	    HAVE_PMULL(features) && HAVE_CRC32(features) && HAVE_SHA3(features))
+		return crc32_arm_pmullx12_crc_eor3;
 #endif
-#ifdef DISPATCH_PMULL
-	if (features & ARM_CPU_FEATURE_PMULL)
-		return crc32_pmull;
+#ifdef crc32_arm_pmullx12_crc
+	if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
+	    HAVE_PMULL(features) && HAVE_CRC32(features))
+		return crc32_arm_pmullx12_crc;
+#endif
+#ifdef crc32_arm_crc_pmullcombine
+	if (HAVE_CRC32(features) && HAVE_PMULL(features))
+		return crc32_arm_crc_pmullcombine;
+#endif
+#ifdef crc32_arm_crc
+	if (HAVE_CRC32(features))
+		return crc32_arm_crc;
+#endif
+#ifdef crc32_arm_pmullx4
+	if (HAVE_PMULL(features))
+		return crc32_arm_pmullx4;
 #endif
 	return NULL;
 }
-#endif /* DISPATCH */
+#define arch_select_crc32_func	arch_select_crc32_func
 
 #endif /* LIB_ARM_CRC32_IMPL_H */

lib/arm/crc32_pmull_helpers.h

@@ -0,0 +1,156 @@
+/*
+ * arm/crc32_pmull_helpers.h - helper functions for CRC-32 folding with PMULL
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file is a "template" for instantiating helper functions for CRC folding
+ * with pmull instructions.  It accepts the following parameters:
+ *
+ * SUFFIX:
+ *	Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ *	Target function attributes to use.
+ * ENABLE_EOR3:
+ *	Use the eor3 instruction (from the sha3 extension).
+ */
+
+/* Create a vector with 'a' in the first 4 bytes, and the rest zeroed out. */
+#undef u32_to_bytevec
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(u32_to_bytevec)(u32 a)
+{
+	return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));
+}
+#define u32_to_bytevec	ADD_SUFFIX(u32_to_bytevec)
+
+/* Load two 64-bit values into a vector. */
+#undef load_multipliers
+static forceinline ATTRIBUTES poly64x2_t
+ADD_SUFFIX(load_multipliers)(const u64 p[2])
+{
+	return vreinterpretq_p64_u64(vld1q_u64(p));
+}
+#define load_multipliers	ADD_SUFFIX(load_multipliers)
+
+/* Do carryless multiplication of the low halves of two vectors. */
+#undef clmul_low
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)
+{
+	return vreinterpretq_u8_p128(
+		     compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),
+				      vgetq_lane_p64(b, 0)));
+}
+#define clmul_low	ADD_SUFFIX(clmul_low)
+
+/* Do carryless multiplication of the high halves of two vectors. */
+#undef clmul_high
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)
+{
+#ifdef __clang__
+	/*
+	 * Use inline asm to ensure that pmull2 is really used.  This works
+	 * around clang bug https://github.com/llvm/llvm-project/issues/52868.
+	 */
+	uint8x16_t res;
+
+	__asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));
+	return res;
+#else
+	return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));
+#endif
+}
+#define clmul_high	ADD_SUFFIX(clmul_high)
+
+#undef eor3
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)
+{
+#if ENABLE_EOR3
+	return veor3q_u8(a, b, c);
+#else
+	return veorq_u8(veorq_u8(a, b), c);
+#endif
+}
+#define eor3	ADD_SUFFIX(eor3)
+
+#undef fold_vec
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)
+{
+	uint8x16_t a = clmul_low(src, multipliers);
+	uint8x16_t b = clmul_high(src, multipliers);
+
+	return eor3(a, b, dst);
+}
+#define fold_vec	ADD_SUFFIX(fold_vec)
+
+/*
+ * Given v containing a 16-byte polynomial, and a pointer 'p' that points to the
+ * next '1 <= len <= 15' data bytes, rearrange the concatenation of v and the
+ * data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,
+ * respectively.  Then fold x0 into x1 and return the result.  Assumes that
+ * 'p + len - 16' is in-bounds.
+ */
+#undef fold_partial_vec
+static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t
+ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,
+			     poly64x2_t multipliers_1)
+{
+	/*
+	 * vqtbl1q_u8(v, shift_tab[len..len+15]) left shifts v by 16-len bytes.
+	 * vqtbl1q_u8(v, shift_tab[len+16..len+31]) right shifts v by len bytes.
+	 */
+	static const u8 shift_tab[48] = {
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	};
+	const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);
+	const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);
+	uint8x16_t x0, x1, bsl_mask;
+
+	/* x0 = v left-shifted by '16 - len' bytes */
+	x0 = vqtbl1q_u8(v, lshift);
+
+	/* Create a vector of '16 - len' 0x00 bytes, then 'len' 0xff bytes. */
+	bsl_mask = vreinterpretq_u8_s8(
+			vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));
+
+	/*
+	 * x1 = the last '16 - len' bytes from v (i.e. v right-shifted by 'len'
+	 * bytes) followed by the remaining data.
+	 */
+	x1 = vbslq_u8(bsl_mask /* 0 bits select from arg3, 1 bits from arg2 */,
+		      vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));
+
+	return fold_vec(x0, x1, multipliers_1);
+}
+#define fold_partial_vec	ADD_SUFFIX(fold_partial_vec)

lib/arm/crc32_pmull_wide.h

@@ -0,0 +1,226 @@
+/*
+ * arm/crc32_pmull_wide.h - gzip CRC-32 with PMULL (extra-wide version)
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file is a "template" for instantiating PMULL-based crc32_arm functions.
+ * The "parameters" are:
+ *
+ * SUFFIX:
+ *	Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ *	Target function attributes to use.
+ * ENABLE_EOR3:
+ *	Use the eor3 instruction (from the sha3 extension).
+ *
+ * This is the extra-wide version; it uses an unusually large stride length of
+ * 12, and it assumes that crc32 instructions are available too.  It's intended
+ * for powerful CPUs that support both pmull and crc32 instructions, but where
+ * throughput of pmull and xor (given enough instructions issued in parallel) is
+ * significantly higher than that of crc32, thus making the crc32 instructions
+ * (counterintuitively) not actually the fastest way to compute the CRC-32.  The
+ * Apple M1 processor is an example of such a CPU.
+ */
+
+#include "crc32_pmull_helpers.h"
+
+static ATTRIBUTES u32
+ADD_SUFFIX(crc32_arm)(u32 crc, const u8 *p, size_t len)
+{
+	uint8x16_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11;
+
+	if (len < 3 * 192) {
+		static const u64 _aligned_attribute(16) mults[3][2] = {
+			{ CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */
+			{ CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */
+			{ CRC32_X159_MODG, CRC32_X95_MODG },  /* 1 vecs */
+		};
+		poly64x2_t multipliers_4, multipliers_2, multipliers_1;
+
+		if (len < 64)
+			goto tail;
+		multipliers_4 = load_multipliers(mults[0]);
+		multipliers_2 = load_multipliers(mults[1]);
+		multipliers_1 = load_multipliers(mults[2]);
+		/*
+		 * Short length; don't bother aligning the pointer, and fold
+		 * 64 bytes (4 vectors) at a time, at most.
+		 */
+		v0 = veorq_u8(vld1q_u8(p + 0), u32_to_bytevec(crc));
+		v1 = vld1q_u8(p + 16);
+		v2 = vld1q_u8(p + 32);
+		v3 = vld1q_u8(p + 48);
+		p += 64;
+		len -= 64;
+		while (len >= 64) {
+			v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_4);
+			v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_4);
+			v2 = fold_vec(v2, vld1q_u8(p + 32), multipliers_4);
+			v3 = fold_vec(v3, vld1q_u8(p + 48), multipliers_4);
+			p += 64;
+			len -= 64;
+		}
+		v0 = fold_vec(v0, v2, multipliers_2);
+		v1 = fold_vec(v1, v3, multipliers_2);
+		if (len >= 32) {
+			v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_2);
+			v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_2);
+			p += 32;
+			len -= 32;
+		}
+		v0 = fold_vec(v0, v1, multipliers_1);
+	} else {
+		static const u64 _aligned_attribute(16) mults[4][2] = {
+			{ CRC32_X1567_MODG, CRC32_X1503_MODG }, /* 12 vecs */
+			{ CRC32_X799_MODG, CRC32_X735_MODG },   /* 6 vecs */
+			{ CRC32_X415_MODG, CRC32_X351_MODG },   /* 3 vecs */
+			{ CRC32_X159_MODG, CRC32_X95_MODG },    /* 1 vecs */
+		};
+		const poly64x2_t multipliers_12 = load_multipliers(mults[0]);
+		const poly64x2_t multipliers_6 = load_multipliers(mults[1]);
+		const poly64x2_t multipliers_3 = load_multipliers(mults[2]);
+		const poly64x2_t multipliers_1 = load_multipliers(mults[3]);
+		const size_t align = -(uintptr_t)p & 15;
+		const uint8x16_t *vp;
+
+		/* Align p to the next 16-byte boundary. */
+		if (align) {
+			if (align & 1)
+				crc = __crc32b(crc, *p++);
+			if (align & 2) {
+				crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+				p += 2;
+			}
+			if (align & 4) {
+				crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+				p += 4;
+			}
+			if (align & 8) {
+				crc = __crc32d(crc, le64_bswap(*(u64 *)p));
+				p += 8;
+			}
+			len -= align;
+		}
+		vp = (const uint8x16_t *)p;
+		v0 = veorq_u8(*vp++, u32_to_bytevec(crc));
+		v1 = *vp++;
+		v2 = *vp++;
+		v3 = *vp++;
+		v4 = *vp++;
+		v5 = *vp++;
+		v6 = *vp++;
+		v7 = *vp++;
+		v8 = *vp++;
+		v9 = *vp++;
+		v10 = *vp++;
+		v11 = *vp++;
+		len -= 192;
+		/* Fold 192 bytes (12 vectors) at a time. */
+		do {
+			v0 = fold_vec(v0, *vp++, multipliers_12);
+			v1 = fold_vec(v1, *vp++, multipliers_12);
+			v2 = fold_vec(v2, *vp++, multipliers_12);
+			v3 = fold_vec(v3, *vp++, multipliers_12);
+			v4 = fold_vec(v4, *vp++, multipliers_12);
+			v5 = fold_vec(v5, *vp++, multipliers_12);
+			v6 = fold_vec(v6, *vp++, multipliers_12);
+			v7 = fold_vec(v7, *vp++, multipliers_12);
+			v8 = fold_vec(v8, *vp++, multipliers_12);
+			v9 = fold_vec(v9, *vp++, multipliers_12);
+			v10 = fold_vec(v10, *vp++, multipliers_12);
+			v11 = fold_vec(v11, *vp++, multipliers_12);
+			len -= 192;
+		} while (len >= 192);
+
+		/*
+		 * Fewer than 192 bytes left.  Fold v0-v11 down to just v0,
+		 * while processing up to 144 more bytes.
+		 */
+		v0 = fold_vec(v0, v6, multipliers_6);
+		v1 = fold_vec(v1, v7, multipliers_6);
+		v2 = fold_vec(v2, v8, multipliers_6);
+		v3 = fold_vec(v3, v9, multipliers_6);
+		v4 = fold_vec(v4, v10, multipliers_6);
+		v5 = fold_vec(v5, v11, multipliers_6);
+		if (len >= 96) {
+			v0 = fold_vec(v0, *vp++, multipliers_6);
+			v1 = fold_vec(v1, *vp++, multipliers_6);
+			v2 = fold_vec(v2, *vp++, multipliers_6);
+			v3 = fold_vec(v3, *vp++, multipliers_6);
+			v4 = fold_vec(v4, *vp++, multipliers_6);
+			v5 = fold_vec(v5, *vp++, multipliers_6);
+			len -= 96;
+		}
+		v0 = fold_vec(v0, v3, multipliers_3);
+		v1 = fold_vec(v1, v4, multipliers_3);
+		v2 = fold_vec(v2, v5, multipliers_3);
+		if (len >= 48) {
+			v0 = fold_vec(v0, *vp++, multipliers_3);
+			v1 = fold_vec(v1, *vp++, multipliers_3);
+			v2 = fold_vec(v2, *vp++, multipliers_3);
+			len -= 48;
+		}
+		v0 = fold_vec(v0, v1, multipliers_1);
+		v0 = fold_vec(v0, v2, multipliers_1);
+		p = (const u8 *)vp;
+	}
+	/* Reduce 128 to 32 bits using crc32 instructions. */
+	crc = __crc32d(0, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 0));
+	crc = __crc32d(crc, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 1));
+tail:
+	/* Finish up the remainder using crc32 instructions. */
+	if (len & 32) {
+		crc = __crc32d(crc, get_unaligned_le64(p + 0));
+		crc = __crc32d(crc, get_unaligned_le64(p + 8));
+		crc = __crc32d(crc, get_unaligned_le64(p + 16));
+		crc = __crc32d(crc, get_unaligned_le64(p + 24));
+		p += 32;
+	}
+	if (len & 16) {
+		crc = __crc32d(crc, get_unaligned_le64(p + 0));
+		crc = __crc32d(crc, get_unaligned_le64(p + 8));
+		p += 16;
+	}
+	if (len & 8) {
+		crc = __crc32d(crc, get_unaligned_le64(p));
+		p += 8;
+	}
+	if (len & 4) {
+		crc = __crc32w(crc, get_unaligned_le32(p));
+		p += 4;
+	}
+	if (len & 2) {
+		crc = __crc32h(crc, get_unaligned_le16(p));
+		p += 2;
+	}
+	if (len & 1)
+		crc = __crc32b(crc, *p);
+	return crc;
+}
+
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef ENABLE_EOR3

lib/arm/matchfinder_impl.h

@@ -28,17 +28,14 @@
 #ifndef LIB_ARM_MATCHFINDER_IMPL_H
 #define LIB_ARM_MATCHFINDER_IMPL_H
 
-#ifdef __ARM_NEON
-#  include <arm_neon.h>
+#include "cpu_features.h"
+
+#if HAVE_NEON_NATIVE
 static forceinline void
 matchfinder_init_neon(mf_pos_t *data, size_t size)
 {
 	int16x8_t *p = (int16x8_t *)data;
-	int16x8_t v = (int16x8_t) {
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-		MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
-	};
+	int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);
 
 	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
 	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
@@ -59,12 +56,7 @@ static forceinline void
 matchfinder_rebase_neon(mf_pos_t *data, size_t size)
 {
 	int16x8_t *p = (int16x8_t *)data;
-	int16x8_t v = (int16x8_t) {
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-		(u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
-	};
+	int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);
 
 	STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
 	STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
@@ -81,6 +73,6 @@ matchfinder_rebase_neon(mf_pos_t *data, size_t size)
 }
 #define matchfinder_rebase matchfinder_rebase_neon
 
-#endif /* __ARM_NEON */
+#endif /* HAVE_NEON_NATIVE */
 
 #endif /* LIB_ARM_MATCHFINDER_IMPL_H */

lib/bt_matchfinder.h

@@ -85,7 +85,7 @@ struct lz_match {
 	u16 offset;
 };
 
-struct bt_matchfinder {
+struct MATCHFINDER_ALIGNED bt_matchfinder {
 
 	/* The hash table for finding length 3 matches  */
 	mf_pos_t hash3_tab[1UL << BT_MATCHFINDER_HASH3_ORDER][BT_MATCHFINDER_HASH3_WAYS];
@@ -98,8 +98,7 @@ struct bt_matchfinder {
 	 * children of the node for the sequence with position 'pos' are
 	 * 'child_tab[pos * 2]' and 'child_tab[pos * 2 + 1]', respectively.  */
 	mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE];
-
-} MATCHFINDER_ALIGNED;
+};
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void

lib/cpu_features_common.h

@@ -29,7 +29,12 @@
 #define LIB_CPU_FEATURES_COMMON_H
 
 #if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
-#  define _GNU_SOURCE 1 /* for strdup() and strtok_r() */
+   /* for strdup() and strtok_r() */
+#  undef _ANSI_SOURCE
+#  ifndef __APPLE__
+#    undef _GNU_SOURCE
+#    define _GNU_SOURCE
+#  endif
 #  include <stdio.h>
 #  include <stdlib.h>
 #  include <string.h>

lib/crc32.c

@@ -70,77 +70,77 @@
  * subtraction can be implemented as bitwise exclusive OR (since we are working
  * in GF(2)).  Here is an unoptimized implementation:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t size)
+ *	static u32 crc32_gzip(const u8 *p, size_t len)
  *	{
- *		u32 remainder = 0;
+ *		u32 crc = 0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < size * 8 + 32; i++) {
+ *		for (size_t i = 0; i < len * 8 + 32; i++) {
  *			int bit;
  *			u32 multiple;
  *
- *			if (i < size * 8)
- *				bit = (buffer[i / 8] >> (i % 8)) & 1;
+ *			if (i < len * 8)
+ *				bit = (p[i / 8] >> (i % 8)) & 1;
  *			else
  *				bit = 0; // one of the 32 appended 0 bits
  *
  *			if (i < 32) // the first 32 bits are inverted
  *				bit ^= 1;
  *
- *			if (remainder & 1)
+ *			if (crc & 1)
  *				multiple = divisor;
  *			else
  *				multiple = 0;
  *
- *			remainder >>= 1;
- *			remainder |= (u32)bit << 31;
- *			remainder ^= multiple;
+ *			crc >>= 1;
+ *			crc |= (u32)bit << 31;
+ *			crc ^= multiple;
  *		}
  *
- *		return ~remainder;
+ *		return ~crc;
  *	}
  *
- * In this implementation, the 32-bit integer 'remainder' maintains the
- * remainder of the currently processed portion of the message (with 32 zero
- * bits appended) when divided by the generator polynomial.  'remainder' is the
- * representation of R(x), and 'divisor' is the representation of G(x) excluding
- * the x^32 coefficient.  For each bit to process, we multiply R(x) by 'x^1',
- * then add 'x^0' if the new bit is a 1.  If this causes R(x) to gain a nonzero
- * x^32 term, then we subtract G(x) from R(x).
+ * In this implementation, the 32-bit integer 'crc' maintains the remainder of
+ * the currently processed portion of the message (with 32 zero bits appended)
+ * when divided by the generator polynomial.  'crc' is the representation of
+ * R(x), and 'divisor' is the representation of G(x) excluding the x^32
+ * coefficient.  For each bit to process, we multiply R(x) by 'x^1', then add
+ * 'x^0' if the new bit is a 1.  If this causes R(x) to gain a nonzero x^32
+ * term, then we subtract G(x) from R(x).
  *
  * We can speed this up by taking advantage of the fact that XOR is commutative
- * and associative, so the order in which we combine the inputs into 'remainder'
- * is unimportant.  And since each message bit we add doesn't affect the choice
- * of 'multiple' until 32 bits later, we need not actually add each message bit
+ * and associative, so the order in which we combine the inputs into 'crc' is
+ * unimportant.  And since each message bit we add doesn't affect the choice of
+ * 'multiple' until 32 bits later, we need not actually add each message bit
  * until that point:
  *
- *	static u32 crc32_gzip(const u8 *buffer, size_t size)
+ *	static u32 crc32_gzip(const u8 *p, size_t len)
  *	{
- *		u32 remainder = ~0;
+ *		u32 crc = ~0;
  *		const u32 divisor = 0xEDB88320;
  *
- *		for (size_t i = 0; i < size * 8; i++) {
+ *		for (size_t i = 0; i < len * 8; i++) {
  *			int bit;
  *			u32 multiple;
  *
- *			bit = (buffer[i / 8] >> (i % 8)) & 1;
- *			remainder ^= bit;
- *			if (remainder & 1)
+ *			bit = (p[i / 8] >> (i % 8)) & 1;
+ *			crc ^= bit;
+ *			if (crc & 1)
  *				multiple = divisor;
  *			else
  *				multiple = 0;
- *			remainder >>= 1;
- *			remainder ^= multiple;
+ *			crc >>= 1;
+ *			crc ^= multiple;
  *		}
  *
- *		return ~remainder;
+ *		return ~crc;
  *	}
  *
  * With the above implementation we get the effect of 32 appended 0 bits for
  * free; they never affect the choice of a divisor, nor would they change the
- * value of 'remainder' if they were to be actually XOR'ed in.  And by starting
- * with a remainder of all 1 bits, we get the effect of complementing the first
- * 32 message bits.
+ * value of 'crc' if they were to be actually XOR'ed in.  And by starting with a
+ * remainder of all 1 bits, we get the effect of complementing the first 32
+ * message bits.
  *
  * The next optimization is to process the input in multi-bit units.  Suppose
  * that we insert the next 'n' message bits into the remainder.  Then we get an
@@ -159,138 +159,86 @@
  * with 256 entries each.  Logically, we form a 64-bit intermediate remainder
  * and cancel out the high 32 bits in 8-bit chunks.  Bits 32-39 are cancelled
  * out by the CRC of those bits, whereas bits 40-47 are be cancelled out by the
- * CRC of those bits with 8 zero bits appended, and so on.  This method is
- * implemented in crc32_slice4(), defined below.
+ * CRC of those bits with 8 zero bits appended, and so on.
  *
  * In crc32_slice8(), this method is extended to 8 bytes at a time.  The
  * intermediate remainder (which we never actually store explicitly) is 96 bits.
  *
  * On CPUs that support fast carryless multiplication, CRCs can be computed even
- * more quickly via "folding".  See e.g. the x86 PCLMUL implementation.
+ * more quickly via "folding".  See e.g. the x86 PCLMUL implementations.
  */
 
 #include "lib_common.h"
-#include "libdeflate.h"
+#include "crc32_multipliers.h"
+#include "crc32_tables.h"
 
-typedef u32 (*crc32_func_t)(u32, const u8 *, size_t);
-
-/* Include architecture-specific implementations if available */
-#undef CRC32_SLICE1
-#undef CRC32_SLICE4
-#undef CRC32_SLICE8
-#undef DEFAULT_IMPL
-#undef DISPATCH
-#if defined(__arm__) || defined(__aarch64__)
-#  include "arm/crc32_impl.h"
-#elif defined(__i386__) || defined(__x86_64__)
-#  include "x86/crc32_impl.h"
-#endif
-
-/*
- * Define a generic implementation (crc32_slice8()) if needed.  crc32_slice1()
- * may also be needed as a fallback for architecture-specific implementations.
- */
-
-#ifndef DEFAULT_IMPL
-#  define CRC32_SLICE8	1
-#  define DEFAULT_IMPL	crc32_slice8
-#endif
-
-#if defined(CRC32_SLICE1) || defined(CRC32_SLICE4) || defined(CRC32_SLICE8)
-#include "crc32_table.h"
-static forceinline u32
-crc32_update_byte(u32 remainder, u8 next_byte)
+/* This is the default implementation.  It uses the slice-by-8 method. */
+static u32 MAYBE_UNUSED
+crc32_slice8(u32 crc, const u8 *p, size_t len)
 {
-	return (remainder >> 8) ^ crc32_table[(u8)remainder ^ next_byte];
-}
-#endif
-
-#ifdef CRC32_SLICE1
-static u32
-crc32_slice1(u32 remainder, const u8 *buffer, size_t size)
-{
-	size_t i;
-
-	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x100);
-
-	for (i = 0; i < size; i++)
-		remainder = crc32_update_byte(remainder, buffer[i]);
-	return remainder;
-}
-#endif /* CRC32_SLICE1 */
-
-#ifdef CRC32_SLICE4
-static u32
-crc32_slice4(u32 remainder, const u8 *buffer, size_t size)
-{
-	const u8 *p = buffer;
-	const u8 *end = buffer + size;
-	const u8 *end32;
-
-	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x400);
-
-	for (; ((uintptr_t)p & 3) && p != end; p++)
-		remainder = crc32_update_byte(remainder, *p);
-
-	end32 = p + ((end - p) & ~3);
-	for (; p != end32; p += 4) {
-		u32 v = le32_bswap(*(const u32 *)p);
-		remainder =
-		    crc32_table[0x300 + (u8)((remainder ^ v) >>  0)] ^
-		    crc32_table[0x200 + (u8)((remainder ^ v) >>  8)] ^
-		    crc32_table[0x100 + (u8)((remainder ^ v) >> 16)] ^
-		    crc32_table[0x000 + (u8)((remainder ^ v) >> 24)];
-	}
-
-	for (; p != end; p++)
-		remainder = crc32_update_byte(remainder, *p);
-
-	return remainder;
-}
-#endif /* CRC32_SLICE4 */
-
-#ifdef CRC32_SLICE8
-static u32
-crc32_slice8(u32 remainder, const u8 *buffer, size_t size)
-{
-	const u8 *p = buffer;
-	const u8 *end = buffer + size;
+	const u8 * const end = p + len;
 	const u8 *end64;
 
-	STATIC_ASSERT(ARRAY_LEN(crc32_table) >= 0x800);
-
 	for (; ((uintptr_t)p & 7) && p != end; p++)
-		remainder = crc32_update_byte(remainder, *p);
+		crc = (crc >> 8) ^ crc32_slice8_table[(u8)crc ^ *p];
 
 	end64 = p + ((end - p) & ~7);
 	for (; p != end64; p += 8) {
 		u32 v1 = le32_bswap(*(const u32 *)(p + 0));
 		u32 v2 = le32_bswap(*(const u32 *)(p + 4));
-		remainder =
-		    crc32_table[0x700 + (u8)((remainder ^ v1) >>  0)] ^
-		    crc32_table[0x600 + (u8)((remainder ^ v1) >>  8)] ^
-		    crc32_table[0x500 + (u8)((remainder ^ v1) >> 16)] ^
-		    crc32_table[0x400 + (u8)((remainder ^ v1) >> 24)] ^
-		    crc32_table[0x300 + (u8)(v2 >>  0)] ^
-		    crc32_table[0x200 + (u8)(v2 >>  8)] ^
-		    crc32_table[0x100 + (u8)(v2 >> 16)] ^
-		    crc32_table[0x000 + (u8)(v2 >> 24)];
+
+		crc = crc32_slice8_table[0x700 + (u8)((crc ^ v1) >> 0)] ^
+		      crc32_slice8_table[0x600 + (u8)((crc ^ v1) >> 8)] ^
+		      crc32_slice8_table[0x500 + (u8)((crc ^ v1) >> 16)] ^
+		      crc32_slice8_table[0x400 + (u8)((crc ^ v1) >> 24)] ^
+		      crc32_slice8_table[0x300 + (u8)(v2 >> 0)] ^
+		      crc32_slice8_table[0x200 + (u8)(v2 >> 8)] ^
+		      crc32_slice8_table[0x100 + (u8)(v2 >> 16)] ^
+		      crc32_slice8_table[0x000 + (u8)(v2 >> 24)];
 	}
 
 	for (; p != end; p++)
-		remainder = crc32_update_byte(remainder, *p);
+		crc = (crc >> 8) ^ crc32_slice8_table[(u8)crc ^ *p];
 
-	return remainder;
+	return crc;
 }
-#endif /* CRC32_SLICE8 */
 
-#ifdef DISPATCH
-static u32 dispatch(u32, const u8 *, size_t);
+/*
+ * This is a more lightweight generic implementation, which can be used as a
+ * subroutine by architecture-specific implementations to process small amounts
+ * of unaligned data at the beginning and/or end of the buffer.
+ */
+static forceinline u32 MAYBE_UNUSED
+crc32_slice1(u32 crc, const u8 *p, size_t len)
+{
+	size_t i;
 
-static volatile crc32_func_t crc32_impl = dispatch;
+	for (i = 0; i < len; i++)
+		crc = (crc >> 8) ^ crc32_slice1_table[(u8)crc ^ p[i]];
+	return crc;
+}
 
-/* Choose the fastest implementation at runtime */
-static u32 dispatch(u32 remainder, const u8 *buffer, size_t size)
+/* Include architecture-specific implementation(s) if available. */
+#undef DEFAULT_IMPL
+#undef arch_select_crc32_func
+typedef u32 (*crc32_func_t)(u32 crc, const u8 *p, size_t len);
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+#  include "arm/crc32_impl.h"
+#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
+#  include "x86/crc32_impl.h"
+#endif
+
+#ifndef DEFAULT_IMPL
+#  define DEFAULT_IMPL crc32_slice8
+#endif
+
+#ifdef arch_select_crc32_func
+static u32 dispatch_crc32(u32 crc, const u8 *p, size_t len);
+
+static volatile crc32_func_t crc32_impl = dispatch_crc32;
+
+/* Choose the best implementation at runtime. */
+static u32 dispatch_crc32(u32 crc, const u8 *p, size_t len)
 {
 	crc32_func_t f = arch_select_crc32_func();
 
@@ -298,16 +246,17 @@ static u32 dispatch(u32 remainder, const u8 *buffer, size_t size)
 		f = DEFAULT_IMPL;
 
 	crc32_impl = f;
-	return crc32_impl(remainder, buffer, size);
+	return f(crc, p, len);
 }
 #else
-#  define crc32_impl DEFAULT_IMPL /* only one implementation, use it */
+/* The best implementation is statically known, so call it directly. */
+#define crc32_impl DEFAULT_IMPL
 #endif
 
-LIBDEFLATEEXPORT u32 LIBDEFLATEAPI
-libdeflate_crc32(u32 remainder, const void *buffer, size_t size)
+LIBDEFLATEAPI u32
+libdeflate_crc32(u32 crc, const void *p, size_t len)
 {
-	if (buffer == NULL) /* return initial value */
+	if (p == NULL) /* Return initial value. */
 		return 0;
-	return ~crc32_impl(~remainder, buffer, size);
+	return ~crc32_impl(~crc, p, len);
 }

lib/crc32_multipliers.h

@@ -0,0 +1,375 @@
+/*
+ * crc32_multipliers.h - constants for CRC-32 folding
+ *
+ * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.
+ */
+
+#define CRC32_X159_MODG 0xae689191 /* x^159 mod G(x) */
+#define CRC32_X95_MODG 0xccaa009e /* x^95 mod G(x) */
+
+#define CRC32_X287_MODG 0xf1da05aa /* x^287 mod G(x) */
+#define CRC32_X223_MODG 0x81256527 /* x^223 mod G(x) */
+
+#define CRC32_X415_MODG 0x3db1ecdc /* x^415 mod G(x) */
+#define CRC32_X351_MODG 0xaf449247 /* x^351 mod G(x) */
+
+#define CRC32_X543_MODG 0x8f352d95 /* x^543 mod G(x) */
+#define CRC32_X479_MODG 0x1d9513d7 /* x^479 mod G(x) */
+
+#define CRC32_X671_MODG 0x1c279815 /* x^671 mod G(x) */
+#define CRC32_X607_MODG 0xae0b5394 /* x^607 mod G(x) */
+
+#define CRC32_X799_MODG 0xdf068dc2 /* x^799 mod G(x) */
+#define CRC32_X735_MODG 0x57c54819 /* x^735 mod G(x) */
+
+#define CRC32_X927_MODG 0x31f8303f /* x^927 mod G(x) */
+#define CRC32_X863_MODG 0x0cbec0ed /* x^863 mod G(x) */
+
+#define CRC32_X1055_MODG 0x33fff533 /* x^1055 mod G(x) */
+#define CRC32_X991_MODG 0x910eeec1 /* x^991 mod G(x) */
+
+#define CRC32_X1183_MODG 0x26b70c3d /* x^1183 mod G(x) */
+#define CRC32_X1119_MODG 0x3f41287a /* x^1119 mod G(x) */
+
+#define CRC32_X1311_MODG 0xe3543be0 /* x^1311 mod G(x) */
+#define CRC32_X1247_MODG 0x9026d5b1 /* x^1247 mod G(x) */
+
+#define CRC32_X1439_MODG 0x5a1bb05d /* x^1439 mod G(x) */
+#define CRC32_X1375_MODG 0xd1df2327 /* x^1375 mod G(x) */
+
+#define CRC32_X1567_MODG 0x596c8d81 /* x^1567 mod G(x) */
+#define CRC32_X1503_MODG 0xf5e48c85 /* x^1503 mod G(x) */
+
+#define CRC32_X1695_MODG 0x682bdd4f /* x^1695 mod G(x) */
+#define CRC32_X1631_MODG 0x3c656ced /* x^1631 mod G(x) */
+
+#define CRC32_X1823_MODG 0x4a28bd43 /* x^1823 mod G(x) */
+#define CRC32_X1759_MODG 0xfe807bbd /* x^1759 mod G(x) */
+
+#define CRC32_X1951_MODG 0x0077f00d /* x^1951 mod G(x) */
+#define CRC32_X1887_MODG 0x1f0c2cdd /* x^1887 mod G(x) */
+
+#define CRC32_X2079_MODG 0xce3371cb /* x^2079 mod G(x) */
+#define CRC32_X2015_MODG 0xe95c1271 /* x^2015 mod G(x) */
+
+#define CRC32_X2207_MODG 0xa749e894 /* x^2207 mod G(x) */
+#define CRC32_X2143_MODG 0xb918a347 /* x^2143 mod G(x) */
+
+#define CRC32_X2335_MODG 0x2c538639 /* x^2335 mod G(x) */
+#define CRC32_X2271_MODG 0x71d54a59 /* x^2271 mod G(x) */
+
+#define CRC32_X2463_MODG 0x32b0733c /* x^2463 mod G(x) */
+#define CRC32_X2399_MODG 0xff6f2fc2 /* x^2399 mod G(x) */
+
+#define CRC32_X2591_MODG 0x0e9bd5cc /* x^2591 mod G(x) */
+#define CRC32_X2527_MODG 0xcec97417 /* x^2527 mod G(x) */
+
+#define CRC32_X2719_MODG 0x76278617 /* x^2719 mod G(x) */
+#define CRC32_X2655_MODG 0x1c63267b /* x^2655 mod G(x) */
+
+#define CRC32_X2847_MODG 0xc51b93e3 /* x^2847 mod G(x) */
+#define CRC32_X2783_MODG 0xf183c71b /* x^2783 mod G(x) */
+
+#define CRC32_X2975_MODG 0x7eaed122 /* x^2975 mod G(x) */
+#define CRC32_X2911_MODG 0x9b9bdbd0 /* x^2911 mod G(x) */
+
+#define CRC32_X3103_MODG 0x2ce423f1 /* x^3103 mod G(x) */
+#define CRC32_X3039_MODG 0xd31343ea /* x^3039 mod G(x) */
+
+#define CRC32_X3231_MODG 0x8b8d8645 /* x^3231 mod G(x) */
+#define CRC32_X3167_MODG 0x4470ac44 /* x^3167 mod G(x) */
+
+#define CRC32_X3359_MODG 0x4b700aa8 /* x^3359 mod G(x) */
+#define CRC32_X3295_MODG 0xeea395c4 /* x^3295 mod G(x) */
+
+#define CRC32_X3487_MODG 0xeff5e99d /* x^3487 mod G(x) */
+#define CRC32_X3423_MODG 0xf9d9c7ee /* x^3423 mod G(x) */
+
+#define CRC32_X3615_MODG 0xad0d2bb2 /* x^3615 mod G(x) */
+#define CRC32_X3551_MODG 0xcd669a40 /* x^3551 mod G(x) */
+
+#define CRC32_X3743_MODG 0x9fb66bd3 /* x^3743 mod G(x) */
+#define CRC32_X3679_MODG 0x6d40f445 /* x^3679 mod G(x) */
+
+#define CRC32_X3871_MODG 0xc2dcc467 /* x^3871 mod G(x) */
+#define CRC32_X3807_MODG 0x9ee62949 /* x^3807 mod G(x) */
+
+#define CRC32_X3999_MODG 0x398e2ff2 /* x^3999 mod G(x) */
+#define CRC32_X3935_MODG 0x145575d5 /* x^3935 mod G(x) */
+
+#define CRC32_X4127_MODG 0x1072db28 /* x^4127 mod G(x) */
+#define CRC32_X4063_MODG 0x0c30f51d /* x^4063 mod G(x) */
+
+#define CRC32_BARRETT_CONSTANT_1 0xb4e5b025f7011641ULL /* floor(x^95 / G(x)) */
+#define CRC32_BARRETT_CONSTANT_2 0x00000001db710641ULL /* G(x) */
+
+#define CRC32_NUM_CHUNKS 4
+#define CRC32_MIN_VARIABLE_CHUNK_LEN 128UL
+#define CRC32_MAX_VARIABLE_CHUNK_LEN 16384UL
+
+/* Multipliers for implementations that use a variable chunk length */
+static const u32 crc32_mults_for_chunklen[][CRC32_NUM_CHUNKS - 1] MAYBE_UNUSED = {
+	{ 0 /* unused row */ },
+	/* chunk_len=128 */
+	{ 0xd31343ea /* x^3039 mod G(x) */, 0xe95c1271 /* x^2015 mod G(x) */, 0x910eeec1 /* x^991 mod G(x) */, },
+	/* chunk_len=256 */
+	{ 0x1d6708a0 /* x^6111 mod G(x) */, 0x0c30f51d /* x^4063 mod G(x) */, 0xe95c1271 /* x^2015 mod G(x) */, },
+	/* chunk_len=384 */
+	{ 0xdb3839f3 /* x^9183 mod G(x) */, 0x1d6708a0 /* x^6111 mod G(x) */, 0xd31343ea /* x^3039 mod G(x) */, },
+	/* chunk_len=512 */
+	{ 0x1753ab84 /* x^12255 mod G(x) */, 0xbbf2f6d6 /* x^8159 mod G(x) */, 0x0c30f51d /* x^4063 mod G(x) */, },
+	/* chunk_len=640 */
+	{ 0x3796455c /* x^15327 mod G(x) */, 0xb8e0e4a8 /* x^10207 mod G(x) */, 0xc352f6de /* x^5087 mod G(x) */, },
+	/* chunk_len=768 */
+	{ 0x3954de39 /* x^18399 mod G(x) */, 0x1753ab84 /* x^12255 mod G(x) */, 0x1d6708a0 /* x^6111 mod G(x) */, },
+	/* chunk_len=896 */
+	{ 0x632d78c5 /* x^21471 mod G(x) */, 0x3fc33de4 /* x^14303 mod G(x) */, 0x9a1b53c8 /* x^7135 mod G(x) */, },
+	/* chunk_len=1024 */
+	{ 0xa0decef3 /* x^24543 mod G(x) */, 0x7b4aa8b7 /* x^16351 mod G(x) */, 0xbbf2f6d6 /* x^8159 mod G(x) */, },
+	/* chunk_len=1152 */
+	{ 0xe9c09bb0 /* x^27615 mod G(x) */, 0x3954de39 /* x^18399 mod G(x) */, 0xdb3839f3 /* x^9183 mod G(x) */, },
+	/* chunk_len=1280 */
+	{ 0xd51917a4 /* x^30687 mod G(x) */, 0xcae68461 /* x^20447 mod G(x) */, 0xb8e0e4a8 /* x^10207 mod G(x) */, },
+	/* chunk_len=1408 */
+	{ 0x154a8a62 /* x^33759 mod G(x) */, 0x41e7589c /* x^22495 mod G(x) */, 0x3e9a43cd /* x^11231 mod G(x) */, },
+	/* chunk_len=1536 */
+	{ 0xf196555d /* x^36831 mod G(x) */, 0xa0decef3 /* x^24543 mod G(x) */, 0x1753ab84 /* x^12255 mod G(x) */, },
+	/* chunk_len=1664 */
+	{ 0x8eec2999 /* x^39903 mod G(x) */, 0xefb0a128 /* x^26591 mod G(x) */, 0x6044fbb0 /* x^13279 mod G(x) */, },
+	/* chunk_len=1792 */
+	{ 0x27892abf /* x^42975 mod G(x) */, 0x48d72bb1 /* x^28639 mod G(x) */, 0x3fc33de4 /* x^14303 mod G(x) */, },
+	/* chunk_len=1920 */
+	{ 0x77bc2419 /* x^46047 mod G(x) */, 0xd51917a4 /* x^30687 mod G(x) */, 0x3796455c /* x^15327 mod G(x) */, },
+	/* chunk_len=2048 */
+	{ 0xcea114a5 /* x^49119 mod G(x) */, 0x68c0a2c5 /* x^32735 mod G(x) */, 0x7b4aa8b7 /* x^16351 mod G(x) */, },
+	/* chunk_len=2176 */
+	{ 0xa1077e85 /* x^52191 mod G(x) */, 0x188cc628 /* x^34783 mod G(x) */, 0x0c21f835 /* x^17375 mod G(x) */, },
+	/* chunk_len=2304 */
+	{ 0xc5ed75e1 /* x^55263 mod G(x) */, 0xf196555d /* x^36831 mod G(x) */, 0x3954de39 /* x^18399 mod G(x) */, },
+	/* chunk_len=2432 */
+	{ 0xca4fba3f /* x^58335 mod G(x) */, 0x0acfa26f /* x^38879 mod G(x) */, 0x6cb21510 /* x^19423 mod G(x) */, },
+	/* chunk_len=2560 */
+	{ 0xcf5bcdc4 /* x^61407 mod G(x) */, 0x4fae7fc0 /* x^40927 mod G(x) */, 0xcae68461 /* x^20447 mod G(x) */, },
+	/* chunk_len=2688 */
+	{ 0xf36b9d16 /* x^64479 mod G(x) */, 0x27892abf /* x^42975 mod G(x) */, 0x632d78c5 /* x^21471 mod G(x) */, },
+	/* chunk_len=2816 */
+	{ 0xf76fd988 /* x^67551 mod G(x) */, 0xed5c39b1 /* x^45023 mod G(x) */, 0x41e7589c /* x^22495 mod G(x) */, },
+	/* chunk_len=2944 */
+	{ 0x6c45d92e /* x^70623 mod G(x) */, 0xff809fcd /* x^47071 mod G(x) */, 0x0c46baec /* x^23519 mod G(x) */, },
+	/* chunk_len=3072 */
+	{ 0x6116b82b /* x^73695 mod G(x) */, 0xcea114a5 /* x^49119 mod G(x) */, 0xa0decef3 /* x^24543 mod G(x) */, },
+	/* chunk_len=3200 */
+	{ 0x4d9899bb /* x^76767 mod G(x) */, 0x9f9d8d9c /* x^51167 mod G(x) */, 0x53deb236 /* x^25567 mod G(x) */, },
+	/* chunk_len=3328 */
+	{ 0x3e7c93b9 /* x^79839 mod G(x) */, 0x6666b805 /* x^53215 mod G(x) */, 0xefb0a128 /* x^26591 mod G(x) */, },
+	/* chunk_len=3456 */
+	{ 0x388b20ac /* x^82911 mod G(x) */, 0xc5ed75e1 /* x^55263 mod G(x) */, 0xe9c09bb0 /* x^27615 mod G(x) */, },
+	/* chunk_len=3584 */
+	{ 0x0956d953 /* x^85983 mod G(x) */, 0x97fbdb14 /* x^57311 mod G(x) */, 0x48d72bb1 /* x^28639 mod G(x) */, },
+	/* chunk_len=3712 */
+	{ 0x55cb4dfe /* x^89055 mod G(x) */, 0x1b37c832 /* x^59359 mod G(x) */, 0xc07331b3 /* x^29663 mod G(x) */, },
+	/* chunk_len=3840 */
+	{ 0x52222fea /* x^92127 mod G(x) */, 0xcf5bcdc4 /* x^61407 mod G(x) */, 0xd51917a4 /* x^30687 mod G(x) */, },
+	/* chunk_len=3968 */
+	{ 0x0603989b /* x^95199 mod G(x) */, 0xb03c8112 /* x^63455 mod G(x) */, 0x5e04b9a5 /* x^31711 mod G(x) */, },
+	/* chunk_len=4096 */
+	{ 0x4470c029 /* x^98271 mod G(x) */, 0x2339d155 /* x^65503 mod G(x) */, 0x68c0a2c5 /* x^32735 mod G(x) */, },
+	/* chunk_len=4224 */
+	{ 0xb6f35093 /* x^101343 mod G(x) */, 0xf76fd988 /* x^67551 mod G(x) */, 0x154a8a62 /* x^33759 mod G(x) */, },
+	/* chunk_len=4352 */
+	{ 0xc46805ba /* x^104415 mod G(x) */, 0x416f9449 /* x^69599 mod G(x) */, 0x188cc628 /* x^34783 mod G(x) */, },
+	/* chunk_len=4480 */
+	{ 0xc3876592 /* x^107487 mod G(x) */, 0x4b809189 /* x^71647 mod G(x) */, 0xc35cf6e7 /* x^35807 mod G(x) */, },
+	/* chunk_len=4608 */
+	{ 0x5b0c98b9 /* x^110559 mod G(x) */, 0x6116b82b /* x^73695 mod G(x) */, 0xf196555d /* x^36831 mod G(x) */, },
+	/* chunk_len=4736 */
+	{ 0x30d13e5f /* x^113631 mod G(x) */, 0x4c5a315a /* x^75743 mod G(x) */, 0x8c224466 /* x^37855 mod G(x) */, },
+	/* chunk_len=4864 */
+	{ 0x54afca53 /* x^116703 mod G(x) */, 0xbccfa2c1 /* x^77791 mod G(x) */, 0x0acfa26f /* x^38879 mod G(x) */, },
+	/* chunk_len=4992 */
+	{ 0x93102436 /* x^119775 mod G(x) */, 0x3e7c93b9 /* x^79839 mod G(x) */, 0x8eec2999 /* x^39903 mod G(x) */, },
+	/* chunk_len=5120 */
+	{ 0xbd2655a8 /* x^122847 mod G(x) */, 0x3e116c9d /* x^81887 mod G(x) */, 0x4fae7fc0 /* x^40927 mod G(x) */, },
+	/* chunk_len=5248 */
+	{ 0x70cd7f26 /* x^125919 mod G(x) */, 0x408e57f2 /* x^83935 mod G(x) */, 0x1691be45 /* x^41951 mod G(x) */, },
+	/* chunk_len=5376 */
+	{ 0x2d546c53 /* x^128991 mod G(x) */, 0x0956d953 /* x^85983 mod G(x) */, 0x27892abf /* x^42975 mod G(x) */, },
+	/* chunk_len=5504 */
+	{ 0xb53410a8 /* x^132063 mod G(x) */, 0x42ebf0ad /* x^88031 mod G(x) */, 0x161f3c12 /* x^43999 mod G(x) */, },
+	/* chunk_len=5632 */
+	{ 0x67a93f75 /* x^135135 mod G(x) */, 0xcf3233e4 /* x^90079 mod G(x) */, 0xed5c39b1 /* x^45023 mod G(x) */, },
+	/* chunk_len=5760 */
+	{ 0x9830ac33 /* x^138207 mod G(x) */, 0x52222fea /* x^92127 mod G(x) */, 0x77bc2419 /* x^46047 mod G(x) */, },
+	/* chunk_len=5888 */
+	{ 0xb0b6fc3e /* x^141279 mod G(x) */, 0x2fde73f8 /* x^94175 mod G(x) */, 0xff809fcd /* x^47071 mod G(x) */, },
+	/* chunk_len=6016 */
+	{ 0x84170f16 /* x^144351 mod G(x) */, 0xced90d99 /* x^96223 mod G(x) */, 0x30de0f98 /* x^48095 mod G(x) */, },
+	/* chunk_len=6144 */
+	{ 0xd7017a0c /* x^147423 mod G(x) */, 0x4470c029 /* x^98271 mod G(x) */, 0xcea114a5 /* x^49119 mod G(x) */, },
+	/* chunk_len=6272 */
+	{ 0xadb25de6 /* x^150495 mod G(x) */, 0x84f40beb /* x^100319 mod G(x) */, 0x2b7e0e1b /* x^50143 mod G(x) */, },
+	/* chunk_len=6400 */
+	{ 0x8282fddc /* x^153567 mod G(x) */, 0xec855937 /* x^102367 mod G(x) */, 0x9f9d8d9c /* x^51167 mod G(x) */, },
+	/* chunk_len=6528 */
+	{ 0x46362bee /* x^156639 mod G(x) */, 0xc46805ba /* x^104415 mod G(x) */, 0xa1077e85 /* x^52191 mod G(x) */, },
+	/* chunk_len=6656 */
+	{ 0xb9077a01 /* x^159711 mod G(x) */, 0xdf7a24ac /* x^106463 mod G(x) */, 0x6666b805 /* x^53215 mod G(x) */, },
+	/* chunk_len=6784 */
+	{ 0xf51d9bc6 /* x^162783 mod G(x) */, 0x2b52dc39 /* x^108511 mod G(x) */, 0x7e774cf6 /* x^54239 mod G(x) */, },
+	/* chunk_len=6912 */
+	{ 0x4ca19a29 /* x^165855 mod G(x) */, 0x5b0c98b9 /* x^110559 mod G(x) */, 0xc5ed75e1 /* x^55263 mod G(x) */, },
+	/* chunk_len=7040 */
+	{ 0xdc0fc3fc /* x^168927 mod G(x) */, 0xb939fcdf /* x^112607 mod G(x) */, 0x3678fed2 /* x^56287 mod G(x) */, },
+	/* chunk_len=7168 */
+	{ 0x63c3d167 /* x^171999 mod G(x) */, 0x70f9947d /* x^114655 mod G(x) */, 0x97fbdb14 /* x^57311 mod G(x) */, },
+	/* chunk_len=7296 */
+	{ 0x5851d254 /* x^175071 mod G(x) */, 0x54afca53 /* x^116703 mod G(x) */, 0xca4fba3f /* x^58335 mod G(x) */, },
+	/* chunk_len=7424 */
+	{ 0xfeacf2a1 /* x^178143 mod G(x) */, 0x7a3c0a6a /* x^118751 mod G(x) */, 0x1b37c832 /* x^59359 mod G(x) */, },
+	/* chunk_len=7552 */
+	{ 0x93b7edc8 /* x^181215 mod G(x) */, 0x1fea4d2a /* x^120799 mod G(x) */, 0x58fa96ee /* x^60383 mod G(x) */, },
+	/* chunk_len=7680 */
+	{ 0x5539e44a /* x^184287 mod G(x) */, 0xbd2655a8 /* x^122847 mod G(x) */, 0xcf5bcdc4 /* x^61407 mod G(x) */, },
+	/* chunk_len=7808 */
+	{ 0xde32a3d2 /* x^187359 mod G(x) */, 0x4ff61aa1 /* x^124895 mod G(x) */, 0x6a6a3694 /* x^62431 mod G(x) */, },
+	/* chunk_len=7936 */
+	{ 0xf0baeeb6 /* x^190431 mod G(x) */, 0x7ae2f6f4 /* x^126943 mod G(x) */, 0xb03c8112 /* x^63455 mod G(x) */, },
+	/* chunk_len=8064 */
+	{ 0xbe15887f /* x^193503 mod G(x) */, 0x2d546c53 /* x^128991 mod G(x) */, 0xf36b9d16 /* x^64479 mod G(x) */, },
+	/* chunk_len=8192 */
+	{ 0x64f34a05 /* x^196575 mod G(x) */, 0xe0ee5efe /* x^131039 mod G(x) */, 0x2339d155 /* x^65503 mod G(x) */, },
+	/* chunk_len=8320 */
+	{ 0x1b6d1aea /* x^199647 mod G(x) */, 0xfeafb67c /* x^133087 mod G(x) */, 0x4fb001a8 /* x^66527 mod G(x) */, },
+	/* chunk_len=8448 */
+	{ 0x82adb0b8 /* x^202719 mod G(x) */, 0x67a93f75 /* x^135135 mod G(x) */, 0xf76fd988 /* x^67551 mod G(x) */, },
+	/* chunk_len=8576 */
+	{ 0x694587c7 /* x^205791 mod G(x) */, 0x3b34408b /* x^137183 mod G(x) */, 0xeccb2978 /* x^68575 mod G(x) */, },
+	/* chunk_len=8704 */
+	{ 0xd2fc57c3 /* x^208863 mod G(x) */, 0x07fcf8c6 /* x^139231 mod G(x) */, 0x416f9449 /* x^69599 mod G(x) */, },
+	/* chunk_len=8832 */
+	{ 0x9dd6837c /* x^211935 mod G(x) */, 0xb0b6fc3e /* x^141279 mod G(x) */, 0x6c45d92e /* x^70623 mod G(x) */, },
+	/* chunk_len=8960 */
+	{ 0x3a9d1f97 /* x^215007 mod G(x) */, 0xefd033b2 /* x^143327 mod G(x) */, 0x4b809189 /* x^71647 mod G(x) */, },
+	/* chunk_len=9088 */
+	{ 0x1eee1d2a /* x^218079 mod G(x) */, 0xf2a6e46e /* x^145375 mod G(x) */, 0x55b4c814 /* x^72671 mod G(x) */, },
+	/* chunk_len=9216 */
+	{ 0xb57c7728 /* x^221151 mod G(x) */, 0xd7017a0c /* x^147423 mod G(x) */, 0x6116b82b /* x^73695 mod G(x) */, },
+	/* chunk_len=9344 */
+	{ 0xf2fc5d61 /* x^224223 mod G(x) */, 0x242aac86 /* x^149471 mod G(x) */, 0x05245cf0 /* x^74719 mod G(x) */, },
+	/* chunk_len=9472 */
+	{ 0x26387824 /* x^227295 mod G(x) */, 0xc15c4ca5 /* x^151519 mod G(x) */, 0x4c5a315a /* x^75743 mod G(x) */, },
+	/* chunk_len=9600 */
+	{ 0x8c151e77 /* x^230367 mod G(x) */, 0x8282fddc /* x^153567 mod G(x) */, 0x4d9899bb /* x^76767 mod G(x) */, },
+	/* chunk_len=9728 */
+	{ 0x8ea1f680 /* x^233439 mod G(x) */, 0xf5ff6cdd /* x^155615 mod G(x) */, 0xbccfa2c1 /* x^77791 mod G(x) */, },
+	/* chunk_len=9856 */
+	{ 0xe8cf3d2a /* x^236511 mod G(x) */, 0x338b1fb1 /* x^157663 mod G(x) */, 0xeda61f70 /* x^78815 mod G(x) */, },
+	/* chunk_len=9984 */
+	{ 0x21f15b59 /* x^239583 mod G(x) */, 0xb9077a01 /* x^159711 mod G(x) */, 0x3e7c93b9 /* x^79839 mod G(x) */, },
+	/* chunk_len=10112 */
+	{ 0x6f68d64a /* x^242655 mod G(x) */, 0x901b0161 /* x^161759 mod G(x) */, 0xb9fd3537 /* x^80863 mod G(x) */, },
+	/* chunk_len=10240 */
+	{ 0x71b74d95 /* x^245727 mod G(x) */, 0xf5ddd5ad /* x^163807 mod G(x) */, 0x3e116c9d /* x^81887 mod G(x) */, },
+	/* chunk_len=10368 */
+	{ 0x4c2e7261 /* x^248799 mod G(x) */, 0x4ca19a29 /* x^165855 mod G(x) */, 0x388b20ac /* x^82911 mod G(x) */, },
+	/* chunk_len=10496 */
+	{ 0x8a2d38e8 /* x^251871 mod G(x) */, 0xd27ee0a1 /* x^167903 mod G(x) */, 0x408e57f2 /* x^83935 mod G(x) */, },
+	/* chunk_len=10624 */
+	{ 0x7e58ca17 /* x^254943 mod G(x) */, 0x69dfedd2 /* x^169951 mod G(x) */, 0x3a76805e /* x^84959 mod G(x) */, },
+	/* chunk_len=10752 */
+	{ 0xf997967f /* x^258015 mod G(x) */, 0x63c3d167 /* x^171999 mod G(x) */, 0x0956d953 /* x^85983 mod G(x) */, },
+	/* chunk_len=10880 */
+	{ 0x48215963 /* x^261087 mod G(x) */, 0x71e1dfe0 /* x^174047 mod G(x) */, 0x42a6d410 /* x^87007 mod G(x) */, },
+	/* chunk_len=11008 */
+	{ 0xa704b94c /* x^264159 mod G(x) */, 0x679f198a /* x^176095 mod G(x) */, 0x42ebf0ad /* x^88031 mod G(x) */, },
+	/* chunk_len=11136 */
+	{ 0x1d699056 /* x^267231 mod G(x) */, 0xfeacf2a1 /* x^178143 mod G(x) */, 0x55cb4dfe /* x^89055 mod G(x) */, },
+	/* chunk_len=11264 */
+	{ 0x6800bcc5 /* x^270303 mod G(x) */, 0x16024f15 /* x^180191 mod G(x) */, 0xcf3233e4 /* x^90079 mod G(x) */, },
+	/* chunk_len=11392 */
+	{ 0x2d48e4ca /* x^273375 mod G(x) */, 0xbe61582f /* x^182239 mod G(x) */, 0x46026283 /* x^91103 mod G(x) */, },
+	/* chunk_len=11520 */
+	{ 0x4c4c2b55 /* x^276447 mod G(x) */, 0x5539e44a /* x^184287 mod G(x) */, 0x52222fea /* x^92127 mod G(x) */, },
+	/* chunk_len=11648 */
+	{ 0xd8ce94cb /* x^279519 mod G(x) */, 0xbc613c26 /* x^186335 mod G(x) */, 0x33776b4b /* x^93151 mod G(x) */, },
+	/* chunk_len=11776 */
+	{ 0xd0b5a02b /* x^282591 mod G(x) */, 0x490d3cc6 /* x^188383 mod G(x) */, 0x2fde73f8 /* x^94175 mod G(x) */, },
+	/* chunk_len=11904 */
+	{ 0xa223f7ec /* x^285663 mod G(x) */, 0xf0baeeb6 /* x^190431 mod G(x) */, 0x0603989b /* x^95199 mod G(x) */, },
+	/* chunk_len=12032 */
+	{ 0x58de337a /* x^288735 mod G(x) */, 0x3bf3d597 /* x^192479 mod G(x) */, 0xced90d99 /* x^96223 mod G(x) */, },
+	/* chunk_len=12160 */
+	{ 0x37f5d8f4 /* x^291807 mod G(x) */, 0x4d5b699b /* x^194527 mod G(x) */, 0xd7262e5f /* x^97247 mod G(x) */, },
+	/* chunk_len=12288 */
+	{ 0xfa8a435d /* x^294879 mod G(x) */, 0x64f34a05 /* x^196575 mod G(x) */, 0x4470c029 /* x^98271 mod G(x) */, },
+	/* chunk_len=12416 */
+	{ 0x238709fe /* x^297951 mod G(x) */, 0x52e7458f /* x^198623 mod G(x) */, 0x9a174cd3 /* x^99295 mod G(x) */, },
+	/* chunk_len=12544 */
+	{ 0x9e1ba6f5 /* x^301023 mod G(x) */, 0xef0272f7 /* x^200671 mod G(x) */, 0x84f40beb /* x^100319 mod G(x) */, },
+	/* chunk_len=12672 */
+	{ 0xcd8b57fa /* x^304095 mod G(x) */, 0x82adb0b8 /* x^202719 mod G(x) */, 0xb6f35093 /* x^101343 mod G(x) */, },
+	/* chunk_len=12800 */
+	{ 0x0aed142f /* x^307167 mod G(x) */, 0xb1650290 /* x^204767 mod G(x) */, 0xec855937 /* x^102367 mod G(x) */, },
+	/* chunk_len=12928 */
+	{ 0xd1f064db /* x^310239 mod G(x) */, 0x6e7340d3 /* x^206815 mod G(x) */, 0x5c28cb52 /* x^103391 mod G(x) */, },
+	/* chunk_len=13056 */
+	{ 0x464ac895 /* x^313311 mod G(x) */, 0xd2fc57c3 /* x^208863 mod G(x) */, 0xc46805ba /* x^104415 mod G(x) */, },
+	/* chunk_len=13184 */
+	{ 0xa0e6beea /* x^316383 mod G(x) */, 0xcfeec3d0 /* x^210911 mod G(x) */, 0x0225d214 /* x^105439 mod G(x) */, },
+	/* chunk_len=13312 */
+	{ 0x78703ce0 /* x^319455 mod G(x) */, 0xc60f6075 /* x^212959 mod G(x) */, 0xdf7a24ac /* x^106463 mod G(x) */, },
+	/* chunk_len=13440 */
+	{ 0xfea48165 /* x^322527 mod G(x) */, 0x3a9d1f97 /* x^215007 mod G(x) */, 0xc3876592 /* x^107487 mod G(x) */, },
+	/* chunk_len=13568 */
+	{ 0xdb89b8db /* x^325599 mod G(x) */, 0xa6172211 /* x^217055 mod G(x) */, 0x2b52dc39 /* x^108511 mod G(x) */, },
+	/* chunk_len=13696 */
+	{ 0x7ca03731 /* x^328671 mod G(x) */, 0x1db42849 /* x^219103 mod G(x) */, 0xc5df246e /* x^109535 mod G(x) */, },
+	/* chunk_len=13824 */
+	{ 0x8801d0aa /* x^331743 mod G(x) */, 0xb57c7728 /* x^221151 mod G(x) */, 0x5b0c98b9 /* x^110559 mod G(x) */, },
+	/* chunk_len=13952 */
+	{ 0xf89cd7f0 /* x^334815 mod G(x) */, 0xcc396a0b /* x^223199 mod G(x) */, 0xdb799c51 /* x^111583 mod G(x) */, },
+	/* chunk_len=14080 */
+	{ 0x1611a808 /* x^337887 mod G(x) */, 0xaeae6105 /* x^225247 mod G(x) */, 0xb939fcdf /* x^112607 mod G(x) */, },
+	/* chunk_len=14208 */
+	{ 0xe3cdb888 /* x^340959 mod G(x) */, 0x26387824 /* x^227295 mod G(x) */, 0x30d13e5f /* x^113631 mod G(x) */, },
+	/* chunk_len=14336 */
+	{ 0x552a4cf6 /* x^344031 mod G(x) */, 0xee2d04bb /* x^229343 mod G(x) */, 0x70f9947d /* x^114655 mod G(x) */, },
+	/* chunk_len=14464 */
+	{ 0x85e248e9 /* x^347103 mod G(x) */, 0x0a79663f /* x^231391 mod G(x) */, 0x53339cf7 /* x^115679 mod G(x) */, },
+	/* chunk_len=14592 */
+	{ 0x1c61c3e9 /* x^350175 mod G(x) */, 0x8ea1f680 /* x^233439 mod G(x) */, 0x54afca53 /* x^116703 mod G(x) */, },
+	/* chunk_len=14720 */
+	{ 0xb14cfc2b /* x^353247 mod G(x) */, 0x2e073302 /* x^235487 mod G(x) */, 0x10897992 /* x^117727 mod G(x) */, },
+	/* chunk_len=14848 */
+	{ 0x6ec444cc /* x^356319 mod G(x) */, 0x9e819f13 /* x^237535 mod G(x) */, 0x7a3c0a6a /* x^118751 mod G(x) */, },
+	/* chunk_len=14976 */
+	{ 0xe2fa5f80 /* x^359391 mod G(x) */, 0x21f15b59 /* x^239583 mod G(x) */, 0x93102436 /* x^119775 mod G(x) */, },
+	/* chunk_len=15104 */
+	{ 0x6d33f4c6 /* x^362463 mod G(x) */, 0x31a27455 /* x^241631 mod G(x) */, 0x1fea4d2a /* x^120799 mod G(x) */, },
+	/* chunk_len=15232 */
+	{ 0xb6dec609 /* x^365535 mod G(x) */, 0x4d437056 /* x^243679 mod G(x) */, 0x42eb1e2a /* x^121823 mod G(x) */, },
+	/* chunk_len=15360 */
+	{ 0x1846c518 /* x^368607 mod G(x) */, 0x71b74d95 /* x^245727 mod G(x) */, 0xbd2655a8 /* x^122847 mod G(x) */, },
+	/* chunk_len=15488 */
+	{ 0x9f947f8a /* x^371679 mod G(x) */, 0x2b501619 /* x^247775 mod G(x) */, 0xa4924b0e /* x^123871 mod G(x) */, },
+	/* chunk_len=15616 */
+	{ 0xb7442f4d /* x^374751 mod G(x) */, 0xba30a5d8 /* x^249823 mod G(x) */, 0x4ff61aa1 /* x^124895 mod G(x) */, },
+	/* chunk_len=15744 */
+	{ 0xe2c93242 /* x^377823 mod G(x) */, 0x8a2d38e8 /* x^251871 mod G(x) */, 0x70cd7f26 /* x^125919 mod G(x) */, },
+	/* chunk_len=15872 */
+	{ 0xcd6863df /* x^380895 mod G(x) */, 0x78fd88dc /* x^253919 mod G(x) */, 0x7ae2f6f4 /* x^126943 mod G(x) */, },
+	/* chunk_len=16000 */
+	{ 0xd512001d /* x^383967 mod G(x) */, 0xe6612dff /* x^255967 mod G(x) */, 0x5c4d0ca9 /* x^127967 mod G(x) */, },
+	/* chunk_len=16128 */
+	{ 0x4e8d6b6c /* x^387039 mod G(x) */, 0xf997967f /* x^258015 mod G(x) */, 0x2d546c53 /* x^128991 mod G(x) */, },
+	/* chunk_len=16256 */
+	{ 0xfa653ba1 /* x^390111 mod G(x) */, 0xc99014d4 /* x^260063 mod G(x) */, 0xa0c9fd27 /* x^130015 mod G(x) */, },
+	/* chunk_len=16384 */
+	{ 0x49893408 /* x^393183 mod G(x) */, 0x29c2448b /* x^262111 mod G(x) */, 0xe0ee5efe /* x^131039 mod G(x) */, },
+};
+
+/* Multipliers for implementations that use a large fixed chunk length */
+#define CRC32_FIXED_CHUNK_LEN 32768UL
+#define CRC32_FIXED_CHUNK_MULT_1 0x29c2448b /* x^262111 mod G(x) */
+#define CRC32_FIXED_CHUNK_MULT_2 0x4b912f53 /* x^524255 mod G(x) */
+#define CRC32_FIXED_CHUNK_MULT_3 0x454c93be /* x^786399 mod G(x) */

lib/crc32_tables.h

@@ -1,12 +1,77 @@
 /*
- * crc32_table.h - data table to accelerate CRC-32 computation
+ * crc32_tables.h - data tables for CRC-32 computation
  *
- * THIS FILE WAS AUTOMATICALLY GENERATED BY gen_crc32_table.c.  DO NOT EDIT.
+ * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.
  */
 
-#include <stdint.h>
+static const u32 crc32_slice1_table[] MAYBE_UNUSED = {
+	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
+	0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
+	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
+	0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
+	0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
+	0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+	0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
+	0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
+	0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
+	0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
+	0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
+	0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
+	0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+	0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
+	0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
+	0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+	0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
+	0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
+	0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
+	0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
+	0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
+	0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
+	0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+	0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
+	0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
+	0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+	0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
+	0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
+	0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
+	0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
+	0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
+	0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
+	0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+	0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
+	0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
+	0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+	0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
+	0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
+	0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
+	0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
+	0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
+	0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
+};
 
-static const uint32_t crc32_table[] = {
+static const u32 crc32_slice8_table[] MAYBE_UNUSED = {
 	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
 	0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
 	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
@@ -71,7 +136,6 @@ static const uint32_t crc32_table[] = {
 	0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
 	0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
 	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
-#if defined(CRC32_SLICE4) || defined(CRC32_SLICE8)
 	0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3,
 	0x646cc504, 0x7d77f445, 0x565aa786, 0x4f4196c7,
 	0xc8d98a08, 0xd1c2bb49, 0xfaefe88a, 0xe3f4d9cb,
@@ -264,8 +328,6 @@ static const uint32_t crc32_table[] = {
 	0x090481f0, 0xb1b8e695, 0xa30d497b, 0x1bb12e1e,
 	0x43d23e48, 0xfb6e592d, 0xe9dbf6c3, 0x516791a6,
 	0xccb0a91f, 0x740cce7a, 0x66b96194, 0xde0506f1,
-#endif /* CRC32_SLICE4 || CRC32_SLICE8 */
-#if defined(CRC32_SLICE8)
 	0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0,
 	0xf580a6c0, 0xc8e08f70, 0x8f40f5a0, 0xb220dc10,
 	0x30704bc1, 0x0d106271, 0x4ab018a1, 0x77d03111,
@@ -522,5 +584,4 @@ static const uint32_t crc32_table[] = {
 	0x7b211ab0, 0xb78b1a2e, 0x39041dcd, 0xf5ae1d53,
 	0x2c8e0fff, 0xe0240f61, 0x6eab0882, 0xa201081c,
 	0xa8c40105, 0x646e019b, 0xeae10678, 0x264b06e6,
-#endif /* CRC32_SLICE8 */
 };

lib/crc32_vec_template.h

@@ -1,61 +0,0 @@
-/*
- * crc32_vec_template.h - template for vectorized CRC-32 implementations
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#define CRC32_SLICE1	1
-static u32 crc32_slice1(u32, const u8 *, size_t);
-
-/*
- * Template for vectorized CRC-32 implementations.
- *
- * Note: on unaligned ends of the buffer, we fall back to crc32_slice1() instead
- * of crc32_slice8() because only a few bytes need to be processed, so a smaller
- * table is preferable.
- */
-static u32 ATTRIBUTES
-FUNCNAME(u32 remainder, const u8 *p, size_t size)
-{
-	if ((uintptr_t)p % IMPL_ALIGNMENT) {
-		size_t n = MIN(size, -(uintptr_t)p % IMPL_ALIGNMENT);
-
-		remainder = crc32_slice1(remainder, p, n);
-		p += n;
-		size -= n;
-	}
-	if (size >= IMPL_SEGMENT_SIZE) {
-		remainder = FUNCNAME_ALIGNED(remainder, (const void *)p,
-					     size / IMPL_SEGMENT_SIZE);
-		p += size - (size % IMPL_SEGMENT_SIZE);
-		size %= IMPL_SEGMENT_SIZE;
-	}
-	return crc32_slice1(remainder, p, size);
-}
-
-#undef FUNCNAME
-#undef FUNCNAME_ALIGNED
-#undef ATTRIBUTES
-#undef IMPL_ALIGNMENT
-#undef IMPL_SEGMENT_SIZE

lib/decompress_template.h

@@ -31,7 +31,17 @@
  * target instruction sets.
  */
 
-static enum libdeflate_result ATTRIBUTES
+#ifndef ATTRIBUTES
+#  define ATTRIBUTES
+#endif
+#ifndef EXTRACT_VARBITS
+#  define EXTRACT_VARBITS(word, count)	((word) & BITMASK(count))
+#endif
+#ifndef EXTRACT_VARBITS8
+#  define EXTRACT_VARBITS8(word, count)	((word) & BITMASK((u8)(count)))
+#endif
+
+static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
 FUNCNAME(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
 	 void * restrict out, size_t out_nbytes_avail,
@@ -39,102 +49,141 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 {
 	u8 *out_next = out;
 	u8 * const out_end = out_next + out_nbytes_avail;
+	u8 * const out_fastloop_end =
+		out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
+
+	/* Input bitstream state; see deflate_decompress.c for documentation */
 	const u8 *in_next = in;
 	const u8 * const in_end = in_next + in_nbytes;
+	const u8 * const in_fastloop_end =
+		in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
 	bitbuf_t bitbuf = 0;
-	unsigned bitsleft = 0;
+	bitbuf_t saved_bitbuf;
+	u32 bitsleft = 0;
 	size_t overread_count = 0;
-	unsigned i;
-	unsigned is_final_block;
+
+	bool is_final_block;
 	unsigned block_type;
-	u16 len;
-	u16 nlen;
 	unsigned num_litlen_syms;
 	unsigned num_offset_syms;
-	u16 tmp16;
-	u32 tmp32;
+	bitbuf_t litlen_tablemask;
+	u32 entry;
 
 next_block:
-	/* Starting to read the next block.  */
+	/* Starting to read the next block */
 	;
 
-	STATIC_ASSERT(CAN_ENSURE(1 + 2 + 5 + 5 + 4));
-	ENSURE_BITS(1 + 2 + 5 + 5 + 4);
+	STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
+	REFILL_BITS();
 
-	/* BFINAL: 1 bit  */
-	is_final_block = POP_BITS(1);
+	/* BFINAL: 1 bit */
+	is_final_block = bitbuf & BITMASK(1);
 
-	/* BTYPE: 2 bits  */
-	block_type = POP_BITS(2);
+	/* BTYPE: 2 bits */
+	block_type = (bitbuf >> 1) & BITMASK(2);
 
 	if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
 
-		/* Dynamic Huffman block.  */
+		/* Dynamic Huffman block */
 
-		/* The order in which precode lengths are stored.  */
+		/* The order in which precode lengths are stored */
 		static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
 			16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
 		};
 
 		unsigned num_explicit_precode_lens;
+		unsigned i;
 
-		/* Read the codeword length counts.  */
+		/* Read the codeword length counts. */
 
-		STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == ((1 << 5) - 1) + 257);
-		num_litlen_syms = POP_BITS(5) + 257;
+		STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
+		num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
 
-		STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == ((1 << 5) - 1) + 1);
-		num_offset_syms = POP_BITS(5) + 1;
+		STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
+		num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
 
-		STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == ((1 << 4) - 1) + 4);
-		num_explicit_precode_lens = POP_BITS(4) + 4;
+		STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
+		num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
 
 		d->static_codes_loaded = false;
 
-		/* Read the precode codeword lengths.  */
+		/*
+		 * Read the precode codeword lengths.
+		 *
+		 * A 64-bit bitbuffer is just one bit too small to hold the
+		 * maximum number of precode lens, so to minimize branches we
+		 * merge one len with the previous fields.
+		 */
 		STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
-		for (i = 0; i < num_explicit_precode_lens; i++) {
-			ENSURE_BITS(3);
-			d->u.precode_lens[deflate_precode_lens_permutation[i]] = POP_BITS(3);
+		if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
+			d->u.precode_lens[deflate_precode_lens_permutation[0]] =
+				(bitbuf >> 17) & BITMASK(3);
+			bitbuf >>= 20;
+			bitsleft -= 20;
+			REFILL_BITS();
+			i = 1;
+			do {
+				d->u.precode_lens[deflate_precode_lens_permutation[i]] =
+					bitbuf & BITMASK(3);
+				bitbuf >>= 3;
+				bitsleft -= 3;
+			} while (++i < num_explicit_precode_lens);
+		} else {
+			bitbuf >>= 17;
+			bitsleft -= 17;
+			i = 0;
+			do {
+				if ((u8)bitsleft < 3)
+					REFILL_BITS();
+				d->u.precode_lens[deflate_precode_lens_permutation[i]] =
+					bitbuf & BITMASK(3);
+				bitbuf >>= 3;
+				bitsleft -= 3;
+			} while (++i < num_explicit_precode_lens);
 		}
-
 		for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
 			d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
 
-		/* Build the decode table for the precode.  */
+		/* Build the decode table for the precode. */
 		SAFETY_CHECK(build_precode_decode_table(d));
 
-		/* Expand the literal/length and offset codeword lengths.  */
-		for (i = 0; i < num_litlen_syms + num_offset_syms; ) {
-			u32 entry;
+		/* Decode the litlen and offset codeword lengths. */
+		i = 0;
+		do {
 			unsigned presym;
 			u8 rep_val;
 			unsigned rep_count;
 
-			ENSURE_BITS(DEFLATE_MAX_PRE_CODEWORD_LEN + 7);
+			if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
+				REFILL_BITS();
 
-			/* (The code below assumes that the precode decode table
-			 * does not have any subtables.)  */
+			/*
+			 * The code below assumes that the precode decode table
+			 * doesn't have any subtables.
+			 */
 			STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
 
-			/* Read the next precode symbol.  */
-			entry = d->u.l.precode_decode_table[BITS(DEFLATE_MAX_PRE_CODEWORD_LEN)];
-			REMOVE_BITS(entry & HUFFDEC_LENGTH_MASK);
-			presym = entry >> HUFFDEC_RESULT_SHIFT;
+			/* Decode the next precode symbol. */
+			entry = d->u.l.precode_decode_table[
+				bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
+			bitbuf >>= (u8)entry;
+			bitsleft -= entry; /* optimization: subtract full entry */
+			presym = entry >> 16;
 
 			if (presym < 16) {
-				/* Explicit codeword length  */
+				/* Explicit codeword length */
 				d->u.l.lens[i++] = presym;
 				continue;
 			}
 
-			/* Run-length encoded codeword lengths  */
+			/* Run-length encoded codeword lengths */
 
-			/* Note: we don't need verify that the repeat count
-			 * doesn't overflow the number of elements, since we
-			 * have enough extra spaces to allow for the worst-case
-			 * overflow (138 zeroes when only 1 length was
-			 * remaining).
+			/*
+			 * Note: we don't need to immediately verify that the
+			 * repeat count doesn't overflow the number of elements,
+			 * since we've sized the lens array to have enough extra
+			 * space to allow for the worst-case overrun (138 zeroes
+			 * when only 1 length was remaining).
 			 *
 			 * In the case of the small repeat counts (presyms 16
 			 * and 17), it is fastest to always write the maximum
@@ -149,11 +198,13 @@ next_block:
 			STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
 
 			if (presym == 16) {
-				/* Repeat the previous length 3 - 6 times  */
+				/* Repeat the previous length 3 - 6 times. */
 				SAFETY_CHECK(i != 0);
 				rep_val = d->u.l.lens[i - 1];
-				STATIC_ASSERT(3 + ((1 << 2) - 1) == 6);
-				rep_count = 3 + POP_BITS(2);
+				STATIC_ASSERT(3 + BITMASK(2) == 6);
+				rep_count = 3 + (bitbuf & BITMASK(2));
+				bitbuf >>= 2;
+				bitsleft -= 2;
 				d->u.l.lens[i + 0] = rep_val;
 				d->u.l.lens[i + 1] = rep_val;
 				d->u.l.lens[i + 2] = rep_val;
@@ -162,9 +213,11 @@ next_block:
 				d->u.l.lens[i + 5] = rep_val;
 				i += rep_count;
 			} else if (presym == 17) {
-				/* Repeat zero 3 - 10 times  */
-				STATIC_ASSERT(3 + ((1 << 3) - 1) == 10);
-				rep_count = 3 + POP_BITS(3);
+				/* Repeat zero 3 - 10 times. */
+				STATIC_ASSERT(3 + BITMASK(3) == 10);
+				rep_count = 3 + (bitbuf & BITMASK(3));
+				bitbuf >>= 3;
+				bitsleft -= 3;
 				d->u.l.lens[i + 0] = 0;
 				d->u.l.lens[i + 1] = 0;
 				d->u.l.lens[i + 2] = 0;
@@ -177,25 +230,48 @@ next_block:
 				d->u.l.lens[i + 9] = 0;
 				i += rep_count;
 			} else {
-				/* Repeat zero 11 - 138 times  */
-				STATIC_ASSERT(11 + ((1 << 7) - 1) == 138);
-				rep_count = 11 + POP_BITS(7);
+				/* Repeat zero 11 - 138 times. */
+				STATIC_ASSERT(11 + BITMASK(7) == 138);
+				rep_count = 11 + (bitbuf & BITMASK(7));
+				bitbuf >>= 7;
+				bitsleft -= 7;
 				memset(&d->u.l.lens[i], 0,
 				       rep_count * sizeof(d->u.l.lens[i]));
 				i += rep_count;
 			}
-		}
+		} while (i < num_litlen_syms + num_offset_syms);
+
+		/* Unnecessary, but check this for consistency with zlib. */
+		SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
+
 	} else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
+		u16 len, nlen;
 
-		/* Uncompressed block: copy 'len' bytes literally from the input
-		 * buffer to the output buffer.  */
+		/*
+		 * Uncompressed block: copy 'len' bytes literally from the input
+		 * buffer to the output buffer.
+		 */
 
-		ALIGN_INPUT();
+		bitsleft -= 3; /* for BTYPE and BFINAL */
+
+		/*
+		 * Align the bitstream to the next byte boundary.  This means
+		 * the next byte boundary as if we were reading a byte at a
+		 * time.  Therefore, we have to rewind 'in_next' by any bytes
+		 * that have been refilled but not actually consumed yet (not
+		 * counting overread bytes, which don't increment 'in_next').
+		 */
+		bitsleft = (u8)bitsleft;
+		SAFETY_CHECK(overread_count <= (bitsleft >> 3));
+		in_next -= (bitsleft >> 3) - overread_count;
+		overread_count = 0;
+		bitbuf = 0;
+		bitsleft = 0;
 
 		SAFETY_CHECK(in_end - in_next >= 4);
-
-		len = READ_U16();
-		nlen = READ_U16();
+		len = get_unaligned_le16(in_next);
+		nlen = get_unaligned_le16(in_next + 2);
+		in_next += 4;
 
 		SAFETY_CHECK(len == (u16)~nlen);
 		if (unlikely(len > out_end - out_next))
@@ -209,6 +285,8 @@ next_block:
 		goto block_done;
 
 	} else {
+		unsigned i;
+
 		SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
 
 		/*
@@ -221,6 +299,9 @@ next_block:
 		 * dynamic Huffman block.
 		 */
 
+		bitbuf >>= 3; /* for BTYPE and BFINAL */
+		bitsleft -= 3;
+
 		if (d->static_codes_loaded)
 			goto have_decode_tables;
 
@@ -245,169 +326,442 @@ next_block:
 		num_offset_syms = 32;
 	}
 
-	/* Decompressing a Huffman block (either dynamic or static)  */
+	/* Decompressing a Huffman block (either dynamic or static) */
 
 	SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
 	SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
 have_decode_tables:
+	litlen_tablemask = BITMASK(d->litlen_tablebits);
 
-	/* The main DEFLATE decode loop  */
-	for (;;) {
-		u32 entry;
-		u32 length;
-		u32 offset;
+	/*
+	 * This is the "fastloop" for decoding literals and matches.  It does
+	 * bounds checks on in_next and out_next in the loop conditions so that
+	 * additional bounds checks aren't needed inside the loop body.
+	 *
+	 * To reduce latency, the bitbuffer is refilled and the next litlen
+	 * decode table entry is preloaded before each loop iteration.
+	 */
+	if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
+		goto generic_loop;
+	REFILL_BITS_IN_FASTLOOP();
+	entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+	do {
+		u32 length, offset, lit;
 		const u8 *src;
 		u8 *dst;
 
-		/* Decode a litlen symbol.  */
-		ENSURE_BITS(DEFLATE_MAX_LITLEN_CODEWORD_LEN);
-		entry = d->u.litlen_decode_table[BITS(LITLEN_TABLEBITS)];
-		if (entry & HUFFDEC_SUBTABLE_POINTER) {
-			/* Litlen subtable required (uncommon case)  */
-			REMOVE_BITS(LITLEN_TABLEBITS);
-			entry = d->u.litlen_decode_table[
-				((entry >> HUFFDEC_RESULT_SHIFT) & 0xFFFF) +
-				BITS(entry & HUFFDEC_LENGTH_MASK)];
-		}
-		REMOVE_BITS(entry & HUFFDEC_LENGTH_MASK);
-		if (entry & HUFFDEC_LITERAL) {
-			/* Literal  */
-			if (unlikely(out_next == out_end))
-				return LIBDEFLATE_INSUFFICIENT_SPACE;
-			*out_next++ = (u8)(entry >> HUFFDEC_RESULT_SHIFT);
-			continue;
-		}
-
-		/* Match or end-of-block  */
-
-		entry >>= HUFFDEC_RESULT_SHIFT;
-		ENSURE_BITS(MAX_ENSURE);
-
-		/* Pop the extra length bits and add them to the length base to
-		 * produce the full length.  */
-		length = (entry >> HUFFDEC_LENGTH_BASE_SHIFT) +
-			 POP_BITS(entry & HUFFDEC_EXTRA_LENGTH_BITS_MASK);
-
-		/* The match destination must not end after the end of the
-		 * output buffer.  For efficiency, combine this check with the
-		 * end-of-block check.  We're using 0 for the special
-		 * end-of-block length, so subtract 1 and it turn it into
-		 * SIZE_MAX.  */
-		STATIC_ASSERT(HUFFDEC_END_OF_BLOCK_LENGTH == 0);
-		if (unlikely((size_t)length - 1 >= out_end - out_next)) {
-			if (unlikely(length != HUFFDEC_END_OF_BLOCK_LENGTH))
-				return LIBDEFLATE_INSUFFICIENT_SPACE;
-			goto block_done;
-		}
-
-		/* Decode the match offset.  */
-
-		entry = d->offset_decode_table[BITS(OFFSET_TABLEBITS)];
-		if (entry & HUFFDEC_SUBTABLE_POINTER) {
-			/* Offset subtable required (uncommon case)  */
-			REMOVE_BITS(OFFSET_TABLEBITS);
-			entry = d->offset_decode_table[
-				((entry >> HUFFDEC_RESULT_SHIFT) & 0xFFFF) +
-				BITS(entry & HUFFDEC_LENGTH_MASK)];
-		}
-		REMOVE_BITS(entry & HUFFDEC_LENGTH_MASK);
-		entry >>= HUFFDEC_RESULT_SHIFT;
-
-		STATIC_ASSERT(CAN_ENSURE(DEFLATE_MAX_EXTRA_LENGTH_BITS +
-					 DEFLATE_MAX_OFFSET_CODEWORD_LEN) &&
-			      CAN_ENSURE(DEFLATE_MAX_EXTRA_OFFSET_BITS));
-		if (!CAN_ENSURE(DEFLATE_MAX_EXTRA_LENGTH_BITS +
-				DEFLATE_MAX_OFFSET_CODEWORD_LEN +
-				DEFLATE_MAX_EXTRA_OFFSET_BITS))
-			ENSURE_BITS(DEFLATE_MAX_EXTRA_OFFSET_BITS);
-
-		/* Pop the extra offset bits and add them to the offset base to
-		 * produce the full offset.  */
-		offset = (entry & HUFFDEC_OFFSET_BASE_MASK) +
-			 POP_BITS(entry >> HUFFDEC_EXTRA_OFFSET_BITS_SHIFT);
-
-		/* The match source must not begin before the beginning of the
-		 * output buffer.  */
-		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
+		/*
+		 * Consume the bits for the litlen decode table entry.  Save the
+		 * original bitbuf for later, in case the extra match length
+		 * bits need to be extracted from it.
+		 */
+		saved_bitbuf = bitbuf;
+		bitbuf >>= (u8)entry;
+		bitsleft -= entry; /* optimization: subtract full entry */
 
 		/*
-		 * Copy the match: 'length' bytes at 'out_next - offset' to
-		 * 'out_next', possibly overlapping.  If the match doesn't end
-		 * too close to the end of the buffer and offset >= WORDBYTES ||
-		 * offset == 1, take a fast path which copies a word at a time
-		 * -- potentially more than the length of the match, but that's
-		 * fine as long as we check for enough extra space.
-		 *
-		 * The remaining cases are not performance-critical so are
-		 * handled by a simple byte-by-byte copy.
+		 * Begin by checking for a "fast" literal, i.e. a literal that
+		 * doesn't need a subtable.
 		 */
+		if (entry & HUFFDEC_LITERAL) {
+			/*
+			 * On 64-bit platforms, we decode up to 2 extra fast
+			 * literals in addition to the primary item, as this
+			 * increases performance and still leaves enough bits
+			 * remaining for what follows.  We could actually do 3,
+			 * assuming LITLEN_TABLEBITS=11, but that actually
+			 * decreases performance slightly (perhaps by messing
+			 * with the branch prediction of the conditional refill
+			 * that happens later while decoding the match offset).
+			 *
+			 * Note: the definitions of FASTLOOP_MAX_BYTES_WRITTEN
+			 * and FASTLOOP_MAX_BYTES_READ need to be updated if the
+			 * number of extra literals decoded here is changed.
+			 */
+			if (/* enough bits for 2 fast literals + length + offset preload? */
+			    CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
+							 LENGTH_MAXBITS,
+							 OFFSET_TABLEBITS) &&
+			    /* enough bits for 2 fast literals + slow literal + litlen preload? */
+			    CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
+							 DEFLATE_MAX_LITLEN_CODEWORD_LEN,
+							 LITLEN_TABLEBITS)) {
+				/* 1st extra fast literal */
+				lit = entry >> 16;
+				entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+				saved_bitbuf = bitbuf;
+				bitbuf >>= (u8)entry;
+				bitsleft -= entry;
+				*out_next++ = lit;
+				if (entry & HUFFDEC_LITERAL) {
+					/* 2nd extra fast literal */
+					lit = entry >> 16;
+					entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+					saved_bitbuf = bitbuf;
+					bitbuf >>= (u8)entry;
+					bitsleft -= entry;
+					*out_next++ = lit;
+					if (entry & HUFFDEC_LITERAL) {
+						/*
+						 * Another fast literal, but
+						 * this one is in lieu of the
+						 * primary item, so it doesn't
+						 * count as one of the extras.
+						 */
+						lit = entry >> 16;
+						entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+						REFILL_BITS_IN_FASTLOOP();
+						*out_next++ = lit;
+						continue;
+					}
+				}
+			} else {
+				/*
+				 * Decode a literal.  While doing so, preload
+				 * the next litlen decode table entry and refill
+				 * the bitbuffer.  To reduce latency, we've
+				 * arranged for there to be enough "preloadable"
+				 * bits remaining to do the table preload
+				 * independently of the refill.
+				 */
+				STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
+						LITLEN_TABLEBITS, LITLEN_TABLEBITS));
+				lit = entry >> 16;
+				entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+				REFILL_BITS_IN_FASTLOOP();
+				*out_next++ = lit;
+				continue;
+			}
+		}
 
+		/*
+		 * It's not a literal entry, so it can be a length entry, a
+		 * subtable pointer entry, or an end-of-block entry.  Detect the
+		 * two unlikely cases by testing the HUFFDEC_EXCEPTIONAL flag.
+		 */
+		if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+			/* Subtable pointer or end-of-block entry */
+
+			if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+				goto block_done;
+
+			/*
+			 * A subtable is required.  Load and consume the
+			 * subtable entry.  The subtable entry can be of any
+			 * type: literal, length, or end-of-block.
+			 */
+			entry = d->u.litlen_decode_table[(entry >> 16) +
+				EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+			saved_bitbuf = bitbuf;
+			bitbuf >>= (u8)entry;
+			bitsleft -= entry;
+
+			/*
+			 * 32-bit platforms that use the byte-at-a-time refill
+			 * method have to do a refill here for there to always
+			 * be enough bits to decode a literal that requires a
+			 * subtable, then preload the next litlen decode table
+			 * entry; or to decode a match length that requires a
+			 * subtable, then preload the offset decode table entry.
+			 */
+			if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
+							  LITLEN_TABLEBITS) ||
+			    !CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
+							  OFFSET_TABLEBITS))
+				REFILL_BITS_IN_FASTLOOP();
+			if (entry & HUFFDEC_LITERAL) {
+				/* Decode a literal that required a subtable. */
+				lit = entry >> 16;
+				entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+				REFILL_BITS_IN_FASTLOOP();
+				*out_next++ = lit;
+				continue;
+			}
+			if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+				goto block_done;
+			/* Else, it's a length that required a subtable. */
+		}
+
+		/*
+		 * Decode the match length: the length base value associated
+		 * with the litlen symbol (which we extract from the decode
+		 * table entry), plus the extra length bits.  We don't need to
+		 * consume the extra length bits here, as they were included in
+		 * the bits consumed by the entry earlier.  We also don't need
+		 * to check for too-long matches here, as this is inside the
+		 * fastloop where it's already been verified that the output
+		 * buffer has enough space remaining to copy a max-length match.
+		 */
+		length = entry >> 16;
+		length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+
+		/*
+		 * Decode the match offset.  There are enough "preloadable" bits
+		 * remaining to preload the offset decode table entry, but a
+		 * refill might be needed before consuming it.
+		 */
+		STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
+							   OFFSET_TABLEBITS));
+		entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
+		if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
+						 LITLEN_TABLEBITS)) {
+			/*
+			 * Decoding a match offset on a 64-bit platform.  We may
+			 * need to refill once, but then we can decode the whole
+			 * offset and preload the next litlen table entry.
+			 */
+			if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+				/* Offset codeword requires a subtable */
+				if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
+					     LITLEN_TABLEBITS - PRELOAD_SLACK))
+					REFILL_BITS_IN_FASTLOOP();
+				bitbuf >>= OFFSET_TABLEBITS;
+				bitsleft -= OFFSET_TABLEBITS;
+				entry = d->offset_decode_table[(entry >> 16) +
+					EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+			} else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
+					    LITLEN_TABLEBITS - PRELOAD_SLACK))
+				REFILL_BITS_IN_FASTLOOP();
+		} else {
+			/* Decoding a match offset on a 32-bit platform */
+			REFILL_BITS_IN_FASTLOOP();
+			if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+				/* Offset codeword requires a subtable */
+				bitbuf >>= OFFSET_TABLEBITS;
+				bitsleft -= OFFSET_TABLEBITS;
+				entry = d->offset_decode_table[(entry >> 16) +
+					EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+				REFILL_BITS_IN_FASTLOOP();
+				/* No further refill needed before extra bits */
+				STATIC_ASSERT(CAN_CONSUME(
+					OFFSET_MAXBITS - OFFSET_TABLEBITS));
+			} else {
+				/* No refill needed before extra bits */
+				STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
+			}
+		}
+		saved_bitbuf = bitbuf;
+		bitbuf >>= (u8)entry;
+		bitsleft -= entry; /* optimization: subtract full entry */
+		offset = entry >> 16;
+		offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+
+		/* Validate the match offset; needed even in the fastloop. */
+		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
 		src = out_next - offset;
 		dst = out_next;
 		out_next += length;
 
-		if (UNALIGNED_ACCESS_IS_FAST &&
-		    /* max overrun is writing 3 words for a min length match */
-		    likely(out_end - out_next >=
-			   3 * WORDBYTES - DEFLATE_MIN_MATCH_LEN)) {
-			if (offset >= WORDBYTES) { /* words don't overlap? */
-				copy_word_unaligned(src, dst);
-				src += WORDBYTES;
-				dst += WORDBYTES;
-				copy_word_unaligned(src, dst);
-				src += WORDBYTES;
-				dst += WORDBYTES;
-				do {
-					copy_word_unaligned(src, dst);
-					src += WORDBYTES;
-					dst += WORDBYTES;
-				} while (dst < out_next);
-			} else if (offset == 1) {
-				/* RLE encoding of previous byte, common if the
-				 * data contains many repeated bytes */
-				machine_word_t v = repeat_byte(*src);
+		/*
+		 * Before starting to issue the instructions to copy the match,
+		 * refill the bitbuffer and preload the litlen decode table
+		 * entry for the next loop iteration.  This can increase
+		 * performance by allowing the latency of the match copy to
+		 * overlap with these other operations.  To further reduce
+		 * latency, we've arranged for there to be enough bits remaining
+		 * to do the table preload independently of the refill, except
+		 * on 32-bit platforms using the byte-at-a-time refill method.
+		 */
+		if (!CAN_CONSUME_AND_THEN_PRELOAD(
+			MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
+			    OFFSET_MAXFASTBITS),
+			LITLEN_TABLEBITS) &&
+		    unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
+			REFILL_BITS_IN_FASTLOOP();
+		entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+		REFILL_BITS_IN_FASTLOOP();
 
-				store_word_unaligned(v, dst);
+		/*
+		 * Copy the match.  On most CPUs the fastest method is a
+		 * word-at-a-time copy, unconditionally copying about 5 words
+		 * since this is enough for most matches without being too much.
+		 *
+		 * The normal word-at-a-time copy works for offset >= WORDBYTES,
+		 * which is most cases.  The case of offset == 1 is also common
+		 * and is worth optimizing for, since it is just RLE encoding of
+		 * the previous byte, which is the result of compressing long
+		 * runs of the same byte.
+		 *
+		 * Writing past the match 'length' is allowed here, since it's
+		 * been ensured there is enough output space left for a slight
+		 * overrun.  FASTLOOP_MAX_BYTES_WRITTEN needs to be updated if
+		 * the maximum possible overrun here is changed.
+		 */
+		if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += WORDBYTES;
+			dst += WORDBYTES;
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += WORDBYTES;
+			dst += WORDBYTES;
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += WORDBYTES;
+			dst += WORDBYTES;
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += WORDBYTES;
+			dst += WORDBYTES;
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += WORDBYTES;
+			dst += WORDBYTES;
+			while (dst < out_next) {
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += WORDBYTES;
 				dst += WORDBYTES;
-				store_word_unaligned(v, dst);
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += WORDBYTES;
+				dst += WORDBYTES;
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += WORDBYTES;
 				dst += WORDBYTES;
-				do {
-					store_word_unaligned(v, dst);
-					dst += WORDBYTES;
-				} while (dst < out_next);
-			} else {
-				*dst++ = *src++;
-				*dst++ = *src++;
-				do {
-					*dst++ = *src++;
-				} while (dst < out_next);
 			}
+		} else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
+			machine_word_t v;
+
+			/*
+			 * This part tends to get auto-vectorized, so keep it
+			 * copying a multiple of 16 bytes at a time.
+			 */
+			v = (machine_word_t)0x0101010101010101 * src[0];
+			store_word_unaligned(v, dst);
+			dst += WORDBYTES;
+			store_word_unaligned(v, dst);
+			dst += WORDBYTES;
+			store_word_unaligned(v, dst);
+			dst += WORDBYTES;
+			store_word_unaligned(v, dst);
+			dst += WORDBYTES;
+			while (dst < out_next) {
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+				store_word_unaligned(v, dst);
+				dst += WORDBYTES;
+			}
+		} else if (UNALIGNED_ACCESS_IS_FAST) {
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += offset;
+			dst += offset;
+			store_word_unaligned(load_word_unaligned(src), dst);
+			src += offset;
+			dst += offset;
+			do {
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += offset;
+				dst += offset;
+				store_word_unaligned(load_word_unaligned(src), dst);
+				src += offset;
+				dst += offset;
+			} while (dst < out_next);
 		} else {
-			STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
 			*dst++ = *src++;
 			*dst++ = *src++;
 			do {
 				*dst++ = *src++;
 			} while (dst < out_next);
 		}
+	} while (in_next < in_fastloop_end && out_next < out_fastloop_end);
+
+	/*
+	 * This is the generic loop for decoding literals and matches.  This
+	 * handles cases where in_next and out_next are close to the end of
+	 * their respective buffers.  Usually this loop isn't performance-
+	 * critical, as most time is spent in the fastloop above instead.  We
+	 * therefore omit some optimizations here in favor of smaller code.
+	 */
+generic_loop:
+	for (;;) {
+		u32 length, offset;
+		const u8 *src;
+		u8 *dst;
+
+		REFILL_BITS();
+		entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+		saved_bitbuf = bitbuf;
+		bitbuf >>= (u8)entry;
+		bitsleft -= entry;
+		if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
+			entry = d->u.litlen_decode_table[(entry >> 16) +
+					EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+			saved_bitbuf = bitbuf;
+			bitbuf >>= (u8)entry;
+			bitsleft -= entry;
+		}
+		length = entry >> 16;
+		if (entry & HUFFDEC_LITERAL) {
+			if (unlikely(out_next == out_end))
+				return LIBDEFLATE_INSUFFICIENT_SPACE;
+			*out_next++ = length;
+			continue;
+		}
+		if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+			goto block_done;
+		length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+		if (unlikely(length > out_end - out_next))
+			return LIBDEFLATE_INSUFFICIENT_SPACE;
+
+		if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
+			REFILL_BITS();
+		entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
+		if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+			bitbuf >>= OFFSET_TABLEBITS;
+			bitsleft -= OFFSET_TABLEBITS;
+			entry = d->offset_decode_table[(entry >> 16) +
+					EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+			if (!CAN_CONSUME(OFFSET_MAXBITS))
+				REFILL_BITS();
+		}
+		offset = entry >> 16;
+		offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
+		bitbuf >>= (u8)entry;
+		bitsleft -= entry;
+
+		SAFETY_CHECK(offset <= out_next - (const u8 *)out);
+		src = out_next - offset;
+		dst = out_next;
+		out_next += length;
+
+		STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
+		*dst++ = *src++;
+		*dst++ = *src++;
+		do {
+			*dst++ = *src++;
+		} while (dst < out_next);
 	}
 
 block_done:
-	/* Finished decoding a block.  */
+	/* Finished decoding a block */
 
 	if (!is_final_block)
 		goto next_block;
 
-	/* That was the last block.  */
+	/* That was the last block. */
 
-	/* Discard any readahead bits and check for excessive overread */
-	ALIGN_INPUT();
+	bitsleft = (u8)bitsleft;
+
+	/*
+	 * If any of the implicit appended zero bytes were consumed (not just
+	 * refilled) before hitting end of stream, then the data is bad.
+	 */
+	SAFETY_CHECK(overread_count <= (bitsleft >> 3));
+
+	/* Optionally return the actual number of bytes consumed. */
+	if (actual_in_nbytes_ret) {
+		/* Don't count bytes that were refilled but not consumed. */
+		in_next -= (bitsleft >> 3) - overread_count;
 
-	/* Optionally return the actual number of bytes read */
-	if (actual_in_nbytes_ret)
 		*actual_in_nbytes_ret = in_next - (u8 *)in;
+	}
 
-	/* Optionally return the actual number of bytes written */
+	/* Optionally return the actual number of bytes written. */
 	if (actual_out_nbytes_ret) {
 		*actual_out_nbytes_ret = out_next - (u8 *)out;
 	} else {
@@ -419,3 +773,5 @@ block_done:
 
 #undef FUNCNAME
 #undef ATTRIBUTES
+#undef EXTRACT_VARBITS
+#undef EXTRACT_VARBITS8

lib/deflate_compress.h

@@ -10,6 +10,6 @@
 
 struct libdeflate_compressor;
 
-unsigned int deflate_get_compression_level(struct libdeflate_compressor *c);
+unsigned int libdeflate_get_compression_level(struct libdeflate_compressor *c);
 
 #endif /* LIB_DEFLATE_COMPRESS_H */

lib/deflate_constants.h

@@ -49,11 +49,8 @@
 /*
  * Maximum number of extra bits that may be required to represent a match
  * length or offset.
- *
- * TODO: are we going to have full DEFLATE64 support?  If so, up to 16
- * length bits must be supported.
  */
 #define DEFLATE_MAX_EXTRA_LENGTH_BITS		5
-#define DEFLATE_MAX_EXTRA_OFFSET_BITS		14
+#define DEFLATE_MAX_EXTRA_OFFSET_BITS		13
 
 #endif /* LIB_DEFLATE_CONSTANTS_H */

lib/gzip_compress.c

@@ -27,11 +27,8 @@
 
 #include "deflate_compress.h"
 #include "gzip_constants.h"
-#include "unaligned.h"
 
-#include "libdeflate.h"
-
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_gzip_compress(struct libdeflate_compressor *c,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail)
@@ -57,7 +54,7 @@ libdeflate_gzip_compress(struct libdeflate_compressor *c,
 	out_next += 4;
 	/* XFL */
 	xfl = 0;
-	compression_level = deflate_get_compression_level(c);
+	compression_level = libdeflate_get_compression_level(c);
 	if (compression_level < 2)
 		xfl |= GZIP_XFL_FASTEST_COMPRESSION;
 	else if (compression_level >= 8)
@@ -84,7 +81,7 @@ libdeflate_gzip_compress(struct libdeflate_compressor *c,
 	return out_next - (u8 *)out;
 }
 
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_gzip_compress_bound(struct libdeflate_compressor *c,
 			       size_t in_nbytes)
 {

lib/gzip_decompress.c

@@ -25,12 +25,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include "lib_common.h"
 #include "gzip_constants.h"
-#include "unaligned.h"
 
-#include "libdeflate.h"
-
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -134,7 +132,7 @@ libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
 	return LIBDEFLATE_SUCCESS;
 }
 
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_gzip_decompress(struct libdeflate_decompressor *d,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,

lib/hc_matchfinder.h

@@ -116,7 +116,7 @@
 	(((1UL << HC_MATCHFINDER_HASH3_ORDER) +		\
 	  (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
 
-struct hc_matchfinder {
+struct MATCHFINDER_ALIGNED hc_matchfinder  {
 
 	/* The hash table for finding length 3 matches  */
 	mf_pos_t hash3_tab[1UL << HC_MATCHFINDER_HASH3_ORDER];
@@ -128,8 +128,7 @@ struct hc_matchfinder {
 	/* The "next node" references for the linked lists.  The "next node" of
 	 * the node for the sequence with position 'pos' is 'next_tab[pos]'.  */
 	mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE];
-
-} MATCHFINDER_ALIGNED;
+};
 
 /* Prepare the matchfinder for a new input buffer.  */
 static forceinline void

lib/ht_matchfinder.h

@@ -54,10 +54,10 @@
 /* Minimum value of max_len for ht_matchfinder_longest_match() */
 #define HT_MATCHFINDER_REQUIRED_NBYTES	5
 
-struct ht_matchfinder {
+struct MATCHFINDER_ALIGNED ht_matchfinder {
 	mf_pos_t hash_tab[1UL << HT_MATCHFINDER_HASH_ORDER]
 			 [HT_MATCHFINDER_BUCKET_SIZE];
-} MATCHFINDER_ALIGNED;
+};
 
 static forceinline void
 ht_matchfinder_init(struct ht_matchfinder *mf)

lib/lib_common.h

@@ -6,62 +6,48 @@
 #define LIB_LIB_COMMON_H
 
 #ifdef LIBDEFLATE_H
+ /*
+  * When building the library, LIBDEFLATEAPI needs to be defined properly before
+  * including libdeflate.h.
+  */
 #  error "lib_common.h must always be included before libdeflate.h"
-   /* because BUILDING_LIBDEFLATE must be set first */
 #endif
 
-#define BUILDING_LIBDEFLATE
-
-#include "../common/common_defs.h"
-
-#include <stdio.h>
-#include <inttypes.h>
-#include <time.h>
-
-static forceinline u64 now(void)
-{
-#ifdef __x86_64__
-	u32 lo, hi;
-	__asm__ volatile("rdtsc" : "=a" (lo), "=d" (hi));
-	return ((u64)hi << 32) | lo;
-#elif defined(__i386__)
-	u32 lo, hi;
-	__asm__ volatile("rdtsc" : "=a" (lo), "=d" (hi));
-	return ((u64)hi << 32) | lo;
+#if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
+#  define LIBDEFLATE_EXPORT_SYM  __declspec(dllexport)
+#elif defined(__GNUC__)
+#  define LIBDEFLATE_EXPORT_SYM  __attribute__((visibility("default")))
 #else
-	struct timespec ts;
-	clock_gettime(CLOCK_MONOTONIC, &ts);
-	return 1000000000 * (u64)ts.tv_sec + ts.tv_nsec;
+#  define LIBDEFLATE_EXPORT_SYM
 #endif
-}
-static u64 g_start_time __attribute__((unused));
-static u64 g_total_time __attribute__((unused));
-static u64 g_num_measurements __attribute__((unused));
-
-#define BEGIN g_start_time = now();
-#define END  g_total_time += now() - g_start_time; g_num_measurements++;
 
 /*
- * Prefix with "_libdeflate_" all global symbols which are not part of the API
- * and don't already have a "libdeflate" prefix.  This avoids exposing overly
- * generic names when libdeflate is built as a static library.
- *
- * Note that the chosen prefix is not really important and can be changed
- * without breaking library users.  It was just chosen so that the resulting
- * symbol names are unlikely to conflict with those from any other software.
- * Also note that this fixup has no useful effect when libdeflate is built as a
- * shared library, since these symbols are not exported.
+ * On i386, gcc assumes that the stack is 16-byte aligned at function entry.
+ * However, some compilers (e.g. MSVC) and programming languages (e.g. Delphi)
+ * only guarantee 4-byte alignment when calling functions.  This is mainly an
+ * issue on Windows, but it has been seen on Linux too.  Work around this ABI
+ * incompatibility by realigning the stack pointer when entering libdeflate.
+ * This prevents crashes in SSE/AVX code.
  */
-#define SYM_FIXUP(sym)			_libdeflate_##sym
-#define deflate_get_compression_level	SYM_FIXUP(deflate_get_compression_level)
-#define _cpu_features			SYM_FIXUP(_cpu_features)
-#define setup_cpu_features		SYM_FIXUP(setup_cpu_features)
+#if defined(__GNUC__) && defined(__i386__)
+#  define LIBDEFLATE_ALIGN_STACK  __attribute__((force_align_arg_pointer))
+#else
+#  define LIBDEFLATE_ALIGN_STACK
+#endif
 
-void *libdeflate_malloc(size_t size);
-void libdeflate_free(void *ptr);
+#define LIBDEFLATEAPI	LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
 
-void *libdeflate_aligned_malloc(size_t alignment, size_t size);
-void libdeflate_aligned_free(void *ptr);
+#include "../common_defs.h"
+
+typedef void *(*malloc_func_t)(size_t);
+typedef void (*free_func_t)(void *);
+
+extern malloc_func_t libdeflate_default_malloc_func;
+extern free_func_t libdeflate_default_free_func;
+
+void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
+				size_t alignment, size_t size);
+void libdeflate_aligned_free(free_func_t free_func, void *ptr);
 
 #ifdef FREESTANDING
 /*
@@ -90,7 +76,14 @@ int memcmp(const void *s1, const void *s2, size_t n);
 
 #undef LIBDEFLATE_ENABLE_ASSERTIONS
 #else
-#include <string.h>
+#  include <string.h>
+   /*
+    * To prevent false positive static analyzer warnings, ensure that assertions
+    * are visible to the static analyzer.
+    */
+#  ifdef __clang_analyzer__
+#    define LIBDEFLATE_ENABLE_ASSERTIONS
+#  endif
 #endif
 
 /*
@@ -98,11 +91,16 @@ int memcmp(const void *s1, const void *s2, size_t n);
  * hurt performance significantly.
  */
 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
-void libdeflate_assertion_failed(const char *expr, const char *file, int line);
+NORETURN void
+libdeflate_assertion_failed(const char *expr, const char *file, int line);
 #define ASSERT(expr) { if (unlikely(!(expr))) \
 	libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
 #else
 #define ASSERT(expr) (void)(expr)
 #endif
 
+#define CONCAT_IMPL(a, b)	a##b
+#define CONCAT(a, b)		CONCAT_IMPL(a, b)
+#define ADD_SUFFIX(name)	CONCAT(name, SUFFIX)
+
 #endif /* LIB_LIB_COMMON_H */

lib/matchfinder_common.h

@@ -6,12 +6,44 @@
 #define LIB_MATCHFINDER_COMMON_H
 
 #include "lib_common.h"
-#include "unaligned.h"
 
 #ifndef MATCHFINDER_WINDOW_ORDER
 #  error "MATCHFINDER_WINDOW_ORDER must be defined!"
 #endif
 
+/*
+ * Given a 32-bit value that was loaded with the platform's native endianness,
+ * return a 32-bit value whose high-order 8 bits are 0 and whose low-order 24
+ * bits contain the first 3 bytes, arranged in octets in a platform-dependent
+ * order, at the memory location from which the input 32-bit value was loaded.
+ */
+static forceinline u32
+loaded_u32_to_u24(u32 v)
+{
+	if (CPU_IS_LITTLE_ENDIAN())
+		return v & 0xFFFFFF;
+	else
+		return v >> 8;
+}
+
+/*
+ * Load the next 3 bytes from @p into the 24 low-order bits of a 32-bit value.
+ * The order in which the 3 bytes will be arranged as octets in the 24 bits is
+ * platform-dependent.  At least 4 bytes (not 3) must be available at @p.
+ */
+static forceinline u32
+load_u24_unaligned(const u8 *p)
+{
+#if UNALIGNED_ACCESS_IS_FAST
+	return loaded_u32_to_u24(load_u32_unaligned(p));
+#else
+	if (CPU_IS_LITTLE_ENDIAN())
+		return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
+	else
+		return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
+#endif
+}
+
 #define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER)
 
 typedef s16 mf_pos_t;
@@ -19,19 +51,44 @@ typedef s16 mf_pos_t;
 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
 
 /*
- * Required alignment of the matchfinder buffer pointer and size.  The values
- * here come from the AVX-2 implementation, which is the worst case.
+ * This is the memory address alignment, in bytes, required for the matchfinder
+ * buffers by the architecture-specific implementations of matchfinder_init()
+ * and matchfinder_rebase().  "Matchfinder buffer" means an entire struct
+ * hc_matchfinder, bt_matchfinder, or ht_matchfinder; the next_tab field of
+ * struct hc_matchfinder; or the child_tab field of struct bt_matchfinder.
+ *
+ * This affects how the entire 'struct deflate_compressor' is allocated, since
+ * the matchfinder structures are embedded inside it.
+ *
+ * Currently the maximum memory address alignment required is 32 bytes, needed
+ * by the AVX-2 matchfinder functions.
  */
 #define MATCHFINDER_MEM_ALIGNMENT	32
-#define MATCHFINDER_SIZE_ALIGNMENT	128
+
+/*
+ * This declares a size, in bytes, that is guaranteed to divide the sizes of the
+ * matchfinder buffers (where "matchfinder buffers" is as defined for
+ * MATCHFINDER_MEM_ALIGNMENT).  The architecture-specific implementations of
+ * matchfinder_init() and matchfinder_rebase() take advantage of this value.
+ *
+ * Currently the maximum size alignment required is 128 bytes, needed by
+ * the AVX-2 matchfinder functions.  However, the RISC-V Vector Extension
+ * matchfinder functions can, in principle, take advantage of a larger size
+ * alignment.  Therefore, we set this to 1024, which still easily divides the
+ * actual sizes that result from the current matchfinder struct definitions.
+ * This value can safely be changed to any power of two that is >= 128.
+ */
+#define MATCHFINDER_SIZE_ALIGNMENT	1024
 
 #undef matchfinder_init
 #undef matchfinder_rebase
 #ifdef _aligned_attribute
 #  define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
-#  if defined(__arm__) || defined(__aarch64__)
+#  if defined(ARCH_ARM32) || defined(ARCH_ARM64)
 #    include "arm/matchfinder_impl.h"
-#  elif defined(__i386__) || defined(__x86_64__)
+#  elif defined(ARCH_RISCV)
+#    include "riscv/matchfinder_impl.h"
+#  elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
 #    include "x86/matchfinder_impl.h"
 #  endif
 #else
@@ -59,28 +116,21 @@ matchfinder_init(mf_pos_t *data, size_t size)
 #endif
 
 /*
- * Slide the matchfinder by WINDOW_SIZE bytes.
+ * Slide the matchfinder by MATCHFINDER_WINDOW_SIZE bytes.
  *
- * This must be called just after each WINDOW_SIZE bytes have been run through
- * the matchfinder.
+ * This must be called just after each MATCHFINDER_WINDOW_SIZE bytes have been
+ * run through the matchfinder.
  *
- * This will subtract WINDOW_SIZE bytes from each entry in the array specified.
- * The effect is that all entries are updated to be relative to the current
- * position, rather than the position WINDOW_SIZE bytes prior.
+ * This subtracts MATCHFINDER_WINDOW_SIZE bytes from each entry in the given
+ * array, making the entries be relative to the current position rather than the
+ * position MATCHFINDER_WINDOW_SIZE bytes prior.  To avoid integer underflows,
+ * entries that would become less than -MATCHFINDER_WINDOW_SIZE stay at
+ * -MATCHFINDER_WINDOW_SIZE, keeping them permanently out of bounds.
  *
- * Underflow is detected and replaced with signed saturation.  This ensures that
- * once the sliding window has passed over a position, that position forever
- * remains out of bounds.
- *
- * The array passed in must contain all matchfinder data that is
- * position-relative.  Concretely, this will include the hash table as well as
- * the table of positions that is used to link together the sequences in each
- * hash bucket.  Note that in the latter table, the links are 1-ary in the case
- * of "hash chains", and 2-ary in the case of "binary trees".  In either case,
- * the links need to be rebased in the same way.
- *
- * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
- * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
+ * The given array must contain all matchfinder data that is position-relative:
+ * the hash table(s) as well as any hash chain or binary tree links.  Its
+ * address must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and its size
+ * must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
  */
 #ifndef matchfinder_rebase
 static forceinline void
@@ -90,25 +140,20 @@ matchfinder_rebase(mf_pos_t *data, size_t size)
 	size_t i;
 
 	if (MATCHFINDER_WINDOW_SIZE == 32768) {
-		/* Branchless version for 32768 byte windows.  If the value was
-		 * already negative, clear all bits except the sign bit; this
-		 * changes the value to -32768.  Otherwise, set the sign bit;
-		 * this is equivalent to subtracting 32768.  */
+		/*
+		 * Branchless version for 32768-byte windows.  Clear all bits if
+		 * the value was already negative, then set the sign bit.  This
+		 * is equivalent to subtracting 32768 with signed saturation.
+		 */
+		for (i = 0; i < num_entries; i++)
+			data[i] = 0x8000 | (data[i] & ~(data[i] >> 15));
+	} else {
 		for (i = 0; i < num_entries; i++) {
-			u16 v = data[i];
-			u16 sign_bit = v & 0x8000;
-			v &= sign_bit - ((sign_bit >> 15) ^ 1);
-			v |= 0x8000;
-			data[i] = v;
+			if (data[i] >= 0)
+				data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
+			else
+				data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
 		}
-		return;
-	}
-
-	for (i = 0; i < num_entries; i++) {
-		if (data[i] >= 0)
-			data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
-		else
-			data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
 	}
 }
 #endif
@@ -130,11 +175,11 @@ lz_hash(u32 seq, unsigned num_bits)
  * Return the number of bytes at @matchptr that match the bytes at @strptr, up
  * to a maximum of @max_len.  Initially, @start_len bytes are matched.
  */
-static forceinline unsigned
+static forceinline u32
 lz_extend(const u8 * const strptr, const u8 * const matchptr,
-	  const unsigned start_len, const unsigned max_len)
+	  const u32 start_len, const u32 max_len)
 {
-	unsigned len = start_len;
+	u32 len = start_len;
 	machine_word_t v_word;
 
 	if (UNALIGNED_ACCESS_IS_FAST) {

lib/riscv/matchfinder_impl.h

@@ -0,0 +1,97 @@
+/*
+ * riscv/matchfinder_impl.h - RISC-V implementations of matchfinder functions
+ *
+ * Copyright 2024 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_RISCV_MATCHFINDER_IMPL_H
+#define LIB_RISCV_MATCHFINDER_IMPL_H
+
+#if defined(ARCH_RISCV) && defined(__riscv_vector)
+#include <riscv_vector.h>
+
+/*
+ * Return the maximum number of 16-bit (mf_pos_t) elements that fit in 8 RISC-V
+ * vector registers and also evenly divide the sizes of the matchfinder buffers.
+ */
+static forceinline size_t
+riscv_matchfinder_vl(void)
+{
+	const size_t vl = __riscv_vsetvlmax_e16m8();
+
+	STATIC_ASSERT(sizeof(mf_pos_t) == sizeof(s16));
+	/*
+	 * MATCHFINDER_SIZE_ALIGNMENT is a power of 2, as is 'vl' because the
+	 * RISC-V Vector Extension requires that the vector register length
+	 * (VLEN) be a power of 2.  Thus, a simple MIN() gives the correct
+	 * answer here; rounding to a power of 2 is not required.
+	 */
+	STATIC_ASSERT((MATCHFINDER_SIZE_ALIGNMENT &
+		       (MATCHFINDER_SIZE_ALIGNMENT - 1)) == 0);
+	ASSERT((vl & (vl - 1)) == 0);
+	return MIN(vl, MATCHFINDER_SIZE_ALIGNMENT / sizeof(mf_pos_t));
+}
+
+/* matchfinder_init() optimized using the RISC-V Vector Extension */
+static forceinline void
+matchfinder_init_rvv(mf_pos_t *p, size_t size)
+{
+	const size_t vl = riscv_matchfinder_vl();
+	const vint16m8_t v = __riscv_vmv_v_x_i16m8(MATCHFINDER_INITVAL, vl);
+
+	ASSERT(size > 0 && size % (vl * sizeof(p[0])) == 0);
+	do {
+		__riscv_vse16_v_i16m8(p, v, vl);
+		p += vl;
+		size -= vl * sizeof(p[0]);
+	} while (size != 0);
+}
+#define matchfinder_init matchfinder_init_rvv
+
+/* matchfinder_rebase() optimized using the RISC-V Vector Extension */
+static forceinline void
+matchfinder_rebase_rvv(mf_pos_t *p, size_t size)
+{
+	const size_t vl = riscv_matchfinder_vl();
+
+	ASSERT(size > 0 && size % (vl * sizeof(p[0])) == 0);
+	do {
+		vint16m8_t v = __riscv_vle16_v_i16m8(p, vl);
+
+		/*
+		 * This should generate the vsadd.vx instruction
+		 * (Vector Saturating Add, integer vector-scalar)
+		 */
+		v = __riscv_vsadd_vx_i16m8(v, (s16)-MATCHFINDER_WINDOW_SIZE,
+					   vl);
+		__riscv_vse16_v_i16m8(p, v, vl);
+		p += vl;
+		size -= vl * sizeof(p[0]);
+	} while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_rvv
+
+#endif /* ARCH_RISCV && __riscv_vector */
+
+#endif /* LIB_RISCV_MATCHFINDER_IMPL_H */

lib/unaligned.h

@@ -1,228 +0,0 @@
-/*
- * unaligned.h - inline functions for unaligned memory accesses
- */
-
-#ifndef LIB_UNALIGNED_H
-#define LIB_UNALIGNED_H
-
-#include "lib_common.h"
-
-/***** Unaligned loads and stores without endianness conversion *****/
-
-/*
- * memcpy() is portable, and it usually gets optimized appropriately by modern
- * compilers.  I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled
- * to a load or store instruction, not to an actual function call.
- *
- * We no longer use the "packed struct" approach, as that is nonstandard, has
- * unclear semantics, and doesn't receive enough testing
- * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
- *
- * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
- * where memcpy() generates inefficient code
- * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366).  However, we no longer
- * consider that one case important enough to maintain different code for.
- * If you run into it, please just use a newer version of gcc (or use clang).
- */
-
-#define DEFINE_UNALIGNED_TYPE(type)				\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	type v;							\
-	memcpy(&v, p, sizeof(v));				\
-	return v;						\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	memcpy(p, &v, sizeof(v));				\
-}
-
-DEFINE_UNALIGNED_TYPE(u16)
-DEFINE_UNALIGNED_TYPE(u32)
-DEFINE_UNALIGNED_TYPE(u64)
-DEFINE_UNALIGNED_TYPE(machine_word_t)
-
-#define load_word_unaligned	load_machine_word_t_unaligned
-#define store_word_unaligned	store_machine_word_t_unaligned
-
-/***** Unaligned loads with endianness conversion *****/
-
-static forceinline u16
-get_unaligned_le16(const u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST)
-		return le16_bswap(load_u16_unaligned(p));
-	else
-		return ((u16)p[1] << 8) | p[0];
-}
-
-static forceinline u16
-get_unaligned_be16(const u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST)
-		return be16_bswap(load_u16_unaligned(p));
-	else
-		return ((u16)p[0] << 8) | p[1];
-}
-
-static forceinline u32
-get_unaligned_le32(const u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST)
-		return le32_bswap(load_u32_unaligned(p));
-	else
-		return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
-			((u32)p[1] << 8) | p[0];
-}
-
-static forceinline u32
-get_unaligned_be32(const u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST)
-		return be32_bswap(load_u32_unaligned(p));
-	else
-		return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
-			((u32)p[2] << 8) | p[3];
-}
-
-static forceinline u64
-get_unaligned_le64(const u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST)
-		return le64_bswap(load_u64_unaligned(p));
-	else
-		return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
-			((u64)p[5] << 40) | ((u64)p[4] << 32) |
-			((u64)p[3] << 24) | ((u64)p[2] << 16) |
-			((u64)p[1] << 8) | p[0];
-}
-
-static forceinline machine_word_t
-get_unaligned_leword(const u8 *p)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		return get_unaligned_le32(p);
-	else
-		return get_unaligned_le64(p);
-}
-
-/***** Unaligned stores with endianness conversion *****/
-
-static forceinline void
-put_unaligned_le16(u16 v, u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST) {
-		store_u16_unaligned(le16_bswap(v), p);
-	} else {
-		p[0] = (u8)(v >> 0);
-		p[1] = (u8)(v >> 8);
-	}
-}
-
-static forceinline void
-put_unaligned_be16(u16 v, u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST) {
-		store_u16_unaligned(be16_bswap(v), p);
-	} else {
-		p[0] = (u8)(v >> 8);
-		p[1] = (u8)(v >> 0);
-	}
-}
-
-static forceinline void
-put_unaligned_le32(u32 v, u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST) {
-		store_u32_unaligned(le32_bswap(v), p);
-	} else {
-		p[0] = (u8)(v >> 0);
-		p[1] = (u8)(v >> 8);
-		p[2] = (u8)(v >> 16);
-		p[3] = (u8)(v >> 24);
-	}
-}
-
-static forceinline void
-put_unaligned_be32(u32 v, u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST) {
-		store_u32_unaligned(be32_bswap(v), p);
-	} else {
-		p[0] = (u8)(v >> 24);
-		p[1] = (u8)(v >> 16);
-		p[2] = (u8)(v >> 8);
-		p[3] = (u8)(v >> 0);
-	}
-}
-
-static forceinline void
-put_unaligned_le64(u64 v, u8 *p)
-{
-	if (UNALIGNED_ACCESS_IS_FAST) {
-		store_u64_unaligned(le64_bswap(v), p);
-	} else {
-		p[0] = (u8)(v >> 0);
-		p[1] = (u8)(v >> 8);
-		p[2] = (u8)(v >> 16);
-		p[3] = (u8)(v >> 24);
-		p[4] = (u8)(v >> 32);
-		p[5] = (u8)(v >> 40);
-		p[6] = (u8)(v >> 48);
-		p[7] = (u8)(v >> 56);
-	}
-}
-
-static forceinline void
-put_unaligned_leword(machine_word_t v, u8 *p)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		put_unaligned_le32(v, p);
-	else
-		put_unaligned_le64(v, p);
-}
-
-/***** 24-bit loads *****/
-
-/*
- * Given a 32-bit value that was loaded with the platform's native endianness,
- * return a 32-bit value whose high-order 8 bits are 0 and whose low-order 24
- * bits contain the first 3 bytes, arranged in octets in a platform-dependent
- * order, at the memory location from which the input 32-bit value was loaded.
- */
-static forceinline u32
-loaded_u32_to_u24(u32 v)
-{
-	if (CPU_IS_LITTLE_ENDIAN())
-		return v & 0xFFFFFF;
-	else
-		return v >> 8;
-}
-
-/*
- * Load the next 3 bytes from the memory location @p into the 24 low-order bits
- * of a 32-bit value.  The order in which the 3 bytes will be arranged as octets
- * in the 24 bits is platform-dependent.  At least LOAD_U24_REQUIRED_NBYTES
- * bytes must be available at @p; note that this may be more than 3.
- */
-static forceinline u32
-load_u24_unaligned(const u8 *p)
-{
-#if UNALIGNED_ACCESS_IS_FAST
-#  define LOAD_U24_REQUIRED_NBYTES 4
-	return loaded_u32_to_u24(load_u32_unaligned(p));
-#else
-#  define LOAD_U24_REQUIRED_NBYTES 3
-	if (CPU_IS_LITTLE_ENDIAN())
-		return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
-	else
-		return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
-#endif
-}
-
-#endif /* LIB_UNALIGNED_H */

lib/utils.c

@@ -27,8 +27,6 @@
 
 #include "lib_common.h"
 
-#include "libdeflate.h"
-
 #ifdef FREESTANDING
 #  define malloc NULL
 #  define free NULL
@@ -36,27 +34,18 @@
 #  include <stdlib.h>
 #endif
 
-static void *(*libdeflate_malloc_func)(size_t) = malloc;
-static void (*libdeflate_free_func)(void *) = free;
+malloc_func_t libdeflate_default_malloc_func = malloc;
+free_func_t libdeflate_default_free_func = free;
 
 void *
-libdeflate_malloc(size_t size)
+libdeflate_aligned_malloc(malloc_func_t malloc_func,
+			  size_t alignment, size_t size)
 {
-	return (*libdeflate_malloc_func)(size);
-}
+	void *ptr = (*malloc_func)(sizeof(void *) + alignment - 1 + size);
 
-void
-libdeflate_free(void *ptr)
-{
-	(*libdeflate_free_func)(ptr);
-}
-
-void *
-libdeflate_aligned_malloc(size_t alignment, size_t size)
-{
-	void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size);
 	if (ptr) {
 		void *orig_ptr = ptr;
+
 		ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
 		((void **)ptr)[-1] = orig_ptr;
 	}
@@ -64,18 +53,17 @@ libdeflate_aligned_malloc(size_t alignment, size_t size)
 }
 
 void
-libdeflate_aligned_free(void *ptr)
+libdeflate_aligned_free(free_func_t free_func, void *ptr)
 {
-	if (ptr)
-		libdeflate_free(((void **)ptr)[-1]);
+	(*free_func)(((void **)ptr)[-1]);
 }
 
-LIBDEFLATEEXPORT void LIBDEFLATEAPI
-libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
-				void (*free_func)(void *))
+LIBDEFLATEAPI void
+libdeflate_set_memory_allocator(malloc_func_t malloc_func,
+				free_func_t free_func)
 {
-	libdeflate_malloc_func = malloc_func;
-	libdeflate_free_func = free_func;
+	libdeflate_default_malloc_func = malloc_func;
+	libdeflate_default_free_func = free_func;
 }
 
 /*
@@ -144,7 +132,7 @@ memcmp(const void *s1, const void *s2, size_t n)
 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
 #include <stdio.h>
 #include <stdlib.h>
-void
+NORETURN void
 libdeflate_assertion_failed(const char *expr, const char *file, int line)
 {
 	fprintf(stderr, "Assertion failed: %s at %s:%d\n", expr, file, line);

lib/x86/adler32_impl.h

@@ -30,308 +30,106 @@
 
 #include "cpu_features.h"
 
+/* SSE2 and AVX2 implementations.  Used on older CPUs. */
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+#  define adler32_x86_sse2	adler32_x86_sse2
+#  define SUFFIX			   _sse2
+#  define ATTRIBUTES		_target_attribute("sse2")
+#  define VL			16
+#  define USE_VNNI		0
+#  define USE_AVX512		0
+#  include "adler32_template.h"
+
+#  define adler32_x86_avx2	adler32_x86_avx2
+#  define SUFFIX			   _avx2
+#  define ATTRIBUTES		_target_attribute("avx2")
+#  define VL			32
+#  define USE_VNNI		0
+#  define USE_AVX512		0
+#  include "adler32_template.h"
+#endif
+
 /*
- * The following macros horizontally sum the s1 counters and add them to the
- * real s1, and likewise for s2.  They do this via a series of reductions, each
- * of which halves the vector length, until just one counter remains.
+ * AVX-VNNI implementation.  This is used on CPUs that have AVX2 and AVX-VNNI
+ * but don't have AVX-512, for example Intel Alder Lake.
  *
- * The s1 reductions don't depend on the s2 reductions and vice versa, so for
- * efficiency they are interleaved.  Also, every other s1 counter is 0 due to
- * the 'psadbw' instruction (_mm_sad_epu8) summing groups of 8 bytes rather than
- * 4; hence, one of the s1 reductions is skipped when going from 128 => 32 bits.
+ * Unusually for a new CPU feature, gcc added support for the AVX-VNNI
+ * intrinsics (in gcc 11.1) slightly before binutils added support for
+ * assembling AVX-VNNI instructions (in binutils 2.36).  Distros can reasonably
+ * have gcc 11 with binutils 2.35.  Because of this issue, we check for gcc 12
+ * instead of gcc 11.  (libdeflate supports direct compilation without a
+ * configure step, so checking the binutils version is not always an option.)
  */
+#if (GCC_PREREQ(12, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI)
+#  define adler32_x86_avx2_vnni	adler32_x86_avx2_vnni
+#  define SUFFIX			   _avx2_vnni
+#  define ATTRIBUTES		_target_attribute("avx2,avxvnni")
+#  define VL			32
+#  define USE_VNNI		1
+#  define USE_AVX512		0
+#  include "adler32_template.h"
+#endif
 
-#define ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2)		    \
-{									    \
-	__v4su s1_last = (v_s1), s2_last = (v_s2);			    \
-									    \
-	/* 128 => 32 bits */						    \
-	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x31);	    \
-	s1_last += (__v4su)_mm_shuffle_epi32((__m128i)s1_last, 0x02);	    \
-	s2_last += (__v4su)_mm_shuffle_epi32((__m128i)s2_last, 0x02);	    \
-									    \
-	*(s1) += (u32)_mm_cvtsi128_si32((__m128i)s1_last);		    \
-	*(s2) += (u32)_mm_cvtsi128_si32((__m128i)s2_last);		    \
-}
-
-#define ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2)		    \
-{									    \
-	__v4su s1_128bit, s2_128bit;					    \
-									    \
-	/* 256 => 128 bits */						    \
-	s1_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 0) +  \
-		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s1), 1);   \
-	s2_128bit = (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 0) +  \
-		    (__v4su)_mm256_extracti128_si256((__m256i)(v_s2), 1);   \
-									    \
-	ADLER32_FINISH_VEC_CHUNK_128((s1), (s2), s1_128bit, s2_128bit);	    \
-}
-
-#define ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2)		    \
-{									    \
-	__v8su s1_256bit, s2_256bit;					    \
-									    \
-	/* 512 => 256 bits */						    \
-	s1_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 0) + \
-		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s1), 1);  \
-	s2_256bit = (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 0) + \
-		    (__v8su)_mm512_extracti64x4_epi64((__m512i)(v_s2), 1);  \
-									    \
-	ADLER32_FINISH_VEC_CHUNK_256((s1), (s2), s1_256bit, s2_256bit);	    \
-}
-
-/* AVX-512BW implementation: like the AVX2 one, but does 64 bytes at a time */
-#undef DISPATCH_AVX512BW
-#if !defined(DEFAULT_IMPL) &&						\
-    /*
-     * clang before v3.9 is missing some AVX-512BW intrinsics including
-     * _mm512_sad_epu8(), a.k.a. __builtin_ia32_psadbw512.  So just make using
-     * AVX-512BW, even when __AVX512BW__ is defined, conditional on
-     * COMPILER_SUPPORTS_AVX512BW_TARGET where we check for that builtin.
-     */									\
-    COMPILER_SUPPORTS_AVX512BW_TARGET &&				\
-    (defined(__AVX512BW__) || (X86_CPU_FEATURES_ENABLED &&		\
-			       COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS))
-#  define FUNCNAME		adler32_avx512bw
-#  define FUNCNAME_CHUNK	adler32_avx512bw_chunk
-#  define IMPL_ALIGNMENT	64
-#  define IMPL_SEGMENT_SIZE	64
-#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
-#  ifdef __AVX512BW__
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	adler32_avx512bw
-#  else
-#    define ATTRIBUTES		__attribute__((target("avx512bw")))
-#    define DISPATCH		1
-#    define DISPATCH_AVX512BW	1
-#  endif
-#  include <immintrin.h>
-static forceinline ATTRIBUTES void
-adler32_avx512bw_chunk(const __m512i *p, const __m512i *const end,
-		       u32 *s1, u32 *s2)
-{
-	const __m512i zeroes = _mm512_setzero_si512();
-	const __v64qi multipliers = (__v64qi){
-		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
-		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
-		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
-		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
-	};
-	const __v32hi ones = (__v32hi)_mm512_set1_epi16(1);
-	__v16si v_s1 = (__v16si)zeroes;
-	__v16si v_s1_sums = (__v16si)zeroes;
-	__v16si v_s2 = (__v16si)zeroes;
-
-	do {
-		/* Load the next 64-byte segment */
-		__m512i bytes = *p++;
-
-		/* Multiply the bytes by 64...1 (the number of times they need
-		 * to be added to s2) and add adjacent products */
-		__v32hi sums = (__v32hi)_mm512_maddubs_epi16(
-						bytes, (__m512i)multipliers);
-
-		/* Keep sum of all previous s1 counters, for adding to s2 later.
-		 * This allows delaying the multiplication by 64 to the end. */
-		v_s1_sums += v_s1;
-
-		/* Add the sum of each group of 8 bytes to the corresponding s1
-		 * counter */
-		v_s1 += (__v16si)_mm512_sad_epu8(bytes, zeroes);
-
-		/* Add the sum of each group of 4 products of the bytes by
-		 * 64...1 to the corresponding s2 counter */
-		v_s2 += (__v16si)_mm512_madd_epi16((__m512i)sums,
-						   (__m512i)ones);
-	} while (p != end);
-
-	/* Finish the s2 counters by adding the sum of the s1 values at the
-	 * beginning of each segment, multiplied by the segment size (64) */
-	v_s2 += (__v16si)_mm512_slli_epi32((__m512i)v_s1_sums, 6);
-
-	/* Add the counters to the real s1 and s2 */
-	ADLER32_FINISH_VEC_CHUNK_512(s1, s2, v_s1, v_s2);
-}
-#  include "../adler32_vec_template.h"
-#endif /* AVX-512BW implementation */
-
-/* AVX2 implementation: like the AVX-512BW one, but does 32 bytes at a time */
-#undef DISPATCH_AVX2
-#if !defined(DEFAULT_IMPL) &&	\
-	(defined(__AVX2__) || (X86_CPU_FEATURES_ENABLED &&	\
-			       COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS))
-#  define FUNCNAME		adler32_avx2
-#  define FUNCNAME_CHUNK	adler32_avx2_chunk
-#  define IMPL_ALIGNMENT	32
-#  define IMPL_SEGMENT_SIZE	32
-#  define IMPL_MAX_CHUNK_SIZE	MAX_CHUNK_SIZE
-#  ifdef __AVX2__
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	adler32_avx2
-#  else
-#    define ATTRIBUTES		__attribute__((target("avx2")))
-#    define DISPATCH		1
-#    define DISPATCH_AVX2	1
-#  endif
-#  include <immintrin.h>
-static forceinline ATTRIBUTES void
-adler32_avx2_chunk(const __m256i *p, const __m256i *const end, u32 *s1, u32 *s2)
-{
-	const __m256i zeroes = _mm256_setzero_si256();
-	const __v32qu multipliers = (__v32qu){
-		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
-		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
-	};
-	const __v16hu ones = (__v16hu)_mm256_set1_epi16(1);
-	__v8su v_s1 = (__v8su)zeroes;
-	__v8su v_s1_sums = (__v8su)zeroes;
-	__v8su v_s2 = (__v8su)zeroes;
-
-	do {
-		/* Load the next 32-byte segment */
-		__m256i bytes = *p++;
-
-		/* Multiply the bytes by 32...1 (the number of times they need
-		 * to be added to s2) and add adjacent products */
-		__v16hu sums = (__v16hu)_mm256_maddubs_epi16(
-						bytes, (__m256i)multipliers);
-
-		/* Keep sum of all previous s1 counters, for adding to s2 later.
-		 * This allows delaying the multiplication by 32 to the end. */
-		v_s1_sums += v_s1;
-
-		/* Add the sum of each group of 8 bytes to the corresponding s1
-		 * counter */
-		v_s1 += (__v8su)_mm256_sad_epu8(bytes, zeroes);
-
-		/* Add the sum of each group of 4 products of the bytes by
-		 * 32...1 to the corresponding s2 counter */
-		v_s2 += (__v8su)_mm256_madd_epi16((__m256i)sums, (__m256i)ones);
-	} while (p != end);
-
-	/* Finish the s2 counters by adding the sum of the s1 values at the
-	 * beginning of each segment, multiplied by the segment size (32) */
-	v_s2 += (__v8su)_mm256_slli_epi32((__m256i)v_s1_sums, 5);
-
-	/* Add the counters to the real s1 and s2 */
-	ADLER32_FINISH_VEC_CHUNK_256(s1, s2, v_s1, v_s2);
-}
-#  include "../adler32_vec_template.h"
-#endif /* AVX2 implementation */
-
-/* SSE2 implementation */
-#undef DISPATCH_SSE2
-#if !defined(DEFAULT_IMPL) &&	\
-	(defined(__SSE2__) || (X86_CPU_FEATURES_ENABLED &&	\
-			       COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS))
-#  define FUNCNAME		adler32_sse2
-#  define FUNCNAME_CHUNK	adler32_sse2_chunk
-#  define IMPL_ALIGNMENT	16
-#  define IMPL_SEGMENT_SIZE	32
+#if (GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI)
 /*
- * The 16-bit precision byte counters must not be allowed to undergo *signed*
- * overflow, otherwise the signed multiplications at the end (_mm_madd_epi16)
- * would behave incorrectly.
+ * AVX512VNNI implementation using 256-bit vectors.  This is very similar to the
+ * AVX-VNNI implementation but takes advantage of masking and more registers.
+ * This is used on certain older Intel CPUs, specifically Ice Lake and Tiger
+ * Lake, which support AVX512VNNI but downclock a bit too eagerly when ZMM
+ * registers are used.
  */
-#  define IMPL_MAX_CHUNK_SIZE	(32 * (0x7FFF / 0xFF))
-#  ifdef __SSE2__
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	adler32_sse2
-#  else
-#    define ATTRIBUTES		__attribute__((target("sse2")))
-#    define DISPATCH		1
-#    define DISPATCH_SSE2	1
-#  endif
-#  include <emmintrin.h>
-static forceinline ATTRIBUTES void
-adler32_sse2_chunk(const __m128i *p, const __m128i *const end, u32 *s1, u32 *s2)
-{
-	const __m128i zeroes = _mm_setzero_si128();
+#  define adler32_x86_avx512_vl256_vnni	adler32_x86_avx512_vl256_vnni
+#  define SUFFIX				   _avx512_vl256_vnni
+#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vl,avx512vnni" NO_EVEX512)
+#  define VL			32
+#  define USE_VNNI		1
+#  define USE_AVX512		1
+#  include "adler32_template.h"
 
-	/* s1 counters: 32-bit, sum of bytes */
-	__v4su v_s1 = (__v4su)zeroes;
+/*
+ * AVX512VNNI implementation using 512-bit vectors.  This is used on CPUs that
+ * have a good AVX-512 implementation including AVX512VNNI.
+ */
+#  define adler32_x86_avx512_vl512_vnni	adler32_x86_avx512_vl512_vnni
+#  define SUFFIX				   _avx512_vl512_vnni
+#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vnni" EVEX512)
+#  define VL			64
+#  define USE_VNNI		1
+#  define USE_AVX512		1
+#  include "adler32_template.h"
+#endif
 
-	/* s2 counters: 32-bit, sum of s1 values */
-	__v4su v_s2 = (__v4su)zeroes;
-
-	/*
-	 * Thirty-two 16-bit counters for byte sums.  Each accumulates the bytes
-	 * that eventually need to be multiplied by a number 32...1 for addition
-	 * into s2.
-	 */
-	__v8hu v_byte_sums_a = (__v8hu)zeroes;
-	__v8hu v_byte_sums_b = (__v8hu)zeroes;
-	__v8hu v_byte_sums_c = (__v8hu)zeroes;
-	__v8hu v_byte_sums_d = (__v8hu)zeroes;
-
-	do {
-		/* Load the next 32 bytes */
-		const __m128i bytes1 = *p++;
-		const __m128i bytes2 = *p++;
-
-		/*
-		 * Accumulate the previous s1 counters into the s2 counters.
-		 * Logically, this really should be v_s2 += v_s1 * 32, but we
-		 * can do the multiplication (or left shift) later.
-		 */
-		v_s2 += v_s1;
-
-		/*
-		 * s1 update: use "Packed Sum of Absolute Differences" to add
-		 * the bytes horizontally with 8 bytes per sum.  Then add the
-		 * sums to the s1 counters.
-		 */
-		v_s1 += (__v4su)_mm_sad_epu8(bytes1, zeroes);
-		v_s1 += (__v4su)_mm_sad_epu8(bytes2, zeroes);
-
-		/*
-		 * Also accumulate the bytes into 32 separate counters that have
-		 * 16-bit precision.
-		 */
-		v_byte_sums_a += (__v8hu)_mm_unpacklo_epi8(bytes1, zeroes);
-		v_byte_sums_b += (__v8hu)_mm_unpackhi_epi8(bytes1, zeroes);
-		v_byte_sums_c += (__v8hu)_mm_unpacklo_epi8(bytes2, zeroes);
-		v_byte_sums_d += (__v8hu)_mm_unpackhi_epi8(bytes2, zeroes);
-
-	} while (p != end);
-
-	/* Finish calculating the s2 counters */
-	v_s2 = (__v4su)_mm_slli_epi32((__m128i)v_s2, 5);
-	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_a,
-				       (__m128i)(__v8hu){ 32, 31, 30, 29, 28, 27, 26, 25 });
-	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_b,
-				       (__m128i)(__v8hu){ 24, 23, 22, 21, 20, 19, 18, 17 });
-	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_c,
-				       (__m128i)(__v8hu){ 16, 15, 14, 13, 12, 11, 10, 9 });
-	v_s2 += (__v4su)_mm_madd_epi16((__m128i)v_byte_sums_d,
-				       (__m128i)(__v8hu){ 8,  7,  6,  5,  4,  3,  2,  1 });
-
-	/* Add the counters to the real s1 and s2 */
-	ADLER32_FINISH_VEC_CHUNK_128(s1, s2, v_s1, v_s2);
-}
-#  include "../adler32_vec_template.h"
-#endif /* SSE2 implementation */
-
-#ifdef DISPATCH
 static inline adler32_func_t
 arch_select_adler32_func(void)
 {
-	u32 features = get_cpu_features();
+	const u32 features MAYBE_UNUSED = get_x86_cpu_features();
 
-#ifdef DISPATCH_AVX512BW
-	if (features & X86_CPU_FEATURE_AVX512BW)
-		return adler32_avx512bw;
+#ifdef adler32_x86_avx512_vl512_vnni
+	if ((features & X86_CPU_FEATURE_ZMM) &&
+	    HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features))
+		return adler32_x86_avx512_vl512_vnni;
 #endif
-#ifdef DISPATCH_AVX2
-	if (features & X86_CPU_FEATURE_AVX2)
-		return adler32_avx2;
+#ifdef adler32_x86_avx512_vl256_vnni
+	if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) &&
+	    HAVE_AVX512VNNI(features))
+		return adler32_x86_avx512_vl256_vnni;
 #endif
-#ifdef DISPATCH_SSE2
-	if (features & X86_CPU_FEATURE_SSE2)
-		return adler32_sse2;
+#ifdef adler32_x86_avx2_vnni
+	if (HAVE_AVX2(features) && HAVE_AVXVNNI(features))
+		return adler32_x86_avx2_vnni;
+#endif
+#ifdef adler32_x86_avx2
+	if (HAVE_AVX2(features))
+		return adler32_x86_avx2;
+#endif
+#ifdef adler32_x86_sse2
+	if (HAVE_SSE2(features))
+		return adler32_x86_sse2;
 #endif
 	return NULL;
 }
-#endif /* DISPATCH */
+#define arch_select_adler32_func	arch_select_adler32_func
 
 #endif /* LIB_X86_ADLER32_IMPL_H */

lib/x86/adler32_template.h

@@ -0,0 +1,518 @@
+/*
+ * x86/adler32_template.h - template for vectorized Adler-32 implementations
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file is a "template" for instantiating Adler-32 functions for x86.
+ * The "parameters" are:
+ *
+ * SUFFIX:
+ *	Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ *	Target function attributes to use.  Must satisfy the dependencies of the
+ *	other parameters as follows:
+ *	   VL=16 && USE_VNNI=0 && USE_AVX512=0: at least sse2
+ *	   VL=32 && USE_VNNI=0 && USE_AVX512=0: at least avx2
+ *	   VL=32 && USE_VNNI=1 && USE_AVX512=0: at least avx2,avxvnni
+ *	   VL=32 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vl,avx512vnni
+ *	   VL=64 && USE_VNNI=1 && USE_AVX512=1: at least avx512bw,avx512vnni
+ *	   (Other combinations are not useful and have not been tested.)
+ * VL:
+ *	Vector length in bytes.  Must be 16, 32, or 64.
+ * USE_VNNI:
+ *	If 1, use the VNNI dot product based algorithm.
+ *	If 0, use the legacy SSE2 and AVX2 compatible algorithm.
+ * USE_AVX512:
+ *	If 1, take advantage of AVX-512 features such as masking.  This doesn't
+ *	enable the use of 512-bit vectors; the vector length is controlled by
+ *	VL.  If 0, assume that the CPU might not support AVX-512.
+ */
+
+#if VL == 16
+#  define vec_t			__m128i
+#  define mask_t		u16
+#  define LOG2_VL		4
+#  define VADD8(a, b)		_mm_add_epi8((a), (b))
+#  define VADD16(a, b)		_mm_add_epi16((a), (b))
+#  define VADD32(a, b)		_mm_add_epi32((a), (b))
+#  if USE_AVX512
+#    define VDPBUSD(a, b, c)	_mm_dpbusd_epi32((a), (b), (c))
+#  else
+#    define VDPBUSD(a, b, c)	_mm_dpbusd_avx_epi32((a), (b), (c))
+#  endif
+#  define VLOAD(p)		_mm_load_si128((const void *)(p))
+#  define VLOADU(p)		_mm_loadu_si128((const void *)(p))
+#  define VMADD16(a, b)		_mm_madd_epi16((a), (b))
+#  define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
+#  define VMULLO32(a, b)	_mm_mullo_epi32((a), (b))
+#  define VSAD8(a, b)		_mm_sad_epu8((a), (b))
+#  define VSET1_8(a)		_mm_set1_epi8(a)
+#  define VSET1_32(a)		_mm_set1_epi32(a)
+#  define VSETZERO()		_mm_setzero_si128()
+#  define VSLL32(a, b)		_mm_slli_epi32((a), (b))
+#  define VUNPACKLO8(a, b)	_mm_unpacklo_epi8((a), (b))
+#  define VUNPACKHI8(a, b)	_mm_unpackhi_epi8((a), (b))
+#elif VL == 32
+#  define vec_t			__m256i
+#  define mask_t		u32
+#  define LOG2_VL		5
+#  define VADD8(a, b)		_mm256_add_epi8((a), (b))
+#  define VADD16(a, b)		_mm256_add_epi16((a), (b))
+#  define VADD32(a, b)		_mm256_add_epi32((a), (b))
+#  if USE_AVX512
+#    define VDPBUSD(a, b, c)	_mm256_dpbusd_epi32((a), (b), (c))
+#  else
+#    define VDPBUSD(a, b, c)	_mm256_dpbusd_avx_epi32((a), (b), (c))
+#  endif
+#  define VLOAD(p)		_mm256_load_si256((const void *)(p))
+#  define VLOADU(p)		_mm256_loadu_si256((const void *)(p))
+#  define VMADD16(a, b)		_mm256_madd_epi16((a), (b))
+#  define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
+#  define VMULLO32(a, b)	_mm256_mullo_epi32((a), (b))
+#  define VSAD8(a, b)		_mm256_sad_epu8((a), (b))
+#  define VSET1_8(a)		_mm256_set1_epi8(a)
+#  define VSET1_32(a)		_mm256_set1_epi32(a)
+#  define VSETZERO()		_mm256_setzero_si256()
+#  define VSLL32(a, b)		_mm256_slli_epi32((a), (b))
+#  define VUNPACKLO8(a, b)	_mm256_unpacklo_epi8((a), (b))
+#  define VUNPACKHI8(a, b)	_mm256_unpackhi_epi8((a), (b))
+#elif VL == 64
+#  define vec_t			__m512i
+#  define mask_t		u64
+#  define LOG2_VL		6
+#  define VADD8(a, b)		_mm512_add_epi8((a), (b))
+#  define VADD16(a, b)		_mm512_add_epi16((a), (b))
+#  define VADD32(a, b)		_mm512_add_epi32((a), (b))
+#  define VDPBUSD(a, b, c)	_mm512_dpbusd_epi32((a), (b), (c))
+#  define VLOAD(p)		_mm512_load_si512((const void *)(p))
+#  define VLOADU(p)		_mm512_loadu_si512((const void *)(p))
+#  define VMADD16(a, b)		_mm512_madd_epi16((a), (b))
+#  define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
+#  define VMULLO32(a, b)	_mm512_mullo_epi32((a), (b))
+#  define VSAD8(a, b)		_mm512_sad_epu8((a), (b))
+#  define VSET1_8(a)		_mm512_set1_epi8(a)
+#  define VSET1_32(a)		_mm512_set1_epi32(a)
+#  define VSETZERO()		_mm512_setzero_si512()
+#  define VSLL32(a, b)		_mm512_slli_epi32((a), (b))
+#  define VUNPACKLO8(a, b)	_mm512_unpacklo_epi8((a), (b))
+#  define VUNPACKHI8(a, b)	_mm512_unpackhi_epi8((a), (b))
+#else
+#  error "unsupported vector length"
+#endif
+
+#define VADD32_3X(a, b, c)	VADD32(VADD32((a), (b)), (c))
+#define VADD32_4X(a, b, c, d)	VADD32(VADD32((a), (b)), VADD32((c), (d)))
+#define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
+#define VADD32_7X(a, b, c, d, e, f, g)	\
+	VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
+
+/* Sum the 32-bit elements of v_s1 and add them to s1, and likewise for s2. */
+#undef reduce_to_32bits
+static forceinline ATTRIBUTES void
+ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
+{
+	__m128i v_s1_128, v_s2_128;
+#if VL == 16
+	{
+		v_s1_128 = v_s1;
+		v_s2_128 = v_s2;
+	}
+#else
+	{
+		__m256i v_s1_256, v_s2_256;
+	#if VL == 32
+		v_s1_256 = v_s1;
+		v_s2_256 = v_s2;
+	#else
+		/* Reduce 512 bits to 256 bits. */
+		v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
+					    _mm512_extracti64x4_epi64(v_s1, 1));
+		v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
+					    _mm512_extracti64x4_epi64(v_s2, 1));
+	#endif
+		/* Reduce 256 bits to 128 bits. */
+		v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
+					 _mm256_extracti128_si256(v_s1_256, 1));
+		v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
+					 _mm256_extracti128_si256(v_s2_256, 1));
+	}
+#endif
+
+	/*
+	 * Reduce 128 bits to 32 bits.
+	 *
+	 * If the bytes were summed into v_s1 using psadbw + paddd, then ignore
+	 * the odd-indexed elements of v_s1_128 since they are zero.
+	 */
+#if USE_VNNI
+	v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
+#endif
+	v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
+	v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
+	v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
+
+	*s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
+	*s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
+}
+#define reduce_to_32bits	ADD_SUFFIX(reduce_to_32bits)
+
+static ATTRIBUTES u32
+ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
+{
+#if USE_VNNI
+	/* This contains the bytes [VL, VL-1, VL-2, ..., 1]. */
+	static const u8 _aligned_attribute(VL) raw_mults[VL] = {
+	#if VL == 64
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+	#endif
+	#if VL >= 32
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+	#endif
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const vec_t ones = VSET1_8(1);
+#else
+	/*
+	 * This contains the 16-bit values [2*VL, 2*VL - 1, 2*VL - 2, ..., 1].
+	 * For VL==32 the ordering is weird because it has to match the way that
+	 * vpunpcklbw and vpunpckhbw work on 128-bit lanes separately.
+	 */
+	static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
+	#if VL == 16
+		{ 32, 31, 30, 29, 28, 27, 26, 25 },
+		{ 24, 23, 22, 21, 20, 19, 18, 17 },
+		{ 16, 15, 14, 13, 12, 11, 10, 9  },
+		{ 8,  7,  6,  5,  4,  3,  2,  1  },
+	#elif VL == 32
+		{ 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
+		{ 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
+		{ 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10,  9 },
+		{ 24, 23, 22, 21, 20, 19, 18, 17,  8,  7,  6,  5,  4,  3,  2,  1 },
+	#else
+	#  error "unsupported parameters"
+	#endif
+	};
+	const vec_t mults_a = VLOAD(raw_mults[0]);
+	const vec_t mults_b = VLOAD(raw_mults[1]);
+	const vec_t mults_c = VLOAD(raw_mults[2]);
+	const vec_t mults_d = VLOAD(raw_mults[3]);
+#endif
+	const vec_t zeroes = VSETZERO();
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+
+	/*
+	 * If the length is large and the pointer is misaligned, align it.
+	 * For smaller lengths, just take the misaligned load penalty.
+	 */
+	if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+			len--;
+		} while ((uintptr_t)p & (VL-1));
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+#if USE_VNNI
+	/*
+	 * This is Adler-32 using the vpdpbusd instruction from AVX512VNNI or
+	 * AVX-VNNI.  vpdpbusd multiplies the unsigned bytes of one vector by
+	 * the signed bytes of another vector and adds the sums in groups of 4
+	 * to the 32-bit elements of a third vector.  We use it in two ways:
+	 * multiplying the data bytes by a sequence like 64,63,62,...,1 for
+	 * calculating part of s2, and multiplying the data bytes by an all-ones
+	 * sequence 1,1,1,...,1 for calculating s1 and part of s2.  The all-ones
+	 * trick seems to be faster than the alternative of vpsadbw + vpaddd.
+	 */
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX.
+		 */
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
+		vec_t mults = VLOAD(raw_mults);
+		vec_t v_s1 = zeroes;
+		vec_t v_s2 = zeroes;
+
+		s2 += s1 * n;
+		len -= n;
+
+		if (n >= 4*VL) {
+			vec_t v_s1_b = zeroes;
+			vec_t v_s1_c = zeroes;
+			vec_t v_s1_d = zeroes;
+			vec_t v_s2_b = zeroes;
+			vec_t v_s2_c = zeroes;
+			vec_t v_s2_d = zeroes;
+			vec_t v_s1_sums   = zeroes;
+			vec_t v_s1_sums_b = zeroes;
+			vec_t v_s1_sums_c = zeroes;
+			vec_t v_s1_sums_d = zeroes;
+			vec_t tmp0, tmp1;
+
+			do {
+				vec_t data_a = VLOADU(p + 0*VL);
+				vec_t data_b = VLOADU(p + 1*VL);
+				vec_t data_c = VLOADU(p + 2*VL);
+				vec_t data_d = VLOADU(p + 3*VL);
+
+				/*
+				 * Workaround for gcc bug where it generates
+				 * unnecessary move instructions
+				 * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
+				 */
+			#if GCC_PREREQ(1, 0)
+				__asm__("" : "+v" (data_a), "+v" (data_b),
+					     "+v" (data_c), "+v" (data_d));
+			#endif
+
+				v_s2   = VDPBUSD(v_s2,   data_a, mults);
+				v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
+				v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
+				v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
+
+				v_s1_sums   = VADD32(v_s1_sums,   v_s1);
+				v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
+				v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
+				v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
+
+				v_s1   = VDPBUSD(v_s1,   data_a, ones);
+				v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
+				v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
+				v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
+
+				/* Same gcc bug workaround.  See above */
+			#if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
+				__asm__("" : "+v" (v_s2), "+v" (v_s2_b),
+					     "+v" (v_s2_c), "+v" (v_s2_d),
+					     "+v" (v_s1_sums),
+					     "+v" (v_s1_sums_b),
+					     "+v" (v_s1_sums_c),
+					     "+v" (v_s1_sums_d),
+					     "+v" (v_s1), "+v" (v_s1_b),
+					     "+v" (v_s1_c), "+v" (v_s1_d));
+			#endif
+				p += 4*VL;
+				n -= 4*VL;
+			} while (n >= 4*VL);
+
+			/*
+			 * Reduce into v_s1 and v_s2 as follows:
+			 *
+			 * v_s2 = v_s2 + v_s2_b + v_s2_c + v_s2_d +
+			 *	  (4*VL)*(v_s1_sums   + v_s1_sums_b +
+			 *		  v_s1_sums_c + v_s1_sums_d) +
+			 *	  (3*VL)*v_s1 + (2*VL)*v_s1_b + VL*v_s1_c
+			 * v_s1 = v_s1 + v_s1_b + v_s1_c + v_s1_d
+			 */
+			tmp0 = VADD32(v_s1, v_s1_b);
+			tmp1 = VADD32(v_s1, v_s1_c);
+			v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
+					      v_s1_sums_c, v_s1_sums_d);
+			v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
+			v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
+					 VSLL32(tmp0, LOG2_VL + 1),
+					 VSLL32(tmp1, LOG2_VL),
+					 v_s2, v_s2_b, v_s2_c, v_s2_d);
+		}
+
+		/* Process the last 0 <= n < 4*VL bytes of the chunk. */
+		if (n >= 2*VL) {
+			const vec_t data_a = VLOADU(p + 0*VL);
+			const vec_t data_b = VLOADU(p + 1*VL);
+
+			v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
+			v_s1 = VDPBUSD(v_s1, data_a, ones);
+			v_s1 = VDPBUSD(v_s1, data_b, ones);
+			v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
+			v_s2 = VDPBUSD(v_s2, data_a, mults);
+			v_s2 = VDPBUSD(v_s2, data_b, mults);
+			p += 2*VL;
+			n -= 2*VL;
+		}
+		if (n) {
+			/* Process the last 0 < n < 2*VL bytes of the chunk. */
+			vec_t data;
+
+			v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
+
+			mults = VADD8(mults, VSET1_8((int)n - VL));
+			if (n > VL) {
+				data = VLOADU(p);
+				v_s1 = VDPBUSD(v_s1, data, ones);
+				v_s2 = VDPBUSD(v_s2, data, mults);
+				p += VL;
+				n -= VL;
+				mults = VADD8(mults, VSET1_8(-VL));
+			}
+			/*
+			 * Process the last 0 < n <= VL bytes of the chunk.
+			 * Utilize a masked load if it's available.
+			 */
+		#if USE_AVX512
+			data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
+		#else
+			data = zeroes;
+			memcpy(&data, p, n);
+		#endif
+			v_s1 = VDPBUSD(v_s1, data, ones);
+			v_s2 = VDPBUSD(v_s2, data, mults);
+			p += n;
+		}
+
+		reduce_to_32bits(v_s1, v_s2, &s1, &s2);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+#else /* USE_VNNI */
+	/*
+	 * This is Adler-32 for SSE2 and AVX2.
+	 *
+	 * To horizontally sum bytes, use psadbw + paddd, where one of the
+	 * arguments to psadbw is all-zeroes.
+	 *
+	 * For the s2 contribution from (2*VL - i)*data[i] for each of the 2*VL
+	 * bytes of each iteration of the inner loop, use punpck{l,h}bw + paddw
+	 * to sum, for each i across iterations, byte i into a corresponding
+	 * 16-bit counter in v_byte_sums_*.  After the inner loop, use pmaddwd
+	 * to multiply each counter by (2*VL - i), then add the products to s2.
+	 *
+	 * An alternative implementation would use pmaddubsw and pmaddwd in the
+	 * inner loop to do (2*VL - i)*data[i] directly and add the products in
+	 * groups of 4 to 32-bit counters.  However, on average that approach
+	 * seems to be slower than the current approach which delays the
+	 * multiplications.  Also, pmaddubsw requires SSSE3; the current
+	 * approach keeps the implementation aligned between SSE2 and AVX2.
+	 *
+	 * The inner loop processes 2*VL bytes per iteration.  Increasing this
+	 * to 4*VL doesn't seem to be helpful here.
+	 */
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX, and every
+		 * v_byte_sums_* counter is guaranteed to not exceed INT16_MAX.
+		 * It's INT16_MAX, not UINT16_MAX, because v_byte_sums_* are
+		 * used with pmaddwd which does signed multiplication.  In the
+		 * SSE2 case this limits chunks to 4096 bytes instead of 5536.
+		 */
+		size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
+					MAX_CHUNK_LEN) & ~(2*VL - 1));
+		len -= n;
+
+		if (n >= 2*VL) {
+			vec_t v_s1 = zeroes;
+			vec_t v_s1_sums = zeroes;
+			vec_t v_byte_sums_a = zeroes;
+			vec_t v_byte_sums_b = zeroes;
+			vec_t v_byte_sums_c = zeroes;
+			vec_t v_byte_sums_d = zeroes;
+			vec_t v_s2;
+
+			s2 += s1 * (n & ~(2*VL - 1));
+
+			do {
+				vec_t data_a = VLOADU(p + 0*VL);
+				vec_t data_b = VLOADU(p + 1*VL);
+
+				v_s1_sums = VADD32(v_s1_sums, v_s1);
+				v_byte_sums_a = VADD16(v_byte_sums_a,
+						       VUNPACKLO8(data_a, zeroes));
+				v_byte_sums_b = VADD16(v_byte_sums_b,
+						       VUNPACKHI8(data_a, zeroes));
+				v_byte_sums_c = VADD16(v_byte_sums_c,
+						       VUNPACKLO8(data_b, zeroes));
+				v_byte_sums_d = VADD16(v_byte_sums_d,
+						       VUNPACKHI8(data_b, zeroes));
+				v_s1 = VADD32(v_s1,
+					      VADD32(VSAD8(data_a, zeroes),
+						     VSAD8(data_b, zeroes)));
+				/*
+				 * Workaround for gcc bug where it generates
+				 * unnecessary move instructions
+				 * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107892)
+				 */
+			#if GCC_PREREQ(1, 0)
+				__asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
+					     "+x" (v_byte_sums_a),
+					     "+x" (v_byte_sums_b),
+					     "+x" (v_byte_sums_c),
+					     "+x" (v_byte_sums_d));
+			#endif
+				p += 2*VL;
+				n -= 2*VL;
+			} while (n >= 2*VL);
+
+			/*
+			 * Calculate v_s2 as (2*VL)*v_s1_sums +
+			 * [2*VL, 2*VL - 1, 2*VL - 2, ..., 1] * v_byte_sums.
+			 * Then update s1 and s2 from v_s1 and v_s2.
+			 */
+			v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
+					 VMADD16(v_byte_sums_a, mults_a),
+					 VMADD16(v_byte_sums_b, mults_b),
+					 VMADD16(v_byte_sums_c, mults_c),
+					 VMADD16(v_byte_sums_d, mults_d));
+			reduce_to_32bits(v_s1, v_s2, &s1, &s2);
+		}
+		/*
+		 * Process the last 0 <= n < 2*VL bytes of the chunk using
+		 * scalar instructions and reduce s1 and s2 mod DIVISOR.
+		 */
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+#endif /* !USE_VNNI */
+	return (s2 << 16) | s1;
+}
+
+#undef vec_t
+#undef mask_t
+#undef LOG2_VL
+#undef VADD8
+#undef VADD16
+#undef VADD32
+#undef VDPBUSD
+#undef VLOAD
+#undef VLOADU
+#undef VMADD16
+#undef VMASKZ_LOADU
+#undef VMULLO32
+#undef VSAD8
+#undef VSET1_8
+#undef VSET1_32
+#undef VSETZERO
+#undef VSLL32
+#undef VUNPACKLO8
+#undef VUNPACKHI8
+
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef VL
+#undef USE_VNNI
+#undef USE_AVX512

lib/x86/cpu_features.c

@@ -1,5 +1,5 @@
 /*
- * x86/cpu_features.c - feature detection for x86 processors
+ * x86/cpu_features.c - feature detection for x86 CPUs
  *
  * Copyright 2016 Eric Biggers
  *
@@ -28,125 +28,186 @@
 #include "../cpu_features_common.h" /* must be included first */
 #include "cpu_features.h"
 
-#if X86_CPU_FEATURES_ENABLED
+#ifdef X86_CPU_FEATURES_KNOWN
+/* Runtime x86 CPU feature detection is supported. */
 
-volatile u32 _cpu_features = 0;
-
-/* With old GCC versions we have to manually save and restore the x86_32 PIC
- * register (ebx).  See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47602  */
-#if defined(__i386__) && defined(__PIC__)
-#  define EBX_CONSTRAINT "=&r"
-#else
-#  define EBX_CONSTRAINT "=b"
-#endif
-
-/* Execute the CPUID instruction.  */
+/* Execute the CPUID instruction. */
 static inline void
 cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
 {
-	__asm__(".ifnc %%ebx, %1; mov  %%ebx, %1; .endif\n"
-		"cpuid                                  \n"
-		".ifnc %%ebx, %1; xchg %%ebx, %1; .endif\n"
-		: "=a" (*a), EBX_CONSTRAINT (*b), "=c" (*c), "=d" (*d)
-		: "a" (leaf), "c" (subleaf));
+#ifdef _MSC_VER
+	int result[4];
+
+	__cpuidex(result, leaf, subleaf);
+	*a = result[0];
+	*b = result[1];
+	*c = result[2];
+	*d = result[3];
+#else
+	__asm__ volatile("cpuid" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
+			 : "a" (leaf), "c" (subleaf));
+#endif
 }
 
-/* Read an extended control register.  */
+/* Read an extended control register. */
 static inline u64
 read_xcr(u32 index)
 {
-	u32 edx, eax;
+#ifdef _MSC_VER
+	return _xgetbv(index);
+#else
+	u32 d, a;
 
-	/* Execute the "xgetbv" instruction.  Old versions of binutils do not
-	 * recognize this instruction, so list the raw bytes instead.  */
-	__asm__ (".byte 0x0f, 0x01, 0xd0" : "=d" (edx), "=a" (eax) : "c" (index));
+	/*
+	 * Execute the "xgetbv" instruction.  Old versions of binutils do not
+	 * recognize this instruction, so list the raw bytes instead.
+	 *
+	 * This must be 'volatile' to prevent this code from being moved out
+	 * from under the check for OSXSAVE.
+	 */
+	__asm__ volatile(".byte 0x0f, 0x01, 0xd0" :
+			 "=d" (d), "=a" (a) : "c" (index));
 
-	return ((u64)edx << 32) | eax;
+	return ((u64)d << 32) | a;
+#endif
 }
 
-#undef BIT
-#define BIT(nr)			(1UL << (nr))
-
-#define XCR0_BIT_SSE		BIT(1)
-#define XCR0_BIT_AVX		BIT(2)
-#define XCR0_BIT_OPMASK		BIT(5)
-#define XCR0_BIT_ZMM_HI256	BIT(6)
-#define XCR0_BIT_HI16_ZMM	BIT(7)
-
-#define IS_SET(reg, nr)		((reg) & BIT(nr))
-#define IS_ALL_SET(reg, mask)	(((reg) & (mask)) == (mask))
-
 static const struct cpu_feature x86_cpu_feature_table[] = {
 	{X86_CPU_FEATURE_SSE2,		"sse2"},
-	{X86_CPU_FEATURE_PCLMUL,	"pclmul"},
+	{X86_CPU_FEATURE_PCLMULQDQ,	"pclmulqdq"},
 	{X86_CPU_FEATURE_AVX,		"avx"},
 	{X86_CPU_FEATURE_AVX2,		"avx2"},
 	{X86_CPU_FEATURE_BMI2,		"bmi2"},
+	{X86_CPU_FEATURE_ZMM,		"zmm"},
 	{X86_CPU_FEATURE_AVX512BW,	"avx512bw"},
+	{X86_CPU_FEATURE_AVX512VL,	"avx512vl"},
+	{X86_CPU_FEATURE_VPCLMULQDQ,	"vpclmulqdq"},
+	{X86_CPU_FEATURE_AVX512VNNI,	"avx512_vnni"},
+	{X86_CPU_FEATURE_AVXVNNI,	"avx_vnni"},
 };
 
-/* Initialize _cpu_features with bits for interesting processor features. */
-void setup_cpu_features(void)
-{
-	u32 features = 0;
-	u32 dummy1, dummy2, dummy3, dummy4;
-	u32 max_function;
-	u32 features_1, features_2, features_3, features_4;
-	bool os_avx_support = false;
-	bool os_avx512_support = false;
+volatile u32 libdeflate_x86_cpu_features = 0;
 
-	/* Get maximum supported function  */
-	cpuid(0, 0, &max_function, &dummy2, &dummy3, &dummy4);
-	if (max_function < 1)
+static inline bool
+os_supports_avx512(u64 xcr0)
+{
+#ifdef __APPLE__
+	/*
+	 * The Darwin kernel had a bug where it could corrupt the opmask
+	 * registers.  See
+	 * https://community.intel.com/t5/Software-Tuning-Performance/MacOS-Darwin-kernel-bug-clobbers-AVX-512-opmask-register-state/m-p/1327259
+	 * Darwin also does not initially set the XCR0 bits for AVX512, but they
+	 * are set if the thread tries to use AVX512 anyway.  Thus, to safely
+	 * and consistently use AVX512 on macOS we'd need to check the kernel
+	 * version as well as detect AVX512 support using a macOS-specific
+	 * method.  We don't bother with this, especially given Apple's
+	 * transition to arm64.
+	 */
+	return false;
+#else
+	return (xcr0 & 0xe6) == 0xe6;
+#endif
+}
+
+/*
+ * Don't use 512-bit vectors (ZMM registers) on Intel CPUs before Rocket Lake
+ * and Sapphire Rapids, due to the overly-eager downclocking which can reduce
+ * the performance of workloads that use ZMM registers only occasionally.
+ */
+static inline bool
+allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
+{
+#ifdef TEST_SUPPORT__DO_NOT_USE
+	return true;
+#endif
+	if (memcmp(manufacturer, "GenuineIntel", 12) != 0)
+		return true;
+	if (family != 6)
+		return true;
+	switch (model) {
+	case 85: /* Skylake (Server), Cascade Lake, Cooper Lake */
+	case 106: /* Ice Lake (Server) */
+	case 108: /* Ice Lake (Server) */
+	case 126: /* Ice Lake (Client) */
+	case 140: /* Tiger Lake */
+	case 141: /* Tiger Lake */
+		return false;
+	}
+	return true;
+}
+
+/* Initialize libdeflate_x86_cpu_features. */
+void libdeflate_init_x86_cpu_features(void)
+{
+	u32 max_leaf;
+	u32 manufacturer[3];
+	u32 family, model;
+	u32 a, b, c, d;
+	u64 xcr0 = 0;
+	u32 features = 0;
+
+	/* EAX=0: Highest Function Parameter and Manufacturer ID */
+	cpuid(0, 0, &max_leaf, &manufacturer[0], &manufacturer[2],
+	      &manufacturer[1]);
+	if (max_leaf < 1)
 		goto out;
 
-	/* Standard feature flags  */
-	cpuid(1, 0, &dummy1, &dummy2, &features_2, &features_1);
-
-	if (IS_SET(features_1, 26))
+	/* EAX=1: Processor Info and Feature Bits */
+	cpuid(1, 0, &a, &b, &c, &d);
+	family = (a >> 8) & 0xf;
+	model = (a >> 4) & 0xf;
+	if (family == 6 || family == 0xf)
+		model += (a >> 12) & 0xf0;
+	if (family == 0xf)
+		family += (a >> 20) & 0xff;
+	if (d & (1 << 26))
 		features |= X86_CPU_FEATURE_SSE2;
-
-	if (IS_SET(features_2, 1))
-		features |= X86_CPU_FEATURE_PCLMUL;
-
-	if (IS_SET(features_2, 27)) { /* OSXSAVE set? */
-		u64 xcr0 = read_xcr(0);
-
-		os_avx_support = IS_ALL_SET(xcr0,
-					    XCR0_BIT_SSE |
-					    XCR0_BIT_AVX);
-
-		os_avx512_support = IS_ALL_SET(xcr0,
-					       XCR0_BIT_SSE |
-					       XCR0_BIT_AVX |
-					       XCR0_BIT_OPMASK |
-					       XCR0_BIT_ZMM_HI256 |
-					       XCR0_BIT_HI16_ZMM);
-	}
-
-	if (os_avx_support && IS_SET(features_2, 28))
+	/*
+	 * No known CPUs have pclmulqdq without sse4.1, so in practice code
+	 * targeting pclmulqdq can use sse4.1 instructions.  But to be safe,
+	 * explicitly check for both the pclmulqdq and sse4.1 bits.
+	 */
+	if ((c & (1 << 1)) && (c & (1 << 19)))
+		features |= X86_CPU_FEATURE_PCLMULQDQ;
+	if (c & (1 << 27))
+		xcr0 = read_xcr(0);
+	if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6))
 		features |= X86_CPU_FEATURE_AVX;
 
-	if (max_function < 7)
+	if (max_leaf < 7)
 		goto out;
 
-	/* Extended feature flags  */
-	cpuid(7, 0, &dummy1, &features_3, &features_4, &dummy4);
-
-	if (os_avx_support && IS_SET(features_3, 5))
-		features |= X86_CPU_FEATURE_AVX2;
-
-	if (IS_SET(features_3, 8))
+	/* EAX=7, ECX=0: Extended Features */
+	cpuid(7, 0, &a, &b, &c, &d);
+	if (b & (1 << 8))
 		features |= X86_CPU_FEATURE_BMI2;
+	if ((xcr0 & 0x6) == 0x6) {
+		if (b & (1 << 5))
+			features |= X86_CPU_FEATURE_AVX2;
+		if (c & (1 << 10))
+			features |= X86_CPU_FEATURE_VPCLMULQDQ;
+	}
+	if (os_supports_avx512(xcr0)) {
+		if (allow_512bit_vectors(manufacturer, family, model))
+			features |= X86_CPU_FEATURE_ZMM;
+		if (b & (1 << 30))
+			features |= X86_CPU_FEATURE_AVX512BW;
+		if (b & (1U << 31))
+			features |= X86_CPU_FEATURE_AVX512VL;
+		if (c & (1 << 11))
+			features |= X86_CPU_FEATURE_AVX512VNNI;
+	}
 
-	if (os_avx512_support && IS_SET(features_3, 30))
-		features |= X86_CPU_FEATURE_AVX512BW;
+	/* EAX=7, ECX=1: Extended Features */
+	cpuid(7, 1, &a, &b, &c, &d);
+	if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6))
+		features |= X86_CPU_FEATURE_AVXVNNI;
 
 out:
 	disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
 					 ARRAY_LEN(x86_cpu_feature_table));
 
-	_cpu_features = features | X86_CPU_FEATURES_KNOWN;
+	libdeflate_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
 }
 
-#endif /* X86_CPU_FEATURES_ENABLED */
+#endif /* X86_CPU_FEATURES_KNOWN */

lib/x86/cpu_features.h

@@ -1,5 +1,28 @@
 /*
- * x86/cpu_features.h - feature detection for x86 processors
+ * x86/cpu_features.h - feature detection for x86 CPUs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #ifndef LIB_X86_CPU_FEATURES_H
@@ -7,35 +30,150 @@
 
 #include "../lib_common.h"
 
-#if (defined(__i386__) || defined(__x86_64__)) && \
-	COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-#  define X86_CPU_FEATURES_ENABLED 1
+#if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+
+#define X86_CPU_FEATURE_SSE2		(1 << 0)
+#define X86_CPU_FEATURE_PCLMULQDQ	(1 << 1)
+#define X86_CPU_FEATURE_AVX		(1 << 2)
+#define X86_CPU_FEATURE_AVX2		(1 << 3)
+#define X86_CPU_FEATURE_BMI2		(1 << 4)
+/*
+ * ZMM indicates whether 512-bit vectors (zmm registers) should be used.  On
+ * some CPUs, to avoid downclocking issues we don't set ZMM even if the CPU and
+ * operating system support AVX-512.  On these CPUs, we may still use AVX-512
+ * instructions, but only with xmm and ymm registers.
+ */
+#define X86_CPU_FEATURE_ZMM		(1 << 5)
+#define X86_CPU_FEATURE_AVX512BW	(1 << 6)
+#define X86_CPU_FEATURE_AVX512VL	(1 << 7)
+#define X86_CPU_FEATURE_VPCLMULQDQ	(1 << 8)
+#define X86_CPU_FEATURE_AVX512VNNI	(1 << 9)
+#define X86_CPU_FEATURE_AVXVNNI		(1 << 10)
+
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+/* Runtime x86 CPU feature detection is supported. */
+#  define X86_CPU_FEATURES_KNOWN	(1U << 31)
+extern volatile u32 libdeflate_x86_cpu_features;
+
+void libdeflate_init_x86_cpu_features(void);
+
+static inline u32 get_x86_cpu_features(void)
+{
+	if (libdeflate_x86_cpu_features == 0)
+		libdeflate_init_x86_cpu_features();
+	return libdeflate_x86_cpu_features;
+}
+/*
+ * x86 intrinsics are also supported.  Include the headers needed to use them.
+ * Normally just immintrin.h suffices.  With clang in MSVC compatibility mode,
+ * immintrin.h incorrectly skips including sub-headers, so include those too.
+ */
+#  include <immintrin.h>
+#  if defined(_MSC_VER) && defined(__clang__)
+#    include <tmmintrin.h>
+#    include <smmintrin.h>
+#    include <wmmintrin.h>
+#    include <avxintrin.h>
+#    include <avx2intrin.h>
+#    include <avx512fintrin.h>
+#    include <avx512bwintrin.h>
+#    include <avx512vlintrin.h>
+#    if __has_include(<avx512vlbwintrin.h>)
+#      include <avx512vlbwintrin.h>
+#    endif
+#    if __has_include(<vpclmulqdqintrin.h>)
+#      include <vpclmulqdqintrin.h>
+#    endif
+#    if __has_include(<avx512vnniintrin.h>)
+#      include <avx512vnniintrin.h>
+#    endif
+#    if __has_include(<avx512vlvnniintrin.h>)
+#      include <avx512vlvnniintrin.h>
+#    endif
+#    if __has_include(<avxvnniintrin.h>)
+#      include <avxvnniintrin.h>
+#    endif
+#  endif
 #else
-#  define X86_CPU_FEATURES_ENABLED 0
+static inline u32 get_x86_cpu_features(void) { return 0; }
 #endif
 
-#if X86_CPU_FEATURES_ENABLED
+#if defined(__SSE2__) || \
+	(defined(_MSC_VER) && \
+	 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
+#  define HAVE_SSE2(features)		1
+#  define HAVE_SSE2_NATIVE		1
+#else
+#  define HAVE_SSE2(features)		((features) & X86_CPU_FEATURE_SSE2)
+#  define HAVE_SSE2_NATIVE		0
+#endif
 
-#define X86_CPU_FEATURE_SSE2		0x00000001
-#define X86_CPU_FEATURE_PCLMUL		0x00000002
-#define X86_CPU_FEATURE_AVX		0x00000004
-#define X86_CPU_FEATURE_AVX2		0x00000008
-#define X86_CPU_FEATURE_BMI2		0x00000010
-#define X86_CPU_FEATURE_AVX512BW	0x00000020
+#if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
+	(defined(_MSC_VER) && defined(__AVX2__))
+#  define HAVE_PCLMULQDQ(features)	1
+#else
+#  define HAVE_PCLMULQDQ(features)	((features) & X86_CPU_FEATURE_PCLMULQDQ)
+#endif
 
-#define X86_CPU_FEATURES_KNOWN		0x80000000
+#ifdef __AVX__
+#  define HAVE_AVX(features)		1
+#else
+#  define HAVE_AVX(features)		((features) & X86_CPU_FEATURE_AVX)
+#endif
 
-extern volatile u32 _cpu_features;
+#ifdef __AVX2__
+#  define HAVE_AVX2(features)		1
+#else
+#  define HAVE_AVX2(features)		((features) & X86_CPU_FEATURE_AVX2)
+#endif
 
-void setup_cpu_features(void);
+#if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
+#  define HAVE_BMI2(features)		1
+#  define HAVE_BMI2_NATIVE		1
+#else
+#  define HAVE_BMI2(features)		((features) & X86_CPU_FEATURE_BMI2)
+#  define HAVE_BMI2_NATIVE		0
+#endif
 
-static inline u32 get_cpu_features(void)
-{
-	if (_cpu_features == 0)
-		setup_cpu_features();
-	return _cpu_features;
-}
+#ifdef __AVX512BW__
+#  define HAVE_AVX512BW(features)	1
+#else
+#  define HAVE_AVX512BW(features)	((features) & X86_CPU_FEATURE_AVX512BW)
+#endif
 
-#endif /* X86_CPU_FEATURES_ENABLED */
+#ifdef __AVX512VL__
+#  define HAVE_AVX512VL(features)	1
+#else
+#  define HAVE_AVX512VL(features)	((features) & X86_CPU_FEATURE_AVX512VL)
+#endif
+
+#ifdef __VPCLMULQDQ__
+#  define HAVE_VPCLMULQDQ(features)	1
+#else
+#  define HAVE_VPCLMULQDQ(features)	((features) & X86_CPU_FEATURE_VPCLMULQDQ)
+#endif
+
+#ifdef __AVX512VNNI__
+#  define HAVE_AVX512VNNI(features)	1
+#else
+#  define HAVE_AVX512VNNI(features)	((features) & X86_CPU_FEATURE_AVX512VNNI)
+#endif
+
+#ifdef __AVXVNNI__
+#  define HAVE_AVXVNNI(features)	1
+#else
+#  define HAVE_AVXVNNI(features)	((features) & X86_CPU_FEATURE_AVXVNNI)
+#endif
+
+#if (GCC_PREREQ(14, 0) || CLANG_PREREQ(18, 0, 18000000)) \
+	&& !defined(__EVEX512__) /* avoid subtracting the evex512 feature */
+#  define EVEX512	",evex512"	/* needed to override potential -mno-evex512 */
+#  define NO_EVEX512	",no-evex512"
+#else
+#  define EVEX512	""
+#  define NO_EVEX512	""
+#endif
+
+#endif /* ARCH_X86_32 || ARCH_X86_64 */
 
 #endif /* LIB_X86_CPU_FEATURES_H */

lib/x86/crc32_impl.h

@@ -1,5 +1,5 @@
 /*
- * x86/crc32_impl.h - x86 implementations of CRC-32 checksum algorithm
+ * x86/crc32_impl.h - x86 implementations of the gzip CRC-32 algorithm
  *
  * Copyright 2016 Eric Biggers
  *
@@ -31,62 +31,129 @@
 #include "cpu_features.h"
 
 /*
- * Include the PCLMUL/AVX implementation?  Although our PCLMUL-optimized CRC-32
- * function doesn't use any AVX intrinsics specifically, it can benefit a lot
- * from being compiled for an AVX target: on Skylake, ~16700 MB/s vs. ~10100
- * MB/s.  I expect this is related to the PCLMULQDQ instructions being assembled
- * in the newer three-operand form rather than the older two-operand form.
- *
- * Note: this is only needed if __AVX__ is *not* defined, since otherwise the
- * "regular" PCLMUL implementation would already be AVX enabled.
+ * pshufb(x, shift_tab[len..len+15]) left shifts x by 16-len bytes.
+ * pshufb(x, shift_tab[len+16..len+31]) right shifts x by len bytes.
  */
-#undef DISPATCH_PCLMUL_AVX
-#if !defined(DEFAULT_IMPL) && !defined(__AVX__) &&	\
-	X86_CPU_FEATURES_ENABLED && COMPILER_SUPPORTS_AVX_TARGET &&	\
-	(defined(__PCLMUL__) || COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS)
-#  define FUNCNAME		crc32_pclmul_avx
-#  define FUNCNAME_ALIGNED	crc32_pclmul_avx_aligned
-#  define ATTRIBUTES		__attribute__((target("pclmul,avx")))
-#  define DISPATCH		1
-#  define DISPATCH_PCLMUL_AVX	1
+static const u8 MAYBE_UNUSED shift_tab[48] = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+/*
+ * PCLMULQDQ implementation.  This targets PCLMULQDQ+SSE4.1, since in practice
+ * all CPUs that support PCLMULQDQ also support SSE4.1.
+ */
+#  define crc32_x86_pclmulqdq	crc32_x86_pclmulqdq
+#  define SUFFIX			 _pclmulqdq
+#  define ATTRIBUTES		_target_attribute("pclmul,sse4.1")
+#  define VL			16
+#  define USE_AVX512		0
+#  include "crc32_pclmul_template.h"
+
+/*
+ * PCLMULQDQ/AVX implementation.  Same as above, but this is compiled with AVX
+ * enabled so that the compiler can generate VEX-coded instructions which can be
+ * slightly more efficient.  It still uses 128-bit vectors.
+ */
+#  define crc32_x86_pclmulqdq_avx	crc32_x86_pclmulqdq_avx
+#  define SUFFIX				 _pclmulqdq_avx
+#  define ATTRIBUTES		_target_attribute("pclmul,avx")
+#  define VL			16
+#  define USE_AVX512		0
 #  include "crc32_pclmul_template.h"
 #endif
 
-/* PCLMUL implementation */
-#undef DISPATCH_PCLMUL
-#if !defined(DEFAULT_IMPL) &&	\
-	(defined(__PCLMUL__) || (X86_CPU_FEATURES_ENABLED &&	\
-				 COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS))
-#  define FUNCNAME		crc32_pclmul
-#  define FUNCNAME_ALIGNED	crc32_pclmul_aligned
-#  ifdef __PCLMUL__
-#    define ATTRIBUTES
-#    define DEFAULT_IMPL	crc32_pclmul
-#  else
-#    define ATTRIBUTES		__attribute__((target("pclmul")))
-#    define DISPATCH		1
-#    define DISPATCH_PCLMUL	1
-#  endif
+/*
+ * VPCLMULQDQ/AVX2 implementation.  This is used on CPUs that have AVX2 and
+ * VPCLMULQDQ but don't have AVX-512, for example Intel Alder Lake.
+ *
+ * Currently this can't be enabled with MSVC because MSVC has a bug where it
+ * incorrectly assumes that VPCLMULQDQ implies AVX-512:
+ * https://developercommunity.visualstudio.com/t/Compiler-incorrectly-assumes-VAES-and-VP/10578785
+ *
+ * gcc 8.1 and 8.2 had a similar bug where they assumed that
+ * _mm256_clmulepi64_epi128() always needed AVX512.  It's fixed in gcc 8.3.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
+ */
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
+#  define crc32_x86_vpclmulqdq_avx2	crc32_x86_vpclmulqdq_avx2
+#  define SUFFIX				 _vpclmulqdq_avx2
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx2")
+#  define VL			32
+#  define USE_AVX512		0
+#  include "crc32_pclmul_template.h"
+#endif
+
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
+/*
+ * VPCLMULQDQ/AVX512 implementation using 256-bit vectors.  This is very similar
+ * to the VPCLMULQDQ/AVX2 implementation but takes advantage of the vpternlog
+ * instruction and more registers.  This is used on certain older Intel CPUs,
+ * specifically Ice Lake and Tiger Lake, which support VPCLMULQDQ and AVX512 but
+ * downclock a bit too eagerly when ZMM registers are used.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
+ */
+#  define crc32_x86_vpclmulqdq_avx512_vl256  crc32_x86_vpclmulqdq_avx512_vl256
+#  define SUFFIX				      _vpclmulqdq_avx512_vl256
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl" NO_EVEX512)
+#  define VL			32
+#  define USE_AVX512		1
+#  include "crc32_pclmul_template.h"
+
+/*
+ * VPCLMULQDQ/AVX512 implementation using 512-bit vectors.  This is used on CPUs
+ * that have a good AVX-512 implementation including VPCLMULQDQ.
+ *
+ * _mm512_zextsi128_si512() requires gcc 10.
+ */
+#  define crc32_x86_vpclmulqdq_avx512_vl512  crc32_x86_vpclmulqdq_avx512_vl512
+#  define SUFFIX				      _vpclmulqdq_avx512_vl512
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl" EVEX512)
+#  define VL			64
+#  define USE_AVX512		1
 #  include "crc32_pclmul_template.h"
 #endif
 
-#ifdef DISPATCH
 static inline crc32_func_t
 arch_select_crc32_func(void)
 {
-	u32 features = get_cpu_features();
+	const u32 features MAYBE_UNUSED = get_x86_cpu_features();
 
-#ifdef DISPATCH_PCLMUL_AVX
-	if ((features & X86_CPU_FEATURE_PCLMUL) &&
-	    (features & X86_CPU_FEATURE_AVX))
-		return crc32_pclmul_avx;
+#ifdef crc32_x86_vpclmulqdq_avx512_vl512
+	if ((features & X86_CPU_FEATURE_ZMM) &&
+	    HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
+		return crc32_x86_vpclmulqdq_avx512_vl512;
 #endif
-#ifdef DISPATCH_PCLMUL
-	if (features & X86_CPU_FEATURE_PCLMUL)
-		return crc32_pclmul;
+#ifdef crc32_x86_vpclmulqdq_avx512_vl256
+	if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
+		return crc32_x86_vpclmulqdq_avx512_vl256;
+#endif
+#ifdef crc32_x86_vpclmulqdq_avx2
+	if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX2(features))
+		return crc32_x86_vpclmulqdq_avx2;
+#endif
+#ifdef crc32_x86_pclmulqdq_avx
+	if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features))
+		return crc32_x86_pclmulqdq_avx;
+#endif
+#ifdef crc32_x86_pclmulqdq
+	if (HAVE_PCLMULQDQ(features))
+		return crc32_x86_pclmulqdq;
 #endif
 	return NULL;
 }
-#endif /* DISPATCH */
+#define arch_select_crc32_func	arch_select_crc32_func
 
 #endif /* LIB_X86_CRC32_IMPL_H */

lib/x86/crc32_pclmul_template.h

@@ -1,5 +1,5 @@
 /*
- * x86/crc32_pclmul_template.h
+ * x86/crc32_pclmul_template.h - gzip CRC-32 with PCLMULQDQ instructions
  *
  * Copyright 2016 Eric Biggers
  *
@@ -25,238 +25,400 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <immintrin.h>
+/*
+ * This file is a "template" for instantiating PCLMULQDQ-based crc32_x86
+ * functions.  The "parameters" are:
+ *
+ * SUFFIX:
+ *	Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ *	Target function attributes to use.  Must satisfy the dependencies of the
+ *	other parameters as follows:
+ *	   VL=16 && USE_AVX512=0: at least pclmul,sse4.1
+ *	   VL=32 && USE_AVX512=0: at least vpclmulqdq,pclmul,avx2
+ *	   VL=32 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
+ *	   VL=64 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
+ *	   (Other combinations are not useful and have not been tested.)
+ * VL:
+ *	Vector length in bytes.  Must be 16, 32, or 64.
+ * USE_AVX512:
+ *	If 1, take advantage of AVX-512 features such as masking and the
+ *	vpternlog instruction.  This doesn't enable the use of 512-bit vectors;
+ *	the vector length is controlled by VL.  If 0, assume that the CPU might
+ *	not support AVX-512.
+ *
+ * The overall algorithm used is CRC folding with carryless multiplication
+ * instructions.  Note that the x86 crc32 instruction cannot be used, as it is
+ * for a different polynomial, not the gzip one.  For an explanation of CRC
+ * folding with carryless multiplication instructions, see
+ * scripts/gen-crc32-consts.py and the following blog posts and papers:
+ *
+ *	"An alternative exposition of crc32_4k_pclmulqdq"
+ *	https://www.corsix.org/content/alternative-exposition-crc32_4k_pclmulqdq
+ *
+ *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
+ *	https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+ *
+ * The original pclmulqdq instruction does one 64x64 to 128-bit carryless
+ * multiplication.  The VPCLMULQDQ feature added instructions that do two
+ * parallel 64x64 to 128-bit carryless multiplications in combination with AVX
+ * or AVX512VL, or four in combination with AVX512F.
+ */
+
+#if VL == 16
+#  define vec_t			__m128i
+#  define fold_vec		fold_vec128
+#  define VLOADU(p)		_mm_loadu_si128((const void *)(p))
+#  define VXOR(a, b)		_mm_xor_si128((a), (b))
+#  define M128I_TO_VEC(a)	a
+#  define MULTS_8V		_mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
+#  define MULTS_4V		_mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
+#  define MULTS_2V		_mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
+#  define MULTS_1V		_mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
+#elif VL == 32
+#  define vec_t			__m256i
+#  define fold_vec		fold_vec256
+#  define VLOADU(p)		_mm256_loadu_si256((const void *)(p))
+#  define VXOR(a, b)		_mm256_xor_si256((a), (b))
+#  define M128I_TO_VEC(a)	_mm256_zextsi128_si256(a)
+#  define MULTS(a, b)		_mm256_set_epi64x(a, b, a, b)
+#  define MULTS_8V		MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
+#  define MULTS_4V		MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
+#  define MULTS_2V		MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
+#  define MULTS_1V		MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
+#elif VL == 64
+#  define vec_t			__m512i
+#  define fold_vec		fold_vec512
+#  define VLOADU(p)		_mm512_loadu_si512((const void *)(p))
+#  define VXOR(a, b)		_mm512_xor_si512((a), (b))
+#  define M128I_TO_VEC(a)	_mm512_zextsi128_si512(a)
+#  define MULTS(a, b)		_mm512_set_epi64(a, b, a, b, a, b, a, b)
+#  define MULTS_8V		MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
+#  define MULTS_4V		MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
+#  define MULTS_2V		MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
+#  define MULTS_1V		MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
+#else
+#  error "unsupported vector length"
+#endif
+
+#undef fold_vec128
+static forceinline ATTRIBUTES __m128i
+ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i /* __v2du */ mults)
+{
+	dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
+	dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
+	return dst;
+}
+#define fold_vec128	ADD_SUFFIX(fold_vec128)
+
+#if VL >= 32
+#undef fold_vec256
+static forceinline ATTRIBUTES __m256i
+ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i /* __v4du */ mults)
+{
+#if USE_AVX512
+	/* vpternlog with immediate 0x96 is a three-argument XOR. */
+	return _mm256_ternarylogic_epi32(
+			_mm256_clmulepi64_epi128(src, mults, 0x00),
+			_mm256_clmulepi64_epi128(src, mults, 0x11),
+			dst,
+			0x96);
+#else
+	return _mm256_xor_si256(
+			_mm256_xor_si256(dst,
+					 _mm256_clmulepi64_epi128(src, mults, 0x00)),
+			_mm256_clmulepi64_epi128(src, mults, 0x11));
+#endif
+}
+#define fold_vec256	ADD_SUFFIX(fold_vec256)
+#endif /* VL >= 32 */
+
+#if VL >= 64
+#undef fold_vec512
+static forceinline ATTRIBUTES __m512i
+ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i /* __v8du */ mults)
+{
+	/* vpternlog with immediate 0x96 is a three-argument XOR. */
+	return _mm512_ternarylogic_epi32(
+			_mm512_clmulepi64_epi128(src, mults, 0x00),
+			_mm512_clmulepi64_epi128(src, mults, 0x11),
+			dst,
+			0x96);
+}
+#define fold_vec512	ADD_SUFFIX(fold_vec512)
+#endif /* VL >= 64 */
 
 /*
- * CRC-32 folding with PCLMULQDQ.
- *
- * The basic idea is to repeatedly "fold" each 512 bits into the next 512 bits,
- * producing an abbreviated message which is congruent the original message
- * modulo the generator polynomial G(x).
- *
- * Folding each 512 bits is implemented as eight 64-bit folds, each of which
- * uses one carryless multiplication instruction.  It's expected that CPUs may
- * be able to execute some of these multiplications in parallel.
- *
- * Explanation of "folding": let A(x) be 64 bits from the message, and let B(x)
- * be 95 bits from a constant distance D later in the message.  The relevant
- * portion of the message can be written as:
- *
- *	M(x) = A(x)*x^D + B(x)
- *
- * ... where + and * represent addition and multiplication, respectively, of
- * polynomials over GF(2).  Note that when implemented on a computer, these
- * operations are equivalent to XOR and carryless multiplication, respectively.
- *
- * For the purpose of CRC calculation, only the remainder modulo the generator
- * polynomial G(x) matters:
- *
- *	M(x) mod G(x) = (A(x)*x^D + B(x)) mod G(x)
- *
- * Since the modulo operation can be applied anywhere in a sequence of additions
- * and multiplications without affecting the result, this is equivalent to:
- *
- *	M(x) mod G(x) = (A(x)*(x^D mod G(x)) + B(x)) mod G(x)
- *
- * For any D, 'x^D mod G(x)' will be a polynomial with maximum degree 31, i.e.
- * a 32-bit quantity.  So 'A(x) * (x^D mod G(x))' is equivalent to a carryless
- * multiplication of a 64-bit quantity by a 32-bit quantity, producing a 95-bit
- * product.  Then, adding (XOR-ing) the product to B(x) produces a polynomial
- * with the same length as B(x) but with the same remainder as 'A(x)*x^D +
- * B(x)'.  This is the basic fold operation with 64 bits.
- *
- * Note that the carryless multiplication instruction PCLMULQDQ actually takes
- * two 64-bit inputs and produces a 127-bit product in the low-order bits of a
- * 128-bit XMM register.  This works fine, but care must be taken to account for
- * "bit endianness".  With the CRC version implemented here, bits are always
- * ordered such that the lowest-order bit represents the coefficient of highest
- * power of x and the highest-order bit represents the coefficient of the lowest
- * power of x.  This is backwards from the more intuitive order.  Still,
- * carryless multiplication works essentially the same either way.  It just must
- * be accounted for that when we XOR the 95-bit product in the low-order 95 bits
- * of a 128-bit XMM register into 128-bits of later data held in another XMM
- * register, we'll really be XOR-ing the product into the mathematically higher
- * degree end of those later bits, not the lower degree end as may be expected.
- *
- * So given that caveat and the fact that we process 512 bits per iteration, the
- * 'D' values we need for the two 64-bit halves of each 128 bits of data are:
- *
- *	D = (512 + 95) - 64	 for the higher-degree half of each 128 bits,
- *				 i.e. the lower order bits in the XMM register
- *
- *	D = (512 + 95) - 128	 for the lower-degree half of each 128 bits,
- *				 i.e. the higher order bits in the XMM register
- *
- * The required 'x^D mod G(x)' values were precomputed.
- *
- * When <= 512 bits remain in the message, we finish up by folding across
- * smaller distances.  This works similarly; the distance D is just different,
- * so different constant multipliers must be used.  Finally, once the remaining
- * message is just 64 bits, it is reduced to the CRC-32 using Barrett reduction
- * (explained later).
- *
- * For more information see the original paper from Intel:
- *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
- *	December 2009
- *	http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+ * Given 'x' containing a 16-byte polynomial, and a pointer 'p' that points to
+ * the next '1 <= len <= 15' data bytes, rearrange the concatenation of 'x' and
+ * the data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,
+ * respectively.  Then fold x0 into x1 and return the result.
+ * Assumes that 'p + len - 16' is in-bounds.
  */
-static u32 ATTRIBUTES
-FUNCNAME_ALIGNED(u32 remainder, const __m128i *p, size_t nr_segs)
+#undef fold_lessthan16bytes
+static forceinline ATTRIBUTES __m128i
+ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
+				 __m128i /* __v2du */ mults_128b)
 {
-	/* Constants precomputed by gen_crc32_multipliers.c.  Do not edit! */
-	const __v2di multipliers_4 = (__v2di){ 0x8F352D95, 0x1D9513D7 };
-	const __v2di multipliers_2 = (__v2di){ 0xF1DA05AA, 0x81256527 };
-	const __v2di multipliers_1 = (__v2di){ 0xAE689191, 0xCCAA009E };
-	const __v2di final_multiplier = (__v2di){ 0xB8BC6765 };
-	const __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };
-	const __v2di barrett_reduction_constants =
-			(__v2di){ 0x00000001F7011641, 0x00000001DB710641 };
+	__m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
+	__m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
+	__m128i x0, x1;
 
-	const __m128i * const end = p + nr_segs;
-	const __m128i * const end512 = p + (nr_segs & ~3);
-	__m128i x0, x1, x2, x3;
+	/* x0 = x left-shifted by '16 - len' bytes */
+	x0 = _mm_shuffle_epi8(x, lshift);
 
 	/*
-	 * Account for the current 'remainder', i.e. the CRC of the part of the
-	 * message already processed.  Explanation: rewrite the message
-	 * polynomial M(x) in terms of the first part A(x), the second part
-	 * B(x), and the length of the second part in bits |B(x)| >= 32:
-	 *
-	 *	M(x) = A(x)*x^|B(x)| + B(x)
-	 *
-	 * Then the CRC of M(x) is:
-	 *
-	 *	CRC(M(x)) = CRC(A(x)*x^|B(x)| + B(x))
-	 *	          = CRC(A(x)*x^32*x^(|B(x)| - 32) + B(x))
-	 *	          = CRC(CRC(A(x))*x^(|B(x)| - 32) + B(x))
-	 *
-	 * Note: all arithmetic is modulo G(x), the generator polynomial; that's
-	 * why A(x)*x^32 can be replaced with CRC(A(x)) = A(x)*x^32 mod G(x).
-	 *
-	 * So the CRC of the full message is the CRC of the second part of the
-	 * message where the first 32 bits of the second part of the message
-	 * have been XOR'ed with the CRC of the first part of the message.
+	 * x1 = the last '16 - len' bytes from x (i.e. x right-shifted by 'len'
+	 * bytes) followed by the remaining data.
 	 */
-	x0 = *p++;
-	x0 ^= (__m128i)(__v4si){ remainder };
+	x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
+			     _mm_loadu_si128((const void *)(p + len - 16)),
+			     /* msb 0/1 of each byte selects byte from arg1/2 */
+			     rshift);
 
-	if (p > end512) /* only 128, 256, or 384 bits of input? */
-		goto _128_bits_at_a_time;
-	x1 = *p++;
-	x2 = *p++;
-	x3 = *p++;
+	return fold_vec128(x0, x1, mults_128b);
+}
+#define fold_lessthan16bytes	ADD_SUFFIX(fold_lessthan16bytes)
 
-	/* Fold 512 bits at a time */
-	for (; p != end512; p += 4) {
-		__m128i y0, y1, y2, y3;
+static ATTRIBUTES u32
+ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
+{
+	/*
+	 * mults_{N}v are the vectors of multipliers for folding across N vec_t
+	 * vectors, i.e. N*VL*8 bits.  mults_128b are the two multipliers for
+	 * folding across 128 bits.  mults_128b differs from mults_1v when
+	 * VL != 16.  All multipliers are 64-bit, to match what pclmulqdq needs,
+	 * but since this is for CRC-32 only their low 32 bits are nonzero.
+	 * For more details, see scripts/gen-crc32-consts.py.
+	 */
+	const vec_t mults_8v = MULTS_8V;
+	const vec_t mults_4v = MULTS_4V;
+	const vec_t mults_2v = MULTS_2V;
+	const vec_t mults_1v = MULTS_1V;
+	const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
+	const __m128i barrett_reduction_constants =
+		_mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
+	vec_t v0, v1, v2, v3, v4, v5, v6, v7;
+	__m128i x0 = _mm_cvtsi32_si128(crc);
+	__m128i x1;
 
-		y0 = p[0];
-		y1 = p[1];
-		y2 = p[2];
-		y3 = p[3];
+	if (len < 8*VL) {
+		if (len < VL) {
+			STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
+			if (len < 16) {
+			#if USE_AVX512
+				if (len < 4)
+					return crc32_slice1(crc, p, len);
+				/*
+				 * Handle 4 <= len <= 15 bytes by doing a masked
+				 * load, XOR'ing the current CRC with the first
+				 * 4 bytes, left-shifting by '16 - len' bytes to
+				 * align the result to the end of x0 (so that it
+				 * becomes the low-order coefficients of a
+				 * 128-bit polynomial), and then doing the usual
+				 * reduction from 128 bits to 32 bits.
+				 */
+				x0 = _mm_xor_si128(
+					x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
+				x0 = _mm_shuffle_epi8(
+					x0, _mm_loadu_si128((const void *)&shift_tab[len]));
+				goto reduce_x0;
+			#else
+				return crc32_slice1(crc, p, len);
+			#endif
+			}
+			/*
+			 * Handle 16 <= len < VL bytes where VL is 32 or 64.
+			 * Use 128-bit instructions so that these lengths aren't
+			 * slower with VL > 16 than with VL=16.
+			 */
+			x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
+			if (len >= 32) {
+				x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
+						 mults_128b);
+				if (len >= 48)
+					x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
+							 mults_128b);
+			}
+			p += len & ~15;
+			goto less_than_16_remaining;
+		}
+		v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
+		if (len < 2*VL) {
+			p += VL;
+			goto less_than_vl_remaining;
+		}
+		v1 = VLOADU(p + 1*VL);
+		if (len < 4*VL) {
+			p += 2*VL;
+			goto less_than_2vl_remaining;
+		}
+		v2 = VLOADU(p + 2*VL);
+		v3 = VLOADU(p + 3*VL);
+		p += 4*VL;
+	} else {
+		/*
+		 * If the length is large and the pointer is misaligned, align
+		 * it.  For smaller lengths, just take the misaligned load
+		 * penalty.  Note that on recent x86 CPUs, vmovdqu with an
+		 * aligned address is just as fast as vmovdqa, so there's no
+		 * need to use vmovdqa in the main loop.
+		 */
+		if (len > 65536 && ((uintptr_t)p & (VL-1))) {
+			size_t align = -(uintptr_t)p & (VL-1);
+
+			len -= align;
+			x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
+			p += 16;
+			if (align & 15) {
+				x0 = fold_lessthan16bytes(x0, p, align & 15,
+							  mults_128b);
+				p += align & 15;
+				align &= ~15;
+			}
+			while (align) {
+				x0 = fold_vec128(x0, *(const __m128i *)p,
+						 mults_128b);
+				p += 16;
+				align -= 16;
+			}
+			v0 = M128I_TO_VEC(x0);
+		#  if VL == 32
+			v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
+		#  elif VL == 64
+			v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
+			v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
+		#  endif
+			p -= 16;
+		} else {
+			v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
+		}
+		v1 = VLOADU(p + 1*VL);
+		v2 = VLOADU(p + 2*VL);
+		v3 = VLOADU(p + 3*VL);
+		v4 = VLOADU(p + 4*VL);
+		v5 = VLOADU(p + 5*VL);
+		v6 = VLOADU(p + 6*VL);
+		v7 = VLOADU(p + 7*VL);
+		p += 8*VL;
 
 		/*
-		 * Note: the immediate constant for PCLMULQDQ specifies which
-		 * 64-bit halves of the 128-bit vectors to multiply:
-		 *
-		 * 0x00 means low halves (higher degree polynomial terms for us)
-		 * 0x11 means high halves (lower degree polynomial terms for us)
+		 * This is the main loop, processing 8*VL bytes per iteration.
+		 * 4*VL is usually enough and would result in smaller code, but
+		 * Skylake and Cascade Lake need 8*VL to get full performance.
 		 */
-		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x00);
-		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x00);
-		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x00);
-		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x00);
-		y0 ^= _mm_clmulepi64_si128(x0, multipliers_4, 0x11);
-		y1 ^= _mm_clmulepi64_si128(x1, multipliers_4, 0x11);
-		y2 ^= _mm_clmulepi64_si128(x2, multipliers_4, 0x11);
-		y3 ^= _mm_clmulepi64_si128(x3, multipliers_4, 0x11);
+		while (len >= 16*VL) {
+			v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
+			v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
+			v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
+			v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
+			v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
+			v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
+			v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
+			v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
+			p += 8*VL;
+			len -= 8*VL;
+		}
 
-		x0 = y0;
-		x1 = y1;
-		x2 = y2;
-		x3 = y3;
+		/* Fewer than 8*VL bytes remain. */
+		v0 = fold_vec(v0, v4, mults_4v);
+		v1 = fold_vec(v1, v5, mults_4v);
+		v2 = fold_vec(v2, v6, mults_4v);
+		v3 = fold_vec(v3, v7, mults_4v);
+		if (len & (4*VL)) {
+			v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
+			v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
+			v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
+			v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
+			p += 4*VL;
+		}
 	}
-
-	/* Fold 512 bits => 128 bits */
-	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x00);
-	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x00);
-	x2 ^= _mm_clmulepi64_si128(x0, multipliers_2, 0x11);
-	x3 ^= _mm_clmulepi64_si128(x1, multipliers_2, 0x11);
-	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x00);
-	x3 ^= _mm_clmulepi64_si128(x2, multipliers_1, 0x11);
-	x0 = x3;
-
-_128_bits_at_a_time:
-	while (p != end) {
-		/* Fold 128 bits into next 128 bits */
-		x1 = *p++;
-		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x00);
-		x1 ^= _mm_clmulepi64_si128(x0, multipliers_1, 0x11);
-		x0 = x1;
+	/* Fewer than 4*VL bytes remain. */
+	v0 = fold_vec(v0, v2, mults_2v);
+	v1 = fold_vec(v1, v3, mults_2v);
+	if (len & (2*VL)) {
+		v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
+		v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
+		p += 2*VL;
 	}
-
-	/* Now there are just 128 bits left, stored in 'x0'. */
-
+less_than_2vl_remaining:
+	/* Fewer than 2*VL bytes remain. */
+	v0 = fold_vec(v0, v1, mults_1v);
+	if (len & VL) {
+		v0 = fold_vec(v0, VLOADU(p), mults_1v);
+		p += VL;
+	}
+less_than_vl_remaining:
 	/*
-	 * Fold 128 => 96 bits.  This also implicitly appends 32 zero bits,
-	 * which is equivalent to multiplying by x^32.  This is needed because
-	 * the CRC is defined as M(x)*x^32 mod G(x), not just M(x) mod G(x).
+	 * Fewer than VL bytes remain.  Reduce v0 (length VL bytes) to x0
+	 * (length 16 bytes) and fold in any 16-byte data segments that remain.
 	 */
-	x0 = _mm_srli_si128(x0, 8) ^
-	     _mm_clmulepi64_si128(x0, multipliers_1, 0x10);
+#if VL == 16
+	x0 = v0;
+#else
+	{
+	#if VL == 32
+		__m256i y0 = v0;
+	#else
+		const __m256i mults_256b =
+			_mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
+					  CRC32_X223_MODG, CRC32_X287_MODG);
+		__m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
+					 _mm512_extracti64x4_epi64(v0, 1),
+					 mults_256b);
+		if (len & 32) {
+			y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
+					 mults_256b);
+			p += 32;
+		}
+	#endif
+		x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
+				 _mm256_extracti128_si256(y0, 1), mults_128b);
+	}
+	if (len & 16) {
+		x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
+				 mults_128b);
+		p += 16;
+	}
+#endif
+less_than_16_remaining:
+	len &= 15;
 
-	/* Fold 96 => 64 bits */
-	x0 = _mm_srli_si128(x0, 4) ^
-	     _mm_clmulepi64_si128(x0 & mask32, final_multiplier, 0x00);
-
-        /*
-	 * Finally, reduce 64 => 32 bits using Barrett reduction.
+	/* Handle any remainder of 1 to 15 bytes. */
+	if (len)
+		x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
+#if USE_AVX512
+reduce_x0:
+#endif
+	/*
+	 * Multiply the remaining 128-bit message polynomial 'x0' by x^32, then
+	 * reduce it modulo the generator polynomial G.  This gives the CRC.
 	 *
-	 * Let M(x) = A(x)*x^32 + B(x) be the remaining message.  The goal is to
-	 * compute R(x) = M(x) mod G(x).  Since degree(B(x)) < degree(G(x)):
-	 *
-	 *	R(x) = (A(x)*x^32 + B(x)) mod G(x)
-	 *	     = (A(x)*x^32) mod G(x) + B(x)
-	 *
-	 * Then, by the Division Algorithm there exists a unique q(x) such that:
-	 *
-	 *	A(x)*x^32 mod G(x) = A(x)*x^32 - q(x)*G(x)
-	 *
-	 * Since the left-hand side is of maximum degree 31, the right-hand side
-	 * must be too.  This implies that we can apply 'mod x^32' to the
-	 * right-hand side without changing its value:
-	 *
-	 *	(A(x)*x^32 - q(x)*G(x)) mod x^32 = q(x)*G(x) mod x^32
-	 *
-	 * Note that '+' is equivalent to '-' in polynomials over GF(2).
-	 *
-	 * We also know that:
-	 *
-	 *	              / A(x)*x^32 \
-	 *	q(x) = floor (  ---------  )
-	 *	              \    G(x)   /
-	 *
-	 * To compute this efficiently, we can multiply the top and bottom by
-	 * x^32 and move the division by G(x) to the top:
-	 *
-	 *	              / A(x) * floor(x^64 / G(x)) \
-	 *	q(x) = floor (  -------------------------  )
-	 *	              \           x^32            /
-	 *
-	 * Note that floor(x^64 / G(x)) is a constant.
-	 *
-	 * So finally we have:
-	 *
-	 *	                          / A(x) * floor(x^64 / G(x)) \
-	 *	R(x) = B(x) + G(x)*floor (  -------------------------  )
-	 *	                          \           x^32            /
+	 * This implementation matches that used in crc-pclmul-template.S from
+	 * https://lore.kernel.org/r/20250210174540.161705-4-ebiggers@kernel.org/
+	 * with the parameters n=32 and LSB_CRC=1 (what the gzip CRC uses).  See
+	 * there for a detailed explanation of the math used here.
 	 */
-	x1 = x0;
-	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x00);
-	x0 = _mm_clmulepi64_si128(x0 & mask32, barrett_reduction_constants, 0x10);
-	return _mm_cvtsi128_si32(_mm_srli_si128(x0 ^ x1, 4));
+	x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
+			   _mm_bsrli_si128(x0, 8));
+	x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
+	x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
+	x0 = _mm_xor_si128(x0, x1);
+	return _mm_extract_epi32(x0, 2);
 }
 
-#define IMPL_ALIGNMENT		16
-#define IMPL_SEGMENT_SIZE	16
-#include "../crc32_vec_template.h"
+#undef vec_t
+#undef fold_vec
+#undef VLOADU
+#undef VXOR
+#undef M128I_TO_VEC
+#undef MULTS
+#undef MULTS_8V
+#undef MULTS_4V
+#undef MULTS_2V
+#undef MULTS_1V
+
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef VL
+#undef USE_AVX512

lib/x86/decompress_impl.h

@@ -3,29 +3,55 @@
 
 #include "cpu_features.h"
 
-/* Include the BMI2-optimized version? */
-#undef DISPATCH_BMI2
-#if !defined(__BMI2__) && X86_CPU_FEATURES_ENABLED && \
-	COMPILER_SUPPORTS_BMI2_TARGET
-#  define FUNCNAME	deflate_decompress_bmi2
-#  define ATTRIBUTES	__attribute__((target("bmi2")))
-#  define DISPATCH	1
-#  define DISPATCH_BMI2	1
+/*
+ * BMI2 optimized decompression function.
+ *
+ * With gcc and clang we just compile the whole function with
+ * __attribute__((target("bmi2"))), and the compiler uses bmi2 automatically.
+ *
+ * With MSVC, there is no target function attribute, but it's still possible to
+ * use bmi2 intrinsics explicitly.  Currently we mostly don't, but there's a
+ * case in which we do (see below), so we at least take advantage of that.
+ * However, MSVC from VS2017 (toolset v141) apparently miscompiles the _bzhi_*()
+ * intrinsics.  It seems to be fixed in VS2022.  Hence, use MSVC_PREREQ(1930).
+ */
+#if defined(__GNUC__) || defined(__clang__) || MSVC_PREREQ(1930)
+#  define deflate_decompress_bmi2	deflate_decompress_bmi2
+#  define FUNCNAME			deflate_decompress_bmi2
+#  define ATTRIBUTES			_target_attribute("bmi2")
+   /*
+    * Even with __attribute__((target("bmi2"))), gcc doesn't reliably use the
+    * bzhi instruction for 'word & BITMASK(count)'.  So use the bzhi intrinsic
+    * explicitly.  EXTRACT_VARBITS() is equivalent to 'word & BITMASK(count)';
+    * EXTRACT_VARBITS8() is equivalent to 'word & BITMASK((u8)count)'.
+    * Nevertheless, their implementation using the bzhi intrinsic is identical,
+    * as the bzhi instruction truncates the count to 8 bits implicitly.
+    */
+#  ifndef __clang__
+#    ifdef ARCH_X86_64
+#      define EXTRACT_VARBITS(word, count)  _bzhi_u64((word), (count))
+#      define EXTRACT_VARBITS8(word, count) _bzhi_u64((word), (count))
+#    else
+#      define EXTRACT_VARBITS(word, count)  _bzhi_u32((word), (count))
+#      define EXTRACT_VARBITS8(word, count) _bzhi_u32((word), (count))
+#    endif
+#  endif
 #  include "../decompress_template.h"
 #endif
 
-#ifdef DISPATCH
+#if defined(deflate_decompress_bmi2) && HAVE_BMI2_NATIVE
+#define DEFAULT_IMPL	deflate_decompress_bmi2
+#else
 static inline decompress_func_t
 arch_select_decompress_func(void)
 {
-	u32 features = get_cpu_features();
-
-#ifdef DISPATCH_BMI2
-	if (features & X86_CPU_FEATURE_BMI2)
+#ifdef deflate_decompress_bmi2
+	if (HAVE_BMI2(get_x86_cpu_features()))
 		return deflate_decompress_bmi2;
 #endif
 	return NULL;
 }
-#endif /* DISPATCH */
+#define arch_select_decompress_func	arch_select_decompress_func
+#endif
 
 #endif /* LIB_X86_DECOMPRESS_IMPL_H */

lib/x86/matchfinder_impl.h

@@ -28,8 +28,9 @@
 #ifndef LIB_X86_MATCHFINDER_IMPL_H
 #define LIB_X86_MATCHFINDER_IMPL_H
 
+#include "cpu_features.h"
+
 #ifdef __AVX2__
-#  include <immintrin.h>
 static forceinline void
 matchfinder_init_avx2(mf_pos_t *data, size_t size)
 {
@@ -73,8 +74,7 @@ matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
 }
 #define matchfinder_rebase matchfinder_rebase_avx2
 
-#elif defined(__SSE2__)
-#  include <emmintrin.h>
+#elif HAVE_SSE2_NATIVE
 static forceinline void
 matchfinder_init_sse2(mf_pos_t *data, size_t size)
 {
@@ -117,6 +117,6 @@ matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
 	} while (size != 0);
 }
 #define matchfinder_rebase matchfinder_rebase_sse2
-#endif /* __SSE2__ */
+#endif /* HAVE_SSE2_NATIVE */
 
 #endif /* LIB_X86_MATCHFINDER_IMPL_H */

lib/zlib_compress.c

@@ -26,12 +26,9 @@
  */
 
 #include "deflate_compress.h"
-#include "unaligned.h"
 #include "zlib_constants.h"
 
-#include "libdeflate.h"
-
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_zlib_compress(struct libdeflate_compressor *c,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail)
@@ -47,7 +44,7 @@ libdeflate_zlib_compress(struct libdeflate_compressor *c,
 
 	/* 2 byte header: CMF and FLG  */
 	hdr = (ZLIB_CM_DEFLATE << 8) | (ZLIB_CINFO_32K_WINDOW << 12);
-	compression_level = deflate_get_compression_level(c);
+	compression_level = libdeflate_get_compression_level(c);
 	if (compression_level < 2)
 		level_hint = ZLIB_FASTEST_COMPRESSION;
 	else if (compression_level < 6)
@@ -76,7 +73,7 @@ libdeflate_zlib_compress(struct libdeflate_compressor *c,
 	return out_next - (u8 *)out;
 }
 
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_zlib_compress_bound(struct libdeflate_compressor *c,
 			       size_t in_nbytes)
 {

lib/zlib_decompress.c

@@ -25,12 +25,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "unaligned.h"
+#include "lib_common.h"
 #include "zlib_constants.h"
 
-#include "libdeflate.h"
-
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -94,7 +92,7 @@ libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
 	return LIBDEFLATE_SUCCESS;
 }
 
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,

libdeflate-config.cmake.in

@@ -0,0 +1,3 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/libdeflate-targets.cmake")

libdeflate.h

@@ -5,57 +5,37 @@
 #ifndef LIBDEFLATE_H
 #define LIBDEFLATE_H
 
+#include <stddef.h>
+#include <stdint.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #define LIBDEFLATE_VERSION_MAJOR	1
-#define LIBDEFLATE_VERSION_MINOR	11
-#define LIBDEFLATE_VERSION_STRING	"1.11"
-
-#include <stddef.h>
-#include <stdint.h>
+#define LIBDEFLATE_VERSION_MINOR	23
+#define LIBDEFLATE_VERSION_STRING	"1.23"
 
 /*
- * On Windows, if you want to link to the DLL version of libdeflate, then
- * #define LIBDEFLATE_DLL.  Note that the calling convention is "stdcall".
+ * Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause
+ * __declspec(dllimport) to be used.  This should be done when it's easy to do.
+ * Otherwise it's fine to skip it, since it is a very minor performance
+ * optimization that is irrelevant for most use cases of libdeflate.
  */
-#ifdef LIBDEFLATE_DLL
-#  ifdef BUILDING_LIBDEFLATE
-#    define LIBDEFLATEEXPORT	LIBEXPORT
-#  elif defined(_WIN32) || defined(__CYGWIN__)
-#    define LIBDEFLATEEXPORT	__declspec(dllimport)
+#ifndef LIBDEFLATEAPI
+#  if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
+#    define LIBDEFLATEAPI	__declspec(dllimport)
+#  else
+#    define LIBDEFLATEAPI
 #  endif
 #endif
-#ifndef LIBDEFLATEEXPORT
-#  define LIBDEFLATEEXPORT
-#endif
-
-#if defined(_WIN32) && !defined(_WIN64) && defined(LIBDEFLATE_DLL)
-#  define LIBDEFLATEAPI_ABI	__stdcall
-#else
-#  define LIBDEFLATEAPI_ABI
-#endif
-
-#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \
-	defined(_WIN32) && !defined(_WIN64)
-    /*
-     * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4.
-     * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX
-     * code when called from an MSVC-compiled application.
-     */
-#  define LIBDEFLATEAPI_STACKALIGN	__attribute__((force_align_arg_pointer))
-#else
-#  define LIBDEFLATEAPI_STACKALIGN
-#endif
-
-#define LIBDEFLATEAPI	LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN
 
 /* ========================================================================== */
 /*                             Compression                                    */
 /* ========================================================================== */
 
 struct libdeflate_compressor;
+struct libdeflate_options;
 
 /*
  * libdeflate_alloc_compressor() allocates a new compressor that supports
@@ -75,17 +55,33 @@ struct libdeflate_compressor;
  * A single compressor is not safe to use by multiple threads concurrently.
  * However, different threads may use different compressors concurrently.
  */
-LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+LIBDEFLATEAPI struct libdeflate_compressor *
 libdeflate_alloc_compressor(int compression_level);
 
 /*
- * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
- * data.  The function attempts to compress 'in_nbytes' bytes of data located at
- * 'in' and write the results to 'out', which has space for 'out_nbytes_avail'
- * bytes.  The return value is the compressed size in bytes, or 0 if the data
- * could not be compressed to 'out_nbytes_avail' bytes or fewer.
+ * Like libdeflate_alloc_compressor(), but adds the 'options' argument.
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI struct libdeflate_compressor *
+libdeflate_alloc_compressor_ex(int compression_level,
+			       const struct libdeflate_options *options);
+
+/*
+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
+ * data.  It attempts to compress 'in_nbytes' bytes of data located at 'in' and
+ * write the result to 'out', which has space for 'out_nbytes_avail' bytes.  The
+ * return value is the compressed size in bytes, or 0 if the data could not be
+ * compressed to 'out_nbytes_avail' bytes or fewer.
+ *
+ * If compression is successful, then the output data is guaranteed to be a
+ * valid DEFLATE stream that decompresses to the input data.  No other
+ * guarantees are made about the output data.  Notably, different versions of
+ * libdeflate can produce different compressed data for the same uncompressed
+ * data, even at the same compression level.  Do ***NOT*** do things like
+ * writing tests that compare compressed data to a golden output, as this can
+ * break when libdeflate is updated.  (This property isn't specific to
+ * libdeflate; the same is true for zlib and other compression libraries too.)
+ */
+LIBDEFLATEAPI size_t
 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
 			    const void *in, size_t in_nbytes,
 			    void *out, size_t out_nbytes_avail);
@@ -94,11 +90,10 @@ libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
  * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
  * number of bytes of compressed data that may be produced by compressing any
  * buffer of length less than or equal to 'in_nbytes' using
- * libdeflate_deflate_compress() with the specified compressor.  Mathematically,
- * this bound will necessarily be a number greater than or equal to 'in_nbytes'.
- * It may be an overestimate of the true upper bound.  The return value is
- * guaranteed to be the same for all invocations with the same compressor and
- * same 'in_nbytes'.
+ * libdeflate_deflate_compress() with the specified compressor.  This bound will
+ * necessarily be a number greater than or equal to 'in_nbytes'.  It may be an
+ * overestimate of the true upper bound.  The return value is guaranteed to be
+ * the same for all invocations with the same compressor and same 'in_nbytes'.
  *
  * As a special case, 'compressor' may be NULL.  This causes the bound to be
  * taken across *any* libdeflate_compressor that could ever be allocated with
@@ -115,15 +110,15 @@ libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
  * libdeflate_deflate_compress() returns 0, indicating that the compressed data
  * did not fit into the provided output buffer.
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
 				  size_t in_nbytes);
 
 /*
- * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper
- * format.
+ * Like libdeflate_deflate_compress(), but uses the zlib wrapper format instead
+ * of raw DEFLATE.
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail);
@@ -133,15 +128,15 @@ libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
  * compressed with libdeflate_zlib_compress() rather than with
  * libdeflate_deflate_compress().
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
 			       size_t in_nbytes);
 
 /*
- * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper
- * format.
+ * Like libdeflate_deflate_compress(), but uses the gzip wrapper format instead
+ * of raw DEFLATE.
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
 			 const void *in, size_t in_nbytes,
 			 void *out, size_t out_nbytes_avail);
@@ -151,7 +146,7 @@ libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
  * compressed with libdeflate_gzip_compress() rather than with
  * libdeflate_deflate_compress().
  */
-LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+LIBDEFLATEAPI size_t
 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
 			       size_t in_nbytes);
 
@@ -160,7 +155,7 @@ libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
  * libdeflate_alloc_compressor().  If a NULL pointer is passed in, no action is
  * taken.
  */
-LIBDEFLATEEXPORT void LIBDEFLATEAPI
+LIBDEFLATEAPI void
 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
 
 /* ========================================================================== */
@@ -168,6 +163,7 @@ libdeflate_free_compressor(struct libdeflate_compressor *compressor);
 /* ========================================================================== */
 
 struct libdeflate_decompressor;
+struct libdeflate_options;
 
 /*
  * libdeflate_alloc_decompressor() allocates a new decompressor that can be used
@@ -181,9 +177,15 @@ struct libdeflate_decompressor;
  * A single decompressor is not safe to use by multiple threads concurrently.
  * However, different threads may use different decompressors concurrently.
  */
-LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
+LIBDEFLATEAPI struct libdeflate_decompressor *
 libdeflate_alloc_decompressor(void);
 
+/*
+ * Like libdeflate_alloc_decompressor(), but adds the 'options' argument.
+ */
+LIBDEFLATEAPI struct libdeflate_decompressor *
+libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
+
 /*
  * Result of a call to libdeflate_deflate_decompress(),
  * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
@@ -192,8 +194,8 @@ enum libdeflate_result {
 	/* Decompression was successful.  */
 	LIBDEFLATE_SUCCESS = 0,
 
-	/* Decompressed failed because the compressed data was invalid, corrupt,
-	 * or otherwise unsupported.  */
+	/* Decompression failed because the compressed data was invalid,
+	 * corrupt, or otherwise unsupported.  */
 	LIBDEFLATE_BAD_DATA = 1,
 
 	/* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
@@ -206,13 +208,12 @@ enum libdeflate_result {
 };
 
 /*
- * libdeflate_deflate_decompress() decompresses the DEFLATE-compressed stream
- * from the buffer 'in' with compressed size up to 'in_nbytes' bytes.  The
- * uncompressed data is written to 'out', a buffer with size 'out_nbytes_avail'
- * bytes.  If decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned.
- * Otherwise, a nonzero result code such as LIBDEFLATE_BAD_DATA is returned.  If
- * a nonzero result code is returned, then the contents of the output buffer are
- * undefined.
+ * libdeflate_deflate_decompress() decompresses a DEFLATE stream from the buffer
+ * 'in' with compressed size up to 'in_nbytes' bytes.  The uncompressed data is
+ * written to 'out', a buffer with size 'out_nbytes_avail' bytes.  If
+ * decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned.  Otherwise,
+ * a nonzero result code such as LIBDEFLATE_BAD_DATA is returned, and the
+ * contents of the output buffer are undefined.
  *
  * Decompression stops at the end of the DEFLATE stream (as indicated by the
  * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
@@ -237,7 +238,7 @@ enum libdeflate_result {
  *     not large enough but no other problems were encountered, or another
  *     nonzero result code if decompression failed for another reason.
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -249,7 +250,7 @@ libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
  * then the actual compressed size of the DEFLATE stream (aligned to the next
  * byte boundary) is written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
 				 const void *in, size_t in_nbytes,
 				 void *out, size_t out_nbytes_avail,
@@ -264,7 +265,7 @@ libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
  * than 'in_nbytes'.  If you need to know exactly where the zlib stream ended,
  * use libdeflate_zlib_decompress_ex().
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,
@@ -277,7 +278,7 @@ libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
  * buffer was decompressed), then the actual number of input bytes consumed is
  * written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -292,7 +293,7 @@ libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
  * will be decompressed.  Use libdeflate_gzip_decompress_ex() if you need
  * multi-member support.
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
 			   const void *in, size_t in_nbytes,
 			   void *out, size_t out_nbytes_avail,
@@ -305,7 +306,7 @@ libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
  * buffer was decompressed), then the actual number of input bytes consumed is
  * written to *actual_in_nbytes_ret.
  */
-LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+LIBDEFLATEAPI enum libdeflate_result
 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
 			      const void *in, size_t in_nbytes,
 			      void *out, size_t out_nbytes_avail,
@@ -317,7 +318,7 @@ libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
  * libdeflate_alloc_decompressor().  If a NULL pointer is passed in, no action
  * is taken.
  */
-LIBDEFLATEEXPORT void LIBDEFLATEAPI
+LIBDEFLATEAPI void
 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
 
 /* ========================================================================== */
@@ -330,7 +331,7 @@ libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
  * required initial value for 'adler' is 1.  This value is also returned when
  * 'buffer' is specified as NULL.
  */
-LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+LIBDEFLATEAPI uint32_t
 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
 
 
@@ -340,7 +341,7 @@ libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
  * initial value for 'crc' is 0.  This value is also returned when 'buffer' is
  * specified as NULL.
  */
-LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+LIBDEFLATEAPI uint32_t
 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
 
 /* ========================================================================== */
@@ -349,16 +350,60 @@ libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
 
 /*
  * Install a custom memory allocator which libdeflate will use for all memory
- * allocations.  'malloc_func' is a function that must behave like malloc(), and
- * 'free_func' is a function that must behave like free().
+ * allocations by default.  'malloc_func' is a function that must behave like
+ * malloc(), and 'free_func' is a function that must behave like free().
  *
- * There must not be any libdeflate_compressor or libdeflate_decompressor
- * structures in existence when calling this function.
+ * The per-(de)compressor custom memory allocator that can be specified in
+ * 'struct libdeflate_options' takes priority over this.
+ *
+ * This doesn't affect the free() function that will be used to free
+ * (de)compressors that were already in existence when this is called.
  */
-LIBDEFLATEEXPORT void LIBDEFLATEAPI
+LIBDEFLATEAPI void
 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
 				void (*free_func)(void *));
 
+/*
+ * Advanced options.  This is the options structure that
+ * libdeflate_alloc_compressor_ex() and libdeflate_alloc_decompressor_ex()
+ * require.  Most users won't need this and should just use the non-"_ex"
+ * functions instead.  If you do need this, it should be initialized like this:
+ *
+ *	struct libdeflate_options options;
+ *
+ *	memset(&options, 0, sizeof(options));
+ *	options.sizeof_options = sizeof(options);
+ *	// Then set the fields that you need to override the defaults for.
+ */
+struct libdeflate_options {
+
+	/*
+	 * This field must be set to the struct size.  This field exists for
+	 * extensibility, so that fields can be appended to this struct in
+	 * future versions of libdeflate while still supporting old binaries.
+	 */
+	size_t sizeof_options;
+
+	/*
+	 * An optional custom memory allocator to use for this (de)compressor.
+	 * 'malloc_func' must be a function that behaves like malloc(), and
+	 * 'free_func' must be a function that behaves like free().
+	 *
+	 * This is useful in cases where a process might have multiple users of
+	 * libdeflate who want to use different memory allocators.  For example,
+	 * a library might want to use libdeflate with a custom memory allocator
+	 * without interfering with user code that might use libdeflate too.
+	 *
+	 * This takes priority over the "global" memory allocator (which by
+	 * default is malloc() and free(), but can be changed by
+	 * libdeflate_set_memory_allocator()).  Moreover, libdeflate will never
+	 * call the "global" memory allocator if a per-(de)compressor custom
+	 * allocator is always given.
+	 */
+	void *(*malloc_func)(size_t);
+	void (*free_func)(void *);
+};
+
 #ifdef __cplusplus
 }
 #endif

libdeflate.pc.in

@@ -1,10 +1,18 @@
-prefix=@PREFIX@
+prefix=@CMAKE_INSTALL_PREFIX@
 exec_prefix=${prefix}
-includedir=@INCDIR@
-libdir=@LIBDIR@
+includedir=@CMAKE_PKGCONFIG_INCLUDEDIR@
+libdir=@CMAKE_PKGCONFIG_LIBDIR@
 
 Name: libdeflate
 Description: Fast implementation of DEFLATE, zlib, and gzip
-Version: @VERSION@
+Version: @PROJECT_VERSION@
 Libs: -L${libdir} -ldeflate
 Cflags: -I${includedir}
+
+# Note: this library's public header allows LIBDEFLATE_DLL to be defined when
+# linking to the DLL on Windows, to make __declspec(dllimport) be used.
+# However, the only way to define a shared-library-only flag in a pkgconfig file
+# is to use the weird workaround of unconditionally defining it in Cflags, then
+# undefining it in Cflags.private.  Just don't bother with this, since
+# __declspec(dllimport) is optional anyway.  It is a very minor performance
+# optimization that is irrelevant for most use cases of libdeflate.

programs/CMakeLists.txt

@@ -0,0 +1,105 @@
+include(CheckSymbolExists)
+
+# Check for the availability of OS functionality and generate the config.h file.
+#
+# Keep CMAKE_REQUIRED_DEFINITIONS in sync with what prog_util.h does.
+if(LINUX)
+    set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE -D_POSIX_C_SOURCE=200809L)
+elseif(APPLE)
+    set(CMAKE_REQUIRED_DEFINITIONS -D_DARWIN_C_SOURCE -U_POSIX_C_SOURCE)
+else()
+    set(CMAKE_REQUIRED_DEFINITIONS -U_POSIX_C_SOURCE)
+endif()
+check_symbol_exists(clock_gettime "time.h" HAVE_CLOCK_GETTIME)
+check_symbol_exists(futimens "fcntl.h;sys/stat.h" HAVE_FUTIMENS)
+check_symbol_exists(posix_fadvise "fcntl.h" HAVE_POSIX_FADVISE)
+check_symbol_exists(posix_madvise "sys/mman.h" HAVE_POSIX_MADVISE)
+check_c_source_compiles("#include <sys/types.h>
+                         #include <sys/stat.h>
+                         int main() { struct stat st; (void)st.st_atim; }"
+                         HAVE_STAT_NANOSECOND_PRECISION)
+configure_file(config.h.in config.h)
+
+# Build a utility library for the programs.  This library is not installed.
+add_library(libdeflate_prog_utils STATIC prog_util.c tgetopt.c ../common_defs.h)
+set_target_properties(libdeflate_prog_utils PROPERTIES
+                      OUTPUT_NAME deflate_prog_utils)
+if(LIBDEFLATE_USE_SHARED_LIB)
+    target_link_libraries(libdeflate_prog_utils PUBLIC libdeflate_shared)
+else()
+    target_link_libraries(libdeflate_prog_utils PUBLIC libdeflate_static)
+endif()
+target_include_directories(libdeflate_prog_utils PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+target_compile_definitions(libdeflate_prog_utils PUBLIC HAVE_CONFIG_H)
+if(WIN32)
+    if(MINGW)
+        target_compile_options(libdeflate_prog_utils PUBLIC -municode)
+        target_link_libraries(libdeflate_prog_utils PUBLIC -municode)
+    else()
+        target_compile_definitions(libdeflate_prog_utils PUBLIC UNICODE _UNICODE)
+    endif()
+endif()
+
+# Build and install libdeflate-gzip and its alias libdeflate-gunzip.
+if(LIBDEFLATE_BUILD_GZIP)
+    add_executable(libdeflate-gzip gzip.c)
+    target_link_libraries(libdeflate-gzip PRIVATE libdeflate_prog_utils)
+    install(TARGETS libdeflate-gzip DESTINATION ${CMAKE_INSTALL_BINDIR})
+    if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14")
+        # Install libdeflate-gunzip as a hard link to libdeflate-gzip.
+        # Fall back to a copy if hard links are unsupported.
+        #
+        # Note: on Windows, prepending DESTDIR like this doesn't work correctly
+        # when ${CMAKE_INSTALL_FULL_BINDIR} includes a drive letter.  But that
+        # is fine since DESTDIR is unsupported on Windows anyway, according to
+        # the CMake documentation.
+        set(GZIP "${CMAKE_INSTALL_FULL_BINDIR}/libdeflate-gzip${CMAKE_EXECUTABLE_SUFFIX}")
+        set(GUNZIP "${CMAKE_INSTALL_FULL_BINDIR}/libdeflate-gunzip${CMAKE_EXECUTABLE_SUFFIX}")
+        install(CODE "message(\"-- Installing: \$ENV{DESTDIR}${GUNZIP}\")")
+        install(CODE "file(CREATE_LINK \"\$ENV{DESTDIR}${GZIP}\"
+                           \"\$ENV{DESTDIR}${GUNZIP}\" COPY_ON_ERROR)")
+    else()
+        # The cmake version is too old to support file(CREATE_LINK).
+        # Just compile gzip.c again to build libdeflate-gunzip.
+        add_executable(libdeflate-gunzip gzip.c)
+        target_link_libraries(libdeflate-gunzip PRIVATE libdeflate_prog_utils)
+        install(TARGETS libdeflate-gunzip DESTINATION ${CMAKE_INSTALL_BINDIR})
+    endif()
+endif()
+
+# Build the test programs, if requested.
+if(LIBDEFLATE_BUILD_TESTS)
+
+    # The test programs depend on zlib for comparison tests.
+    find_package(ZLIB REQUIRED)
+
+    # Build a utility library for the test programs.
+    add_library(libdeflate_test_utils STATIC test_util.c)
+    set_target_properties(libdeflate_test_utils PROPERTIES
+                          OUTPUT_NAME deflate_test_utils)
+    target_link_libraries(libdeflate_test_utils PUBLIC
+                          libdeflate_prog_utils ZLIB::ZLIB)
+
+    # Build the benchmark and checksum programs.
+    add_executable(benchmark benchmark.c)
+    target_link_libraries(benchmark PRIVATE libdeflate_test_utils)
+    add_executable(checksum checksum.c)
+    target_link_libraries(checksum PRIVATE libdeflate_test_utils)
+
+    # Build the unit test programs and register them with CTest.
+    set(UNIT_TEST_PROGS
+        test_checksums
+        test_custom_malloc
+        test_incomplete_codes
+        test_invalid_streams
+        test_litrunlen_overflow
+        test_overread
+        test_slow_decompression
+        test_trailing_bytes
+    )
+    foreach(PROG ${UNIT_TEST_PROGS})
+        add_executable(${PROG} ${PROG}.c)
+        target_link_libraries(${PROG} PRIVATE libdeflate_test_utils)
+        add_test(NAME ${PROG} COMMAND ${PROG})
+    endforeach()
+endif()

programs/checksum.c

@@ -27,19 +27,20 @@
 
 #include "test_util.h"
 
-static const tchar *const optstring = T("Ahs:tZ");
+static const tchar *const optstring = T("Ahm:s:tZ");
 
 static void
 show_usage(FILE *fp)
 {
 	fprintf(fp,
-"Usage: %"TS" [-A] [-h] [-s SIZE] [-t] [-Z] [FILE]...\n"
+"Usage: %"TS" [-A] [-h] [-m ALIGN] [-s SIZE] [-t] [-Z] [FILE]...\n"
 "Calculate Adler-32 or CRC-32 checksums of the specified FILEs.\n"
 "\n"
 "Options:\n"
 "  -A        use Adler-32 (default is CRC-32)\n"
 "  -h        print this help\n"
-"  -s SIZE   chunk size\n"
+"  -m ALIGN  misalign the buffer by ALIGN bytes\n"
+"  -s SIZE   chunk size in bytes\n"
 "  -t        show checksum speed, excluding I/O\n"
 "  -Z        use zlib implementation instead of libdeflate\n",
 	prog_invocation_name);
@@ -107,7 +108,9 @@ tmain(int argc, tchar *argv[])
 	bool use_adler32 = false;
 	bool use_zlib_impl = false;
 	bool do_timing = false;
+	void *orig_buf = NULL;
 	void *buf;
+	size_t misalignment = 0;
 	size_t bufsize = 131072;
 	tchar *default_file_list[] = { NULL };
 	cksum_fn_t cksum;
@@ -125,9 +128,16 @@ tmain(int argc, tchar *argv[])
 		case 'h':
 			show_usage(stdout);
 			return 0;
+		case 'm':
+			misalignment = tstrtoul(toptarg, NULL, 10);
+			if (misalignment >= 4096) {
+				msg("invalid misalignment: \"%"TS"\"", toptarg);
+				return 1;
+			}
+			break;
 		case 's':
 			bufsize = tstrtoul(toptarg, NULL, 10);
-			if (bufsize == 0) {
+			if (bufsize == 0 || bufsize > SIZE_MAX / 2) {
 				msg("invalid chunk size: \"%"TS"\"", toptarg);
 				return 1;
 			}
@@ -159,9 +169,10 @@ tmain(int argc, tchar *argv[])
 			cksum = crc32_libdeflate;
 	}
 
-	buf = xmalloc(bufsize);
-	if (buf == NULL)
+	orig_buf = xmalloc(bufsize + 4096 + misalignment);
+	if (orig_buf == NULL)
 		return 1;
+	buf = (u8 *)orig_buf + (-(uintptr_t)orig_buf % 4096) + misalignment;
 
 	if (argc == 0) {
 		argv = default_file_list;
@@ -202,6 +213,6 @@ tmain(int argc, tchar *argv[])
 	}
 	ret = 0;
 out:
-	free(buf);
+	free(orig_buf);
 	return -ret;
 }

programs/config.h.in

@@ -0,0 +1,19 @@
+#ifndef CONFIG_H
+#define CONFIG_H
+
+/* Is the clock_gettime() function available? */
+#cmakedefine HAVE_CLOCK_GETTIME
+
+/* Is the futimens() function available? */
+#cmakedefine HAVE_FUTIMENS
+
+/* Is the posix_fadvise() function available? */
+#cmakedefine HAVE_POSIX_FADVISE
+
+/* Is the posix_madvise() function available? */
+#cmakedefine HAVE_POSIX_MADVISE
+
+/* Does stat() provide nanosecond-precision timestamps? */
+#cmakedefine HAVE_STAT_NANOSECOND_PRECISION
+
+#endif /* CONFIG_H */

programs/gzip.c

@@ -28,7 +28,6 @@
 #include "prog_util.h"
 
 #include <errno.h>
-#include <sys/types.h>
 #include <sys/stat.h>
 #ifdef _WIN32
 #  include <sys/utime.h>
@@ -185,13 +184,6 @@ out:
 	return ret;
 }
 
-static u32
-load_u32_gzip(const u8 *p)
-{
-	return ((u32)p[0] << 0) | ((u32)p[1] << 8) |
-		((u32)p[2] << 16) | ((u32)p[3] << 24);
-}
-
 static int
 do_decompress(struct libdeflate_decompressor *decompressor,
 	      struct file_stream *in, struct file_stream *out,
@@ -223,7 +215,8 @@ do_decompress(struct libdeflate_decompressor *decompressor,
 	 * not be the largest; or the real size may be >= 4 GiB, causing ISIZE
 	 * to overflow.  In any case, make sure to allocate at least one byte.
 	 */
-	uncompressed_size = load_u32_gzip(&compressed_data[compressed_size - 4]);
+	uncompressed_size =
+		get_unaligned_le32(&compressed_data[compressed_size - 4]);
 	if (uncompressed_size == 0)
 		uncompressed_size = 1;
 
@@ -358,21 +351,19 @@ restore_timestamps(struct file_stream *out, const tchar *newpath,
 		   const stat_t *stbuf)
 {
 	int ret;
-#if defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION)
-	struct timespec times[2] = {
-		stbuf->st_atim, stbuf->st_mtim,
-	};
+#ifdef __APPLE__
+	struct timespec times[2] = { stbuf->st_atimespec, stbuf->st_mtimespec };
+
+	ret = futimens(out->fd, times);
+#elif (defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION)) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(UTIME_NOW))
+	struct timespec times[2] = { stbuf->st_atim, stbuf->st_mtim };
+
 	ret = futimens(out->fd, times);
-#elif defined(HAVE_FUTIMES) && defined(HAVE_STAT_NANOSECOND_PRECISION)
-	struct timeval times[2] = {
-		{ stbuf->st_atim.tv_sec, stbuf->st_atim.tv_nsec / 1000, },
-		{ stbuf->st_mtim.tv_sec, stbuf->st_mtim.tv_nsec / 1000, },
-	};
-	ret = futimes(out->fd, times);
 #else
-	struct tutimbuf times = {
-		stbuf->st_atime, stbuf->st_mtime,
-	};
+	struct tutimbuf times = { stbuf->st_atime, stbuf->st_mtime };
+
 	ret = tutime(newpath, &times);
 #endif
 	if (ret != 0)

programs/prog_util.c

@@ -202,7 +202,10 @@ xopen_for_read(const tchar *path, bool symlink_ok, struct file_stream *strm)
 		return -1;
 	}
 
-#if defined(HAVE_POSIX_FADVISE) && (O_SEQUENTIAL == 0)
+#if O_SEQUENTIAL == 0 && \
+	(defined(HAVE_POSIX_FADVISE) || \
+	 /* fallback detection method for direct compilation */ \
+	 (!defined(HAVE_CONFIG_H) && defined(POSIX_FADV_SEQUENTIAL)))
 	(void)posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
 #endif
 
@@ -351,8 +354,11 @@ map_file_contents(struct file_stream *strm, u64 size)
 	strm->mmap_mem = mmap(NULL, size, PROT_READ, MAP_SHARED, strm->fd, 0);
 	if (strm->mmap_mem == MAP_FAILED) {
 		strm->mmap_mem = NULL;
-		if (errno == ENODEV) /* mmap isn't supported on this file */
+		if (errno == ENODEV /* standard */ ||
+		    errno == EINVAL /* macOS */) {
+			/* mmap isn't supported on this file */
 			return read_full_contents(strm);
+		}
 		if (errno == ENOMEM) {
 			msg("%"TS" is too large to be processed by this "
 			    "program", strm->name);
@@ -363,7 +369,9 @@ map_file_contents(struct file_stream *strm, u64 size)
 		return -1;
 	}
 
-#ifdef HAVE_POSIX_MADVISE
+#if defined(HAVE_POSIX_MADVISE) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(POSIX_MADV_SEQUENTIAL))
 	(void)posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL);
 #endif
 	strm->mmap_token = strm; /* anything that's not NULL */

programs/prog_util.h

@@ -1,5 +1,5 @@
 /*
- * prog_util.h - utility functions for programs
+ * prog_util.h - common header for the programs; must be included first
  *
  * Copyright 2016 Eric Biggers
  *
@@ -28,12 +28,78 @@
 #ifndef PROGRAMS_PROG_UTIL_H
 #define PROGRAMS_PROG_UTIL_H
 
+/*
+ * This header provides some utility functions and macros for the programs.  It
+ * also defines some macros that control the behavior of system headers, and for
+ * that reason it must be included before any system header.
+ *
+ * The latter part could be handled in this directory's CMakeLists.txt instead.
+ * We put as much as possible here, directly in the source, to make it easier to
+ * build the programs using other build systems (or "no build system").
+ *
+ * Note: CMakeLists.txt does do some dynamic feature detection, which can't be
+ * done in the source code.  For that reason, it duplicates some of the logic
+ * that defines macros like _GNU_SOURCE.  Keep this logic in sync.
+ */
+
+#ifdef _WIN32
+
+  /*
+   * To keep the code similar on all platforms, sometimes we intentionally use
+   * the "deprecated" non-underscore-prefixed variants of functions in msvcrt.
+   */
+#  undef _CRT_NONSTDC_NO_DEPRECATE
+#  define _CRT_NONSTDC_NO_DEPRECATE	1
+
+  /*
+   * Similarly, to match other platforms we intentionally use the "non-secure"
+   * variants, which aren't actually any less secure when used properly.
+   */
+#  undef _CRT_SECURE_NO_WARNINGS
+#  define _CRT_SECURE_NO_WARNINGS	1
+
+#else
+
+   /* Needed to work with files >= 2 GiB on 32-bit systems */
+#  undef _FILE_OFFSET_BITS
+#  define _FILE_OFFSET_BITS	64
+
+   /* Note: when making changes here, update programs/CMakeLists.txt too. */
+#  if defined(__linux__)
+     /*
+      * May be needed for clock_gettime(), posix_fadvise(), posix_madvise(),
+      * futimens(), and MAP_ANONYMOUS, depending on the C library version.
+      */
+#    undef _GNU_SOURCE
+#    define _GNU_SOURCE
+#    undef _POSIX_C_SOURCE
+#    define _POSIX_C_SOURCE	200809L
+#  elif defined(__APPLE__)
+     /* Needed for O_NOFOLLOW and MAP_ANON */
+#    undef _DARWIN_C_SOURCE
+#    define _DARWIN_C_SOURCE
+#    undef _POSIX_C_SOURCE
+#  elif defined(__sun)
+     /* Needed for futimens() */
+#    undef __EXTENSIONS__
+#    define __EXTENSIONS__
+#    undef _POSIX_C_SOURCE
+#  else
+     /*
+      * Else assume that nothing else is needed.  Don't use _POSIX_C_SOURCE on
+      * BSD, since it causes anything non-POSIX, such as MAP_ANON, to be hidden.
+      */
+#    undef _POSIX_C_SOURCE
+#  endif
+#endif
+
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
 
-#include "libdeflate.h"
+#include "../common_defs.h"
 
+#include <inttypes.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -42,36 +108,13 @@
 #  include <sys/types.h>
 #endif
 
-#include "../common/common_defs.h"
-
-#ifdef __GNUC__
+#if defined(__GNUC__) || __has_attribute(format)
 # define _printf(str_idx, args_idx)	\
 		__attribute__((format(printf, str_idx, args_idx)))
 #else
 # define _printf(str_idx, args_idx)
 #endif
 
-#ifdef _MSC_VER
-/*
- * Old versions (e.g. VS2010) of MSC have stdint.h but not the C99 header
- * inttypes.h.  Work around this by defining the PRI* macros ourselves.
- */
-# define PRIu8  "hhu"
-# define PRIu16 "hu"
-# define PRIu32 "u"
-# define PRIu64 "llu"
-# define PRIi8  "hhi"
-# define PRIi16 "hi"
-# define PRIi32 "i"
-# define PRIi64 "lli"
-# define PRIx8  "hhx"
-# define PRIx16 "hx"
-# define PRIx32 "x"
-# define PRIx64 "llx"
-#else
-# include <inttypes.h>
-#endif
-
 #ifdef _WIN32
 
 /*
@@ -80,6 +123,7 @@
  * get full Unicode support on Windows...
  */
 
+#include <io.h>
 #include <wchar.h>
 int wmain(int argc, wchar_t **argv);
 #  define	tmain		wmain

programs/test_checksums.c

@@ -5,11 +5,11 @@
  * results as their zlib equivalents.
  */
 
+#include "test_util.h"
+
 #include <stdlib.h>
 #include <time.h>
 
-#include "test_util.h"
-
 static unsigned int rng_seed;
 
 typedef u32 (*cksum_fn_t)(u32, const void *, size_t);
@@ -122,8 +122,8 @@ test_adler32(const void *buffer, size_t size, u32 initial_value)
 		       adler32_libdeflate, adler32_zlib, initial_value);
 }
 
-static void test_random_buffers(u8 *buffer, u8 *guarded_buf_end,
-				size_t limit, u32 num_iter)
+static void test_random_buffers(u8 *buf_start, u8 *buf_end, size_t limit,
+				u32 num_iter)
 {
 	for (u32 i = 0; i < num_iter; i++) {
 		size_t start = rand() % limit;
@@ -132,28 +132,32 @@ static void test_random_buffers(u8 *buffer, u8 *guarded_buf_end,
 		u32 c0 = select_initial_crc();
 
 		for (size_t j = start; j < start + len; j++)
-			buffer[j] = rand();
+			buf_start[j] = rand();
 
 		/* Test with chosen size and alignment */
-		test_adler32(&buffer[start], len, a0);
-		test_crc32(&buffer[start], len, c0);
+		test_adler32(&buf_start[start], len, a0);
+		test_crc32(&buf_start[start], len, c0);
+
+		/* Test with chosen size, with guard page before input buffer */
+		memmove(buf_start, &buf_start[start], len);
+		test_adler32(buf_start, len, a0);
+		test_crc32(buf_start, len, c0);
 
 		/* Test with chosen size, with guard page after input buffer */
-		memcpy(guarded_buf_end - len, &buffer[start], len);
-		test_adler32(guarded_buf_end - len, len, a0);
-		test_crc32(guarded_buf_end - len, len, c0);
+		memmove(buf_end - len, buf_start, len);
+		test_adler32(buf_end - len, len, a0);
+		test_crc32(buf_end - len, len, c0);
 	}
 }
 
 int
 tmain(int argc, tchar *argv[])
 {
-	u8 *buffer = xmalloc(32768);
-	u8 *guarded_buf_start, *guarded_buf_end;
+	u8 *buf_start, *buf_end;
 
 	begin_program(argv);
 
-	alloc_guarded_buffer(32768, &guarded_buf_start, &guarded_buf_end);
+	alloc_guarded_buffer(262144, &buf_start, &buf_end);
 
 	rng_seed = time(NULL);
 	srand(rng_seed);
@@ -164,9 +168,10 @@ tmain(int argc, tchar *argv[])
 	test_initial_values(crc32_zlib, 0);
 
 	/* Test different buffer sizes and alignments */
-	test_random_buffers(buffer, guarded_buf_end, 256, 5000);
-	test_random_buffers(buffer, guarded_buf_end, 1024, 500);
-	test_random_buffers(buffer, guarded_buf_end, 32768, 50);
+	test_random_buffers(buf_start, buf_end, 256,  5000);
+	test_random_buffers(buf_start, buf_end, 1024,  500);
+	test_random_buffers(buf_start, buf_end, 32768,  50);
+	test_random_buffers(buf_start, buf_end, 262144, 50);
 
 	/*
 	 * Test Adler-32 overflow cases.  For example, given all 0xFF bytes and
@@ -176,7 +181,7 @@ tmain(int argc, tchar *argv[])
 	 * before that point.  Also, some implementations make use of 16-bit
 	 * counters which can overflow earlier.
 	 */
-	memset(buffer, 0xFF, 32768);
+	memset(buf_start, 0xFF, 32768);
 	for (u32 i = 0; i < 20; i++) {
 		u32 initial_value;
 
@@ -185,12 +190,11 @@ tmain(int argc, tchar *argv[])
 		else
 			initial_value = select_initial_adler();
 
-		test_adler32(buffer, 5553, initial_value);
-		test_adler32(buffer, rand() % 32769, initial_value);
-		buffer[rand() % 32768] = 0xFE;
+		test_adler32(buf_start, 5553, initial_value);
+		test_adler32(buf_start, rand() % 32769, initial_value);
+		buf_start[rand() % 32768] = 0xFE;
 	}
 
-	free(buffer);
-	free_guarded_buffer(guarded_buf_start, guarded_buf_end);
+	free_guarded_buffer(buf_start, buf_end);
 	return 0;
 }

programs/test_custom_malloc.c

@@ -1,7 +1,7 @@
 /*
  * test_custom_malloc.c
  *
- * Test libdeflate_set_memory_allocator().
+ * Test the support for custom memory allocators.
  * Also test injecting allocation failures.
  */
 
@@ -28,24 +28,34 @@ static void do_free(void *ptr)
 	free(ptr);
 }
 
-int
-tmain(int argc, tchar *argv[])
+static void reset_state(void)
 {
+	libdeflate_set_memory_allocator(malloc, free);
+	malloc_count = 0;
+	free_count = 0;
+}
+
+/* Test that the custom allocator is actually used when requested. */
+static void do_custom_memalloc_test(bool global)
+{
+	static const struct libdeflate_options options = {
+		.sizeof_options = sizeof(options),
+		.malloc_func = do_malloc,
+		.free_func = do_free,
+	};
 	int level;
 	struct libdeflate_compressor *c;
 	struct libdeflate_decompressor *d;
 
-	begin_program(argv);
-
-	/* Test that the custom allocator is actually used when requested. */
-
-	libdeflate_set_memory_allocator(do_malloc, do_free);
-	ASSERT(malloc_count == 0);
-	ASSERT(free_count == 0);
+	if (global)
+		libdeflate_set_memory_allocator(do_malloc, do_free);
 
 	for (level = 0; level <= 12; level++) {
 		malloc_count = free_count = 0;
-		c = libdeflate_alloc_compressor(level);
+		if (global)
+			c = libdeflate_alloc_compressor(level);
+		else
+			c = libdeflate_alloc_compressor_ex(level, &options);
 		ASSERT(c != NULL);
 		ASSERT(malloc_count == 1);
 		ASSERT(free_count == 0);
@@ -55,7 +65,10 @@ tmain(int argc, tchar *argv[])
 	}
 
 	malloc_count = free_count = 0;
-	d = libdeflate_alloc_decompressor();
+	if (global)
+		d = libdeflate_alloc_decompressor();
+	else
+		d = libdeflate_alloc_decompressor_ex(&options);
 	ASSERT(d != NULL);
 	ASSERT(malloc_count == 1);
 	ASSERT(free_count == 0);
@@ -63,7 +76,52 @@ tmain(int argc, tchar *argv[])
 	ASSERT(malloc_count == 1);
 	ASSERT(free_count == 1);
 
-	/* As long as we're here, also test injecting allocation failures. */
+	reset_state();
+}
+
+#define offsetofend(type, field) \
+	(offsetof(type, field) + sizeof(((type *)NULL)->field))
+
+/* Test some edge cases involving libdeflate_options. */
+static void do_options_test(void)
+{
+	struct libdeflate_options options = { 0 };
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	/* Size in libdeflate v1.19 */
+	size_t min_size = offsetofend(struct libdeflate_options, free_func);
+
+	/* sizeof_options must be at least the minimum size. */
+	for (; options.sizeof_options < min_size;
+	     options.sizeof_options++) {
+		c = libdeflate_alloc_compressor_ex(6, &options);
+		ASSERT(c == NULL);
+		d = libdeflate_alloc_decompressor_ex(&options);
+		ASSERT(d == NULL);
+	}
+
+	/* NULL malloc_func and free_func means "use the global allocator". */
+	options.sizeof_options = min_size;
+	malloc_count = free_count = 0;
+	libdeflate_set_memory_allocator(do_malloc, do_free);
+	c = libdeflate_alloc_compressor_ex(6, &options);
+	libdeflate_free_compressor(c);
+	ASSERT(malloc_count == 1);
+	ASSERT(free_count == 1);
+	d = libdeflate_alloc_decompressor_ex(&options);
+	libdeflate_free_decompressor(d);
+	ASSERT(malloc_count == 2);
+	ASSERT(free_count == 2);
+
+	reset_state();
+}
+
+/* Test injecting memory allocation failures. */
+static void do_fault_injection_test(void)
+{
+	int level;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
 
 	libdeflate_set_memory_allocator(do_fail_malloc, do_free);
 
@@ -81,5 +139,17 @@ tmain(int argc, tchar *argv[])
 	ASSERT(malloc_count == 1);
 	ASSERT(free_count == 0);
 
+	reset_state();
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	begin_program(argv);
+
+	do_custom_memalloc_test(true);
+	do_custom_memalloc_test(false);
+	do_options_test();
+	do_fault_injection_test();
 	return 0;
 }

programs/test_invalid_streams.c

@@ -0,0 +1,130 @@
+/*
+ * test_invalid_streams.c
+ *
+ * Test that invalid DEFLATE streams are rejected with LIBDEFLATE_BAD_DATA.
+ *
+ * This isn't actually very important, since DEFLATE doesn't have built-in error
+ * detection, so corruption of a DEFLATE stream can only be reliably detected
+ * using a separate checksum anyway.  As long as the DEFLATE decompressor
+ * handles all streams safely (no crashes, etc.), in practice it is fine for it
+ * to automatically remap invalid streams to valid streams, instead of returning
+ * an error.  Corruption detection is the responsibility of the zlib or gzip
+ * layer, or the application when an external checksum is used.
+ *
+ * Nevertheless, to reduce surprises when people intentionally compare zlib's
+ * and libdeflate's handling of invalid DEFLATE streams, libdeflate implements
+ * zlib's strict behavior when decoding DEFLATE, except when it would have a
+ * significant performance cost.
+ */
+
+#include "test_util.h"
+
+static void
+assert_decompression_error(const u8 *in, size_t in_nbytes)
+{
+	struct libdeflate_decompressor *d;
+	z_stream z;
+	u8 out[128];
+	const size_t out_nbytes_avail = sizeof(out);
+	size_t actual_out_nbytes;
+	enum libdeflate_result res;
+
+	/* libdeflate */
+	d = libdeflate_alloc_decompressor();
+	ASSERT(d != NULL);
+	res = libdeflate_deflate_decompress(d, in, in_nbytes,
+					    out, out_nbytes_avail,
+					    &actual_out_nbytes);
+	ASSERT(res == LIBDEFLATE_BAD_DATA);
+	libdeflate_free_decompressor(d);
+
+	/* zlib, as a control */
+	memset(&z, 0, sizeof(z));
+	res = inflateInit2(&z, -15);
+	ASSERT(res == Z_OK);
+	z.next_in = (void *)in;
+	z.avail_in = in_nbytes;
+	z.next_out = (void *)out;
+	z.avail_out = out_nbytes_avail;
+	res = inflate(&z, Z_FINISH);
+	ASSERT(res == Z_DATA_ERROR);
+	inflateEnd(&z);
+}
+
+/*
+ * Test that DEFLATE decompression returns an error if a block header contains
+ * too many encoded litlen and offset codeword lengths.
+ */
+static void
+test_too_many_codeword_lengths(void)
+{
+	u8 in[128];
+	struct output_bitstream os = { .next = in, .end = in + sizeof(in) };
+	int i;
+
+	ASSERT(put_bits(&os, 1, 1));	/* BFINAL: 1 */
+	ASSERT(put_bits(&os, 2, 2));	/* BTYPE: DYNAMIC_HUFFMAN */
+
+	/*
+	 * Litlen code:
+	 *	litlensym_255			len=1 codeword=0
+	 *	litlensym_256 (end-of-block)	len=1 codeword=1
+	 * Offset code:
+	 *	(empty)
+	 *
+	 * Litlen and offset codeword lengths:
+	 *	[0..254] = 0	presym_{18,18}
+	 *	[255]	 = 1	presym_1
+	 *	[256]	 = 1	presym_1
+	 *	[257...] = 0	presym_18 [TOO MANY]
+	 *
+	 * Precode:
+	 *	presym_1	len=1 codeword=0
+	 *	presym_18	len=1 codeword=1
+	 */
+
+	ASSERT(put_bits(&os, 0, 5));	/* num_litlen_syms: 0 + 257 */
+	ASSERT(put_bits(&os, 0, 5));	/* num_offset_syms: 0 + 1 */
+	ASSERT(put_bits(&os, 14, 4));	/* num_explicit_precode_lens: 14 + 4 */
+
+	/*
+	 * Precode codeword lengths: order is
+	 * [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+	 */
+	for (i = 0; i < 2; i++)		/* presym_{16,17}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_18: len=1 */
+	ASSERT(put_bits(&os, 0, 3));	/* presym_0: len=0 */
+	for (i = 0; i < 13; i++)	/* presym_{8,...,14}: len=0 */
+		ASSERT(put_bits(&os, 0, 3));
+	ASSERT(put_bits(&os, 1, 3));	/* presym_1: len=1 */
+
+	/* Litlen and offset codeword lengths */
+	ASSERT(put_bits(&os, 0x1, 1) &&	/* presym_18, 128 zeroes */
+	       put_bits(&os, 117, 7));
+	ASSERT(put_bits(&os, 0x1, 1) &&	/* presym_18, 127 zeroes */
+	       put_bits(&os, 116, 7));
+	ASSERT(put_bits(&os, 0x0, 1));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x0, 1));	/* presym_1 */
+	ASSERT(put_bits(&os, 0x1, 1) &&	/* presym_18, 128 zeroes [TOO MANY] */
+	       put_bits(&os, 117, 7));
+
+	/* Literal */
+	ASSERT(put_bits(&os, 0x0, 0));	/* litlensym_255 */
+
+	/* End of block */
+	ASSERT(put_bits(&os, 0x1, 1));	/* litlensym_256 */
+
+	ASSERT(flush_bits(&os));
+
+	assert_decompression_error(in, os.next - in);
+}
+
+int
+tmain(int argc, tchar *argv[])
+{
+	begin_program(argv);
+
+	test_too_many_codeword_lengths();
+	return 0;
+}

programs/test_litrunlen_overflow.c

@@ -20,7 +20,7 @@
 int
 tmain(int argc, tchar *argv[])
 {
-	const size_t data_size = 2 * 250 * 251;
+	const int data_size = 2 * 250 * 251;
 	u8 *orig_data, *compressed_data, *decompressed_data;
 	int i, stride, multiple, j = 0;
 	struct libdeflate_decompressor *d;

programs/test_trailing_bytes.c

@@ -8,16 +8,15 @@
 #include "test_util.h"
 
 static const struct {
-	size_t (LIBDEFLATEAPI *compress)(
-			struct libdeflate_compressor *compressor,
-			const void *in, size_t in_nbytes,
-			void *out, size_t out_nbytes_avail);
-	enum libdeflate_result (LIBDEFLATEAPI *decompress)(
+	size_t (*compress)(struct libdeflate_compressor *compressor,
+			   const void *in, size_t in_nbytes,
+			   void *out, size_t out_nbytes_avail);
+	enum libdeflate_result (*decompress)(
 			struct libdeflate_decompressor *decompressor,
 			const void *in, size_t in_nbytes,
 			void *out, size_t out_nbytes_avail,
 			size_t *actual_out_nbytes_ret);
-	enum libdeflate_result (LIBDEFLATEAPI *decompress_ex)(
+	enum libdeflate_result (*decompress_ex)(
 			struct libdeflate_decompressor *decompressor,
 			const void *in, size_t in_nbytes,
 			void *out, size_t out_nbytes_avail,

programs/test_util.c

@@ -25,16 +25,6 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef _WIN32
-/* for MAP_ANONYMOUS or MAP_ANON, which unfortunately aren't part of POSIX... */
-#  undef _POSIX_C_SOURCE
-#  ifdef __APPLE__
-#    define _DARWIN_C_SOURCE
-#  elif defined(__linux__)
-#    define _GNU_SOURCE
-#  endif
-#endif
-
 #include "test_util.h"
 
 #include <fcntl.h>
@@ -52,7 +42,7 @@
 #endif
 
 /* Abort with an error message */
-_noreturn void
+NORETURN void
 assertion_failed(const char *expr, const char *file, int line)
 {
 	msg("Assertion failed: %s at %s:%d", expr, file, line);
@@ -158,7 +148,9 @@ timer_ticks(void)
 
 	QueryPerformanceCounter(&count);
 	return count.QuadPart;
-#elif defined(HAVE_CLOCK_GETTIME)
+#elif defined(HAVE_CLOCK_GETTIME) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(CLOCK_MONOTONIC))
 	struct timespec ts;
 
 	clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -182,7 +174,9 @@ timer_frequency(void)
 
 	QueryPerformanceFrequency(&freq);
 	return freq.QuadPart;
-#elif defined(HAVE_CLOCK_GETTIME)
+#elif defined(HAVE_CLOCK_GETTIME) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(CLOCK_MONOTONIC))
 	return 1000000000;
 #else
 	return 1000000;

programs/test_util.h

@@ -28,17 +28,11 @@
 #ifndef PROGRAMS_TEST_UTIL_H
 #define PROGRAMS_TEST_UTIL_H
 
-#include "prog_util.h"
+#include "prog_util.h" /* must be included first */
 
 #include <zlib.h> /* for comparison purposes */
 
-#ifdef __GNUC__
-# define _noreturn __attribute__((noreturn))
-#else
-# define _noreturn
-#endif
-
-void _noreturn
+NORETURN void
 assertion_failed(const char *expr, const char *file, int line);
 
 #define ASSERT(expr) { if (unlikely(!(expr))) \

scripts/afl-fuzz/Makefile

@@ -1,11 +0,0 @@
-SRC := $(wildcard */*.c)
-EXE := $(SRC:.c=)
-
-LDLIBS := -ldeflate
-LDFLAGS := -L../..
-CPPFLAGS := -I../..
-
-all:$(EXE)
-
-clean:
-	rm -f $(EXE)

scripts/afl-fuzz/deflate_compress/fuzz.c

@@ -1,56 +0,0 @@
-#include <assert.h>
-#include <libdeflate.h>
-#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-int main(int argc, char **argv)
-{
-	struct libdeflate_decompressor *d;
-	struct libdeflate_compressor *c;
-	int ret;
-	int fd = open(argv[1], O_RDONLY);
-	struct stat stbuf;
-	unsigned char level;
-	unsigned char use_bound;
-
-	assert(fd >= 0);
-	ret = fstat(fd, &stbuf);
-	assert(!ret);
-
-	if (stbuf.st_size < 2)
-		return 0;
-	ret = read(fd, &level, 1);
-	assert(ret == 1);
-	level %= 13;
-
-	ret = read(fd, &use_bound, 1);
-	assert(ret == 1);
-	use_bound %= 2;
-
-	char in[stbuf.st_size - 2];
-	ret = read(fd, in, sizeof in);
-	assert(ret == sizeof in);
-
-	c = libdeflate_alloc_compressor(level);
-	d = libdeflate_alloc_decompressor();
-
-	size_t outsize = use_bound ? libdeflate_deflate_compress_bound(c, sizeof(in)) : sizeof(in);
-	char out[outsize];
-	char checkarray[sizeof(in)];
-
-	size_t csize = libdeflate_deflate_compress(c, in,sizeof in, out, outsize);
-	if (csize != 0) {
-		enum libdeflate_result res;
-		res = libdeflate_deflate_decompress(d, out, csize, checkarray, sizeof in, NULL);
-		assert(!res);
-		assert(!memcmp(in, checkarray, sizeof in));
-	} else {
-		assert(!use_bound);
-	}
-
-	libdeflate_free_compressor(c);
-	libdeflate_free_decompressor(d);
-	return 0;
-}

scripts/afl-fuzz/deflate_decompress/fuzz.c

@@ -1,28 +0,0 @@
-#include <assert.h>
-#include <libdeflate.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-int main(int argc, char **argv)
-{
-	struct libdeflate_decompressor *d;
-	int ret;
-	int fd = open(argv[1], O_RDONLY);
-	struct stat stbuf;
-	assert(fd >= 0);
-	ret = fstat(fd, &stbuf);
-	assert(!ret);
-
-	char in[stbuf.st_size];
-	ret = read(fd, in, sizeof in);
-	assert(ret == sizeof in);
-
-	char out[sizeof(in) * 3];
-
-	d = libdeflate_alloc_decompressor();
-
-	libdeflate_deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	libdeflate_free_decompressor(d);
-	return 0;
-}

scripts/afl-fuzz/fuzz.sh

@@ -1,149 +0,0 @@
-#!/bin/bash
-
-set -e -u -o pipefail
-
-cd "$(dirname "$0")"
-
-read -r -a AVAILABLE_TARGETS < <(echo */fuzz.c | sed 's@/fuzz.c@@g')
-
-usage()
-{
-	cat << EOF
-Usage: $0 [OPTION]... [TARGET]...
-
-Fuzz libdeflate with afl-fuzz.
-
-Options:
-   --asan          Enable AddressSanitizer
-   --no-resume     Don't resume existing afl-fuzz session; start a new one
-   --ubsan         Enable UndefinedBehaviorSanitizer
-
-Available targets: ${AVAILABLE_TARGETS[*]}
-EOF
-}
-
-die()
-{
-	echo "$*" 1>&2
-	exit 1
-}
-
-asan=false
-ubsan=false
-may_resume=true
-
-longopts_array=(
-asan
-help
-no-resume
-ubsan
-)
-longopts=$(echo "${longopts_array[@]}" | tr ' ' ',')
-
-if ! options=$(getopt -o "" -l "$longopts" -- "$@"); then
-	usage 1>&2
-	exit 1
-fi
-eval set -- "$options"
-while (( $# >= 0 )); do
-	case "$1" in
-	--asan)
-		asan=true
-		;;
-	--help)
-		usage
-		exit 0
-		;;
-	--no-resume)
-		may_resume=false
-		;;
-	--ubsan)
-		ubsan=true
-		;;
-	--)
-		shift
-		break
-		;;
-	*)
-		echo 1>&2 "Invalid option: \"$1\""
-		usage 1>&2
-		exit 1
-	esac
-	shift
-done
-
-if $asan && $ubsan; then
-	die "--asan and --ubsan are mutually exclusive"
-fi
-
-if ! type -P afl-fuzz > /dev/null; then
-	die "afl-fuzz is not installed"
-fi
-
-if (( $# == 0 )); then
-	targets=("${AVAILABLE_TARGETS[@]}")
-else
-	for target; do
-		found=false
-		for t in "${AVAILABLE_TARGETS[@]}"; do
-			if [ "$target" = "$t" ]; then
-				found=true
-			fi
-		done
-		if ! $found; then
-			echo 1>&2 "Unknown target '$target'"
-			echo 1>&2 "Available targets: ${AVAILABLE_TARGETS[*]}"
-			exit 1
-		fi
-	done
-	targets=("$@")
-fi
-if (( ${#targets[@]} > 1 )) && ! type -P urxvt > /dev/null; then
-	die "urxvt is not installed"
-fi
-
-afl_opts=""
-if $asan; then
-	export AFL_USE_ASAN=1
-	export CFLAGS="-O2 -m32"
-	export CC=afl-clang
-	afl_opts+=" -m 800"
-elif $ubsan; then
-	export CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined"
-	export CC=afl-gcc
-else
-	export AFL_HARDEN=1
-	export CFLAGS="-O2"
-	export CC=afl-clang-fast
-fi
-CFLAGS+=" -DLIBDEFLATE_ENABLE_ASSERTIONS"
-
-sudo sh -c "echo core > /proc/sys/kernel/core_pattern"
-sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
-
-NPROC=$(getconf _NPROCESSORS_ONLN)
-
-make "-j$NPROC" -C ../../ libdeflate.a
-make "-j$NPROC" -B
-
-for dir in "${targets[@]}"; do
-	cp -vaT "$dir" "/tmp/$dir"
-	indir=/tmp/$dir/inputs
-	outdir=/tmp/$dir/outputs
-	if [ -e "$outdir" ]; then
-		if $may_resume; then
-			indir="-"
-		else
-			rm -rf "${outdir:?}"/*
-		fi
-	else
-		mkdir "$outdir"
-	fi
-	cmd="afl-fuzz -i $indir -o $outdir -T $dir $afl_opts -- /tmp/$dir/fuzz @@"
-	if (( ${#targets[@]} > 1 )); then
-		urxvt -e bash -c "$cmd" &
-	else
-		$cmd
-	fi
-done
-wait

scripts/afl-fuzz/gzip_decompress/fuzz.c

@@ -1,28 +0,0 @@
-#include <assert.h>
-#include <libdeflate.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-int main(int argc, char **argv)
-{
-	struct libdeflate_decompressor *d;
-	int ret;
-	int fd = open(argv[1], O_RDONLY);
-	struct stat stbuf;
-	assert(fd >= 0);
-	ret = fstat(fd, &stbuf);
-	assert(!ret);
-
-	char in[stbuf.st_size];
-	ret = read(fd, in, sizeof in);
-	assert(ret == sizeof in);
-
-	char out[sizeof(in) * 3];
-
-	d = libdeflate_alloc_decompressor();
-
-	libdeflate_gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	libdeflate_free_decompressor(d);
-	return 0;
-}

scripts/afl-fuzz/zlib_decompress/fuzz.c

@@ -1,28 +0,0 @@
-#include <assert.h>
-#include <libdeflate.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-int main(int argc, char **argv)
-{
-	struct libdeflate_decompressor *d;
-	int ret;
-	int fd = open(argv[1], O_RDONLY);
-	struct stat stbuf;
-	assert(fd >= 0);
-	ret = fstat(fd, &stbuf);
-	assert(!ret);
-
-	char in[stbuf.st_size];
-	ret = read(fd, in, sizeof in);
-	assert(ret == sizeof in);
-
-	char out[sizeof(in) * 3];
-
-	d = libdeflate_alloc_decompressor();
-
-	libdeflate_zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
-	libdeflate_free_decompressor(d);
-	return 0;
-}

scripts/android_build.sh

@@ -2,20 +2,22 @@
 
 set -eu -o pipefail
 
+SCRIPTDIR="$(dirname "$0")"
+BUILDDIR="$SCRIPTDIR/../build"
 API_LEVEL=28
 ARCH=arm64
-CFLAGS=
+CFLAGS=${CFLAGS:-}
 ENABLE_CRC=false
 ENABLE_CRYPTO=false
-NDKDIR=$HOME/android-ndk-r21d
+NDKDIR=$HOME/android-ndk-r25b
 
 usage() {
 	cat << EOF
-Usage: $0 [OPTION]... -- [MAKE_TARGET]...
+Usage: $0 [OPTION]...
 Build libdeflate for Android.
 
   --api-level=LEVEL    Android API level to target (default: $API_LEVEL)
-  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
+  --arch=ARCH          Architecture: arm32|arm64|x86|x86_64 (default: $ARCH)
   --enable-crc         Enable crc instructions
   --enable-crypto      Enable crypto instructions
   --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
@@ -65,13 +67,11 @@ while [ $# -gt 0 ]; do
 	shift
 done
 
-BINDIR=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/
-
 case "$ARCH" in
-arm|arm32|aarch32)
-	CC=$BINDIR/armv7a-linux-androideabi$API_LEVEL-clang
+arm|arm32|aarch32|armeabi-v7a)
+	ANDROID_ABI=armeabi-v7a
 	if $ENABLE_CRC || $ENABLE_CRYPTO; then
-		CFLAGS="-march=armv8-a"
+		CFLAGS+=" -march=armv8-a"
 		if $ENABLE_CRC; then
 			CFLAGS+=" -mcrc"
 		else
@@ -84,8 +84,8 @@ arm|arm32|aarch32)
 		fi
 	fi
 	;;
-arm64|aarch64)
-	CC=$BINDIR/aarch64-linux-android$API_LEVEL-clang
+arm64|aarch64|arm64-v8a)
+	ANDROID_ABI=arm64-v8a
 	features=""
 	if $ENABLE_CRC; then
 		features+="+crc"
@@ -94,15 +94,25 @@ arm64|aarch64)
 		features+="+crypto"
 	fi
 	if [ -n "$features" ]; then
-		CFLAGS="-march=armv8-a$features"
+		CFLAGS+=" -march=armv8-a$features"
 	fi
 	;;
+x86)
+	ANDROID_ABI=x86
+	;;
+x86_64)
+	ANDROID_ABI=x86_64
+	;;
 *)
 	echo 1>&2 "Unknown architecture: \"$ARCH\""
 	usage 1>&2
 	exit 1
 esac
 
-cmd=(make "-j$(grep -c processor /proc/cpuinfo)" "CC=$CC" "CFLAGS=$CFLAGS" "$@")
-echo "${cmd[*]}"
-"${cmd[@]}"
+"$SCRIPTDIR"/cmake-helper.sh -G Ninja \
+	-DCMAKE_TOOLCHAIN_FILE="$NDKDIR"/build/cmake/android.toolchain.cmake \
+	-DCMAKE_C_FLAGS="$CFLAGS" \
+	-DANDROID_ABI="$ANDROID_ABI" \
+	-DANDROID_PLATFORM="$API_LEVEL" \
+	-DLIBDEFLATE_BUILD_TESTS=1
+cmake --build "$BUILDDIR"

scripts/android_tests.sh

@@ -12,7 +12,7 @@ if [ $# -ne 0 ]; then
 fi
 
 # Use NDKDIR if specified in environment, else use default value.
-: "${NDKDIR:=$HOME/android-ndk-r21d}"
+: "${NDKDIR:=$HOME/android-ndk-r25b}"
 if [ ! -e "$NDKDIR" ]; then
 	cat 1>&2 << EOF
 Android NDK was not found in NDKDIR=$NDKDIR!  Set the
@@ -45,10 +45,9 @@ CLEANUP_CMDS+=("rm -r '$TMPDIR'")
 android_build_and_test() {
 	echo "Running Android tests with $*"
 
-	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" \
-		all test_programs > /dev/null
-	adb push "$TESTDATA" ./scripts/exec_tests.sh benchmark test_* \
-		/data/local/tmp/ > /dev/null
+	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" > /dev/null
+	adb push "$TESTDATA" ./scripts/exec_tests.sh \
+		./build/programs/{benchmark,test_*} /data/local/tmp/ > /dev/null
 
 	# Note: adb shell always returns 0, even if the shell command fails...
 	adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \
@@ -60,10 +59,11 @@ android_build_and_test() {
 	fi
 }
 
-for arch in arm32 arm64; do
-	android_build_and_test --arch=$arch
-	android_build_and_test --arch=$arch --enable-crc
-	android_build_and_test --arch=$arch --enable-crypto
-	android_build_and_test --arch=$arch --enable-crc --enable-crypto
-done
+android_build_and_test --arch=arm32
+android_build_and_test --arch=arm32 --enable-crc
+android_build_and_test --arch=arm64
+android_build_and_test --arch=arm64 --enable-crc
+android_build_and_test --arch=arm64 --enable-crypto
+android_build_and_test --arch=arm64 --enable-crc --enable-crypto
+
 echo "Android tests passed"

scripts/benchmark.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTDIR="$(dirname "$(realpath "$0")")"
+BUILDDIR="$SCRIPTDIR/../build"
+
+"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja > /dev/null
+ninja -C "$BUILDDIR" --quiet benchmark
+"$BUILDDIR"/programs/benchmark "$@"

scripts/checksum.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTDIR="$(dirname "$(realpath "$0")")"
+BUILDDIR="$SCRIPTDIR/../build"
+
+"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja > /dev/null
+ninja -C "$BUILDDIR" --quiet checksum
+"$BUILDDIR"/programs/checksum "$@"

scripts/checksum_benchmarks.sh

@@ -2,7 +2,7 @@
 
 set -eu -o pipefail
 
-have_cpu_feature() {
+__have_cpu_feature() {
 	local feature="$1"
 	local tag
 	case $ARCH in
@@ -16,14 +16,25 @@ have_cpu_feature() {
 	grep -q "^$tag"$'[ \t]'"*:.*\<$feature\>" /proc/cpuinfo
 }
 
+have_cpu_features() {
+	local feature
+	for feature; do
+		__have_cpu_feature "$feature" || return 1
+	done
+}
+
 make_and_test() {
 	# Build the checksum program and tests.  Set the special test support
 	# flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES.
-	make "$@" TEST_SUPPORT__DO_NOT_USE=1 checksum test_checksums > /dev/null
+	rm -rf build
+	CFLAGS="$CFLAGS -DTEST_SUPPORT__DO_NOT_USE=1" \
+		cmake -B build -G Ninja -DLIBDEFLATE_BUILD_TESTS=1 \
+		"${EXTRA_CMAKE_FLAGS[@]}" > /dev/null
+	cmake --build build > /dev/null
 
 	# Run the checksum tests, for good measure.  (This isn't actually part
 	# of the benchmarking.)
-	./test_checksums > /dev/null
+	./build/programs/test_checksums > /dev/null
 }
 
 __do_benchmark() {
@@ -31,21 +42,22 @@ __do_benchmark() {
 	shift
 	local flags=("$@")
 
-	speed=$(./checksum "${CKSUM_FLAGS[@]}" "${flags[@]}" -t "$FILE" | \
+	speed=$(./build/programs/checksum "${CKSUM_FLAGS[@]}" \
+		"${flags[@]}" -t "$FILE" | \
 		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
-	printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
+	printf "%-60s%-10s\n" "$CKSUM_NAME ($impl)" "$speed"
 }
 
 do_benchmark() {
 	local impl="$1"
 
+	CFLAGS="${EXTRA_CFLAGS[*]}" make_and_test
 	if [ "$impl" = zlib ]; then
 		__do_benchmark "$impl" "-Z"
 	else
-		make_and_test CFLAGS="${EXTRA_CFLAGS[*]}"
 		__do_benchmark "libdeflate, $impl"
-		if [ "$ARCH" = x86_64 ]; then
-			make_and_test CFLAGS="-m32 ${EXTRA_CFLAGS[*]}"
+		if $ENABLE_32BIT; then
+			CFLAGS="-m32 ${EXTRA_CFLAGS[*]}" make_and_test
 			__do_benchmark "libdeflate, $impl, 32-bit"
 		fi
 	fi
@@ -56,12 +68,11 @@ sort_by_speed() {
 }
 
 disable_cpu_feature() {
-	local name="$1"
+	LIBDEFLATE_DISABLE_CPU_FEATURES+=",$1"
 	shift
-	local extra_cflags=("$@")
-
-	LIBDEFLATE_DISABLE_CPU_FEATURES+=",$name"
-	EXTRA_CFLAGS+=("${extra_cflags[@]}")
+	if (( $# > 0 )); then
+		EXTRA_CFLAGS+=("$@")
+	fi
 }
 
 cleanup() {
@@ -72,27 +83,66 @@ cleanup() {
 
 ARCH="$(uname -m)"
 USING_TMPFILE=false
-
-if (( $# > 1 )); then
-	echo "Usage: $0 [FILE]" 1>&2
-	exit 1
-fi
+EXTRA_CMAKE_FLAGS=()
+ENABLE_32BIT=false
 
 trap cleanup EXIT
 
+longopts="help"
+longopts+=",cmake-flag:"
+longopts+=",enable-32bit"
+
+usage() {
+	echo "Usage: $0 [--cmake-flag=FLAG]... [--enable-32bit] [FILE]"
+}
+
+if ! options=$(getopt -o "" -l "$longopts" -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+eval set -- "$options"
+while (( $# >= 1 )); do
+	case "$1" in
+	--cmake-flag)
+		EXTRA_CMAKE_FLAGS+=("$2")
+		shift
+		;;
+	--enable-32bit)
+		ENABLE_32BIT=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Invalid option: '$1'"
+		usage 1>&2
+		exit 1
+		;;
+	esac
+	shift
+done
+
 if (( $# == 0 )); then
 	# Generate default test data file.
 	FILE=$(mktemp -t checksum_testdata.XXXXXXXXXX)
 	USING_TMPFILE=true
-	echo "Generating 100 MB test file: $FILE"
-	head -c 100000000 /dev/urandom > "$FILE"
-else
+	echo "Generating 250 MB test file: $FILE"
+	head -c 250000000 /dev/urandom > "$FILE"
+elif (( $# == 1 )); then
 	FILE="$1"
+else
+	usage 1>&2
+	exit 1
 fi
 
 cat << EOF
-Method                                       Speed (MB/s)
-------                                       ------------
+Method                                                      Speed (MB/s)
+------                                                      ------------
 EOF
 
 # CRC-32
@@ -103,23 +153,44 @@ export LIBDEFLATE_DISABLE_CPU_FEATURES=""
 {
 case $ARCH in
 i386|x86_64)
-	if have_cpu_feature pclmulqdq && have_cpu_feature avx; then
-		do_benchmark "PCLMUL/AVX"
-		disable_cpu_feature "avx" "-mno-avx"
+	if have_cpu_features vpclmulqdq pclmulqdq avx512bw avx512vl; then
+		do_benchmark "VPCLMULQDQ/AVX512/VL512"
+		disable_cpu_feature zmm
+		do_benchmark "VPCLMULQDQ/AVX512/VL256"
+		disable_cpu_feature avx512vl "-mno-avx512vl"
+		disable_cpu_feature avx512bw "-mno-avx512bw"
 	fi
-	if have_cpu_feature pclmulqdq; then
-		do_benchmark "PCLMUL"
-		disable_cpu_feature "pclmul" "-mno-pclmul"
+	if have_cpu_features vpclmulqdq pclmulqdq avx2; then
+		do_benchmark "VPCLMULQDQ/AVX2"
+		disable_cpu_feature vpclmulqdq "-mno-vpclmulqdq"
+	fi
+	if have_cpu_features pclmulqdq avx; then
+		do_benchmark "PCLMULQDQ/AVX"
+		disable_cpu_feature avx "-mno-avx"
+	fi
+	if have_cpu_features pclmulqdq; then
+		do_benchmark "PCLMULQDQ"
+		disable_cpu_feature pclmulqdq "-mno-pclmul"
 	fi
 	;;
-arm*|aarch*)
-	if have_cpu_feature crc32; then
-		do_benchmark "ARM"
-		disable_cpu_feature "crc32" "-march=armv8-a+nocrc"
+aarch*)
+	EXTRA_CFLAGS=("-march=armv8-a")
+	if have_cpu_features pmull crc32 sha3; then
+		do_benchmark "pmullx12_crc_eor3"
+		disable_cpu_feature sha3
 	fi
-	if have_cpu_feature pmull; then
-		do_benchmark "PMULL"
-		disable_cpu_feature "pmull" "-march=armv8-a+nocrc+nocrypto"
+	if have_cpu_features pmull crc32; then
+		do_benchmark "pmullx12_crc"
+		disable_cpu_feature prefer_pmull
+		do_benchmark "crc_pmullcombine"
+	fi
+	if have_cpu_features crc32; then
+		do_benchmark "crc"
+		disable_cpu_feature crc32
+	fi
+	if have_cpu_features pmull; then
+		do_benchmark "pmull4x"
+		disable_cpu_feature pmull
 	fi
 	;;
 esac
@@ -136,29 +207,44 @@ echo
 {
 case $ARCH in
 i386|x86_64)
-	if have_cpu_feature avx512bw; then
-		do_benchmark "AVX-512BW"
-		disable_cpu_feature "avx512bw" "-mno-avx512bw"
+	if have_cpu_features avx512bw avx512_vnni; then
+		do_benchmark "AVX512VNNI/VL512"
+		disable_cpu_feature zmm
+		if have_cpu_features avx512vl; then
+			do_benchmark "AVX512VNNI/VL256"
+		fi
+		disable_cpu_feature avx512_vnni "-mno-avx512vnni"
+		disable_cpu_feature avx512bw "-mno-avx512bw"
 	fi
-	if have_cpu_feature avx2; then
+	if have_cpu_features avx2 avx_vnni; then
+		do_benchmark "AVX-VNNI"
+		disable_cpu_feature avx_vnni "-mno-avxvnni"
+	fi
+	if have_cpu_features avx2; then
 		do_benchmark "AVX2"
-		disable_cpu_feature "avx2" "-mno-avx2"
+		disable_cpu_feature avx2 "-mno-avx2"
 	fi
-	if have_cpu_feature sse2; then
+	if have_cpu_features sse2; then
 		do_benchmark "SSE2"
-		disable_cpu_feature "sse2" "-mno-sse2"
+		disable_cpu_feature sse2 "-mno-sse2"
 	fi
 	;;
 arm*)
-	if have_cpu_feature neon; then
+	if have_cpu_features neon; then
 		do_benchmark "NEON"
-		disable_cpu_feature "neon" "-mfpu=vfpv3"
+		disable_cpu_feature neon "-mfpu=vfpv3"
 	fi
 	;;
 aarch*)
-	if have_cpu_feature asimd; then
+	EXTRA_CFLAGS=("-march=armv8-a")
+	if have_cpu_features asimd asimddp; then
+		do_benchmark "DOTPROD"
+		disable_cpu_feature dotprod
+	fi
+	if have_cpu_features asimd; then
 		do_benchmark "NEON"
-		disable_cpu_feature "neon" "-march=armv8-a+nosimd"
+		disable_cpu_feature neon
+		EXTRA_CFLAGS=("-march=armv8-a+nosimd")
 	fi
 	;;
 esac

scripts/cmake-helper.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# This script ensures that the 'build' directory has been created and configured
+# with the given CMake options and environment.
+
+set -e
+
+TOPDIR="$(dirname "$0")"/..
+BUILDDIR="$TOPDIR"/build
+
+flags=$(env; echo "@CMAKEOPTS@=$*")
+if [ "$flags" != "$(cat "$BUILDDIR"/.flags 2>/dev/null || true)" ]; then
+	rm -rf "$BUILDDIR"/CMakeCache.txt "$BUILDDIR"/CMakeFiles
+	mkdir -p "$BUILDDIR"
+	cmake -S "$TOPDIR" -B "$BUILDDIR" "$@"
+	echo "$flags" > "$BUILDDIR"/.flags
+fi

scripts/deflate_benchmarks.sh

@@ -35,7 +35,8 @@ multifile()
 	for file in "$@"; do
 		echo -n "$(basename "$file")"
 		results=()
-		cmd=("$topdir/benchmark" -s"$(stat -c "%s" "$file")" "$file")
+		cmd=("$topdir/build/programs/benchmark"
+		     -s"$(stat -c "%s" "$file")" "$file")
 		run_benchmark "${cmd[@]}" -Y -6
 		results+=("$CSIZE")
 		run_benchmark "${cmd[@]}" -Y -6
@@ -71,7 +72,7 @@ single_file()
 	local include_old=false
 
 	usize=$(stat -c "%s" "$file")
-	: ${NUM_ITERATIONS:=3}
+	: "${NUM_ITERATIONS:=3}"
 
 	if [ -e "$topdir/benchmark-old" ]; then
 		include_old=true
@@ -90,7 +91,7 @@ single_file()
 		echo -n "$level"
 		args=("$file" -s "$usize" "-$level")
 
-		run_benchmark "$topdir/benchmark" "${args[@]}"
+		run_benchmark "$topdir/build/programs/benchmark" "${args[@]}"
 		echo -n " | $CSIZE / $CTIME"
 
 		if $include_old; then
@@ -101,7 +102,8 @@ single_file()
 		if (( level > 9 )); then
 			echo -n " | N/A"
 		else
-			run_benchmark "$topdir/benchmark" "${args[@]}" -Y
+			run_benchmark "$topdir/build/programs/benchmark" \
+				      "${args[@]}" -Y
 			echo -n " | $CSIZE / $CTIME"
 		fi
 		echo

scripts/detect.sh

@@ -1,63 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-# Use CC if specified in environment, else default to "cc".
-: "${CC:=cc}"
-
-# Use CFLAGS if specified in environment.
-: "${CFLAGS:=}"
-
-echo "/* THIS FILE WAS AUTOMATICALLY GENERATED.  DO NOT EDIT. */"
-echo "#ifndef CONFIG_H"
-echo "#define CONFIG_H"
-
-program_compiles() {
-	echo "$1" | $CC $CFLAGS -Wno-error -x c - -o /dev/null > /dev/null 2>&1
-}
-
-check_function() {
-	funcname=$1
-	macro="HAVE_$(echo "$funcname" | tr '[:lower:]' '[:upper:]')"
-
-	echo
-	echo "/* Is the $funcname() function available? */"
-	if program_compiles "int main() { $funcname(); }"; then
-		echo "#define $macro 1"
-	else
-		echo "/* $macro is not set */"
-	fi
-}
-
-have_stat_field() {
-	program_compiles "#include <sys/types.h>
-			  #include <sys/stat.h>
-			  int main() { struct stat st; st.$1; }"
-}
-
-check_stat_nanosecond_precision() {
-	echo
-	echo "/* Does stat() provide nanosecond-precision timestamps? */"
-	if have_stat_field st_atim; then
-		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
-	elif have_stat_field st_atimespec; then
-		# Nonstandard field names used by OS X and older BSDs
-		echo "#define HAVE_STAT_NANOSECOND_PRECISION 1"
-		echo "#define st_atim st_atimespec"
-		echo "#define st_mtim st_mtimespec"
-		echo "#define st_ctim st_ctimespec"
-	else
-		echo "/* HAVE_STAT_NANOSECOND_PRECISION is not set */"
-	fi
-}
-
-check_function clock_gettime
-check_function futimens
-check_function futimes
-check_function posix_fadvise
-check_function posix_madvise
-
-check_stat_nanosecond_precision
-
-echo
-echo "#endif /* CONFIG_H */"

scripts/exec_tests.sh

@@ -6,6 +6,10 @@
 
 set -eu
 
+DIR=${1:-.}
+
+cd "$DIR"
+
 run_cmd() {
 	echo "$WRAPPER $*"
 	$WRAPPER "$@" > /dev/null

scripts/gen-crc32-consts.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+#
+# This script generates constants for efficient computation of the gzip CRC-32.
+
+import sys
+
+# This is the generator polynomial G(x) of the gzip CRC-32, represented as an
+# int using the natural mapping between bits and polynomial coefficients.
+G = 0x104c11db7
+
+# XOR (add) an iterable of polynomials.
+def xor(iterable):
+    res = 0
+    for val in iterable:
+        res ^= val
+    return res
+
+# Multiply two polynomials.
+def clmul(a, b):
+    return xor(a << i for i in range(b.bit_length()) if (b & (1 << i)) != 0)
+
+# Polynomial division floor(a / b).
+def div(a, b):
+    q = 0
+    while a.bit_length() >= b.bit_length():
+        q ^= 1 << (a.bit_length() - b.bit_length())
+        a ^= b << (a.bit_length() - b.bit_length())
+    return q
+
+# Reduce the polynomial 'a' modulo the polynomial 'b'.
+def reduce(a, b):
+    return a ^ clmul(div(a, b), b)
+
+# Reverse the bits of a polynomial.
+def bitreverse(poly, num_bits):
+    return xor(1 << (num_bits - 1 - i) for i in range(num_bits)
+               if (poly & (1 << i)) != 0)
+
+# Compute x^d mod G.
+def x_to_the_d(d):
+    if d < G.bit_length() - 1:
+        return 1 << d
+    t = x_to_the_d(d//2)
+    t = clmul(t, t)
+    if d % 2 != 0:
+        t <<= 1
+    return reduce(t, G)
+
+def gen_tables():
+    print('/*')
+    print(' * crc32_tables.h - data tables for CRC-32 computation')
+    print(' *')
+    print(' * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.')
+    print(' */')
+    for n in [1, 8]:
+        print('')
+        print(f'static const u32 crc32_slice{n}_table[] MAYBE_UNUSED = {{')
+        # The i'th table entry is the CRC-32 of the message consisting of byte
+        # i % 256 followed by i // 256 zero bytes.
+        polys = [bitreverse(i % 256, 8) << (32 + 8*(i//256)) for i in range(256 * n)]
+        polys = [bitreverse(reduce(poly, G), 32) for poly in polys]
+        for i in range(0, len(polys), 4):
+            print(f'\t0x{polys[i+0]:08x}, 0x{polys[i+1]:08x}, 0x{polys[i+2]:08x}, 0x{polys[i+3]:08x},')
+        print('};')
+
+# Compute the constant multipliers needed for "folding" over various distances
+# with the gzip CRC-32.  Each such multiplier is x^d mod G(x) for some distance
+# d, in bits, over which the folding is occurring.
+#
+# Folding works as follows: let A(x) be a polynomial (possibly reduced partially
+# or fully mod G(x)) for part of the message, and let B(x) be a polynomial
+# (possibly reduced partially or fully mod G(x)) for a later part of the
+# message.  The unreduced combined polynomial is A(x)*x^d + B(x), where d is the
+# number of bits separating the two parts of the message plus len(B(x)).  Since
+# mod G(x) can be applied at any point, x^d mod G(x) can be precomputed and used
+# instead of x^d unreduced.  That allows the combined polynomial to be computed
+# relatively easily in a partially-reduced form A(x)*(x^d mod G(x)) + B(x), with
+# length max(len(A(x)) + 31, len(B(x))).  This does require doing a polynomial
+# multiplication (carryless multiplication).
+#
+# "Folding" in this way can be used for the entire CRC computation except the
+# final reduction to 32 bits; this works well when CPU support for carryless
+# multiplication is available.  It can also be used to combine CRCs of different
+# parts of the message that were computed using a different method.
+#
+# Note that the gzip CRC-32 uses bit-reversed polynomials.  I.e., the low order
+# bits are really the high order polynomial coefficients.
+def gen_multipliers():
+    print('/*')
+    print(' * crc32_multipliers.h - constants for CRC-32 folding')
+    print(' *')
+    print(' * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.')
+    print(' */')
+    print('')
+
+    # Compute the multipliers needed for CRC-32 folding with carryless
+    # multiplication instructions that operate on the 64-bit halves of 128-bit
+    # segments.  Using the terminology from earlier, for each 64-bit fold
+    # len(A(x)) = 64, and len(B(x)) = 95 since a 64-bit polynomial multiplied by
+    # a 32-bit one produces a 95-bit one.  When A(x) is the low order polynomial
+    # half of a 128-bit segments (high order physical half), the separation
+    # between the message parts is the total length of the 128-bit segments
+    # separating the values.  When A(x) is the high order polynomial half, the
+    # separation is 64 bits greater.
+    for i in range(1, 33):
+        sep_lo = 128 * (i - 1)
+        sep_hi = sep_lo + 64
+        len_B = 95
+        for d in [sep_hi + len_B, # A(x) = high 64 polynomial bits (low 64 physical bits)
+                  sep_lo + len_B # A(x) = low 64 polynomial bits (high 64 physical bits)
+                  ]:
+            poly = bitreverse(x_to_the_d(d), 32)
+            print(f'#define CRC32_X{d}_MODG 0x{poly:08x} /* x^{d} mod G(x) */')
+        print('')
+
+    # Compute constants for the final 128 => 32 bit reduction.
+    poly = bitreverse(div(1 << 95, G), 64)
+    print(f'#define CRC32_BARRETT_CONSTANT_1 0x{poly:016x}ULL /* floor(x^95 / G(x)) */')
+    poly = bitreverse(G, 33)
+    print(f'#define CRC32_BARRETT_CONSTANT_2 0x{poly:016x}ULL /* G(x) */')
+
+    # Compute multipliers for combining the CRCs of separate chunks.
+    print('')
+    num_chunks = 4
+    table_len = 129
+    min_chunk_len = 128
+    print(f'#define CRC32_NUM_CHUNKS {num_chunks}')
+    print(f'#define CRC32_MIN_VARIABLE_CHUNK_LEN {min_chunk_len}UL')
+    print(f'#define CRC32_MAX_VARIABLE_CHUNK_LEN {(table_len-1) * min_chunk_len}UL')
+    print('')
+    print('/* Multipliers for implementations that use a variable chunk length */')
+    print('static const u32 crc32_mults_for_chunklen[][CRC32_NUM_CHUNKS - 1] MAYBE_UNUSED = {')
+    print('\t{ 0 /* unused row */ },')
+    for i in range(1, table_len):
+        chunk_len = i * min_chunk_len
+        print(f'\t/* chunk_len={chunk_len} */')
+        print('\t{ ', end='')
+        for j in range(num_chunks - 1, 0, -1):
+            d = (j * 8 * chunk_len) - 33
+            poly = bitreverse(x_to_the_d(d), 32)
+            print(f'0x{poly:08x} /* x^{d} mod G(x) */, ', end='')
+        print('},')
+    print('};')
+    fixed_chunk_len = 32768
+    print('')
+    print('/* Multipliers for implementations that use a large fixed chunk length */')
+    print(f'#define CRC32_FIXED_CHUNK_LEN {fixed_chunk_len}UL')
+    for j in range(1, num_chunks):
+        d = (j * 8 * fixed_chunk_len) - 33
+        poly = bitreverse(x_to_the_d(d), 32)
+        print(f'#define CRC32_FIXED_CHUNK_MULT_{j} 0x{poly:08x} /* x^{d} mod G(x) */')
+
+with open('lib/crc32_tables.h', 'w') as f:
+    sys.stdout = f
+    gen_tables()
+with open('lib/crc32_multipliers.h', 'w') as f:
+    sys.stdout = f
+    gen_multipliers()

scripts/gen-release-archives.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This script generates source and binary archives that should be posted for
+# each new release of libdeflate.
+
+prefix="libdeflate-$(git describe HEAD | sed 's/^v//')"
+
+# Generate source code archive libdeflate-*.tar.gz
+tarball="${prefix}.tar.gz"
+echo "Generating $tarball"
+git archive --format=tar --prefix="${prefix}/" HEAD \
+	| libdeflate-gzip -12 > "$tarball"
+
+# Generate Windows binary releases libdeflate-*-windows-*-bin.zip
+for arch in 'i686' 'x86_64'; do
+	dir=${prefix}-windows-${arch}-bin
+	zipfile="${dir}.zip"
+	echo "Generating $zipfile"
+	rm -rf build "$dir" "$zipfile"
+	CFLAGS="-Werror" ${arch}-w64-mingw32-cmake -B build -G Ninja \
+		-DLIBDEFLATE_BUILD_TESTS=1 > /dev/null
+	cmake --build build > /dev/null
+	mkdir "$dir"
+	cp libdeflate.h build/libdeflate.{dll,dll.a,a} \
+		build/programs/{benchmark,checksum}.exe "$dir"
+	cp build/programs/libdeflate-gzip.exe "$dir"/gzip.exe
+	cp build/programs/libdeflate-gzip.exe "$dir"/gunzip.exe
+	${arch}-w64-mingw32-strip "$dir"/libdeflate.dll "$dir"/*.exe
+	for file in COPYING NEWS.md README.md; do
+		sed < $file > "$dir/${file}.txt" -e 's/$/\r/g'
+	done
+	(cd "$dir" && zip -q -r "../${zipfile}" .)
+done
+
+echo "Successfully generated release archives"

scripts/gen_crc32_multipliers.c

@@ -1,108 +0,0 @@
-/*
- * gen_crc32_multipliers.c
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <inttypes.h>
-#include <stdio.h>
-
-/* generator polynomial G(x) */
-#define CRCPOLY		0xEDB88320 /* G(x) without x^32 term */
-#define CRCPOLY_FULL	(((uint64_t)CRCPOLY << 1) | 1) /* G(x) */
-
-/* Compute x^D mod G(x) */
-static uint32_t
-compute_multiplier(int D)
-{
-	/* Start with x^0 mod G(x) */
-	uint32_t remainder = 0x80000000;
-
-	/* Each iteration, 'remainder' becomes x^i mod G(x) */
-	for (int i = 1; i <= D; i++)
-		remainder = (remainder >> 1) ^ ((remainder & 1) ? CRCPOLY : 0);
-
-	/* Now 'remainder' is x^D mod G(x) */
-	return remainder;
-}
-
-/* Compute floor(x^64 / G(x)) */
-static uint64_t
-compute_barrett_reduction_constant(void)
-{
-	uint64_t quotient = 0;
-	uint64_t dividend = 0x1;
-
-	for (int i = 0; i < 64 - 32 + 1; i++) {
-		if ((dividend >> i) & 1) {
-			quotient |= (uint64_t)1 << i;
-			dividend ^= CRCPOLY_FULL << i;
-		}
-	}
-
-	return quotient;
-}
-
-/*
- * This program computes the constant multipliers needed for carryless
- * multiplication accelerated CRC-32.  It assumes 128-bit vectors divided into
- * two 64-bit halves which are multiplied separately with different 32-bit
- * multipliers, producing two 95-bit products.  For a given number of 128-bit
- * vectors per iteration, the program outputs a pair of multipliers, one for
- * each 64-bit half.
- *
- * Careful: all polynomials are "bit-reversed", meaning that the low-order bits
- * have the highest degree and the high-order bits have the lowest degree!
- */
-int
-main(void)
-{
-	printf("\t/* Constants precomputed by gen_crc32_multipliers.c.  "
-	       "Do not edit! */\n");
-
-	/* High and low multipliers for each needed vector count */
-	for (int order = 2; order >= 0; order--) {
-		int vecs_per_iteration = 1 << order;
-		int right = (128 * vecs_per_iteration) + 95;
-		printf("\tconst __v2di multipliers_%d = (__v2di)"
-		       "{ 0x%08"PRIX32", 0x%08"PRIX32" };\n",
-		       vecs_per_iteration,
-		       compute_multiplier(right - 64) /* higher degree half */,
-		       compute_multiplier(right - 128) /* lower degree half */);
-	}
-
-	/* Multiplier for final 96 => 64 bit fold */
-	printf("\tconst __v2di final_multiplier = (__v2di){ 0x%08"PRIX32" };\n",
-	       compute_multiplier(63));
-
-	/* 32-bit mask */
-	printf("\tconst __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };\n");
-
-	/* Constants for final 64 => 32 bit reduction */
-	printf("\tconst __v2di barrett_reduction_constants =\n"
-	       "\t\t\t(__v2di){ 0x%016"PRIX64", 0x%016"PRIX64" };\n",
-	       compute_barrett_reduction_constant(), CRCPOLY_FULL);
-
-	return 0;
-}

scripts/gen_crc32_table.c

@@ -1,98 +0,0 @@
-/*
- * gen_crc32_table.c - a program for CRC-32 table generation
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <stdint.h>
-#include <stdio.h>
-
-static uint32_t crc32_table[0x800];
-
-static uint32_t
-crc32_update_bit(uint32_t remainder, uint8_t next_bit)
-{
-	return (remainder >> 1) ^
-		(((remainder ^ next_bit) & 1) ? 0xEDB88320 : 0);
-}
-
-static uint32_t
-crc32_update_byte(uint32_t remainder, uint8_t next_byte)
-{
-	for (int j = 0; j < 8; j++, next_byte >>= 1)
-		remainder = crc32_update_bit(remainder, next_byte & 1);
-	return remainder;
-}
-
-static void
-print_256_entries(const uint32_t *entries)
-{
-	for (size_t i = 0; i < 256 / 4; i++) {
-		printf("\t");
-		for (size_t j = 0; j < 4; j++) {
-			printf("0x%08x,", entries[i * 4 + j]);
-			if (j != 3)
-				printf(" ");
-		}
-		printf("\n");
-	}
-}
-
-int
-main(void)
-{
-	/* crc32_table[i] for 0 <= i < 0x100 is the CRC-32 of byte i.  */
-	for (int i = 0; i < 0x100; i++)
-		crc32_table[i] = crc32_update_byte(0, i);
-
-	/* crc32_table[i] for 0x100 <= i < 0x800 is the CRC-32 of byte i % 0x100
-	 * followed by i / 0x100 zero bytes.  */
-	for (int i = 0x100; i < 0x800; i++)
-		crc32_table[i] = crc32_update_byte(crc32_table[i - 0x100], 0);
-
-	printf("/*\n");
-	printf(" * crc32_table.h - data table to accelerate CRC-32 computation\n");
-	printf(" *\n");
-	printf(" * THIS FILE WAS AUTOMATICALLY GENERATED "
-	       "BY gen_crc32_table.c.  DO NOT EDIT.\n");
-	printf(" */\n");
-	printf("\n");
-	printf("#include <stdint.h>\n");
-	printf("\n");
-	printf("static const uint32_t crc32_table[] = {\n");
-	print_256_entries(&crc32_table[0x000]);
-	printf("#if defined(CRC32_SLICE4) || defined(CRC32_SLICE8)\n");
-	print_256_entries(&crc32_table[0x100]);
-	print_256_entries(&crc32_table[0x200]);
-	print_256_entries(&crc32_table[0x300]);
-	printf("#endif /* CRC32_SLICE4 || CRC32_SLICE8 */\n");
-	printf("#if defined(CRC32_SLICE8)\n");
-	print_256_entries(&crc32_table[0x400]);
-	print_256_entries(&crc32_table[0x500]);
-	print_256_entries(&crc32_table[0x600]);
-	print_256_entries(&crc32_table[0x700]);
-	printf("#endif /* CRC32_SLICE8 */\n");
-	printf("};\n");
-	return 0;
-}

scripts/gen_offset_slot_map.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
-# This script generates the deflate_offset_slot[] array, which is a condensed
-# map from offsets to offset slots.
+# This script generates the deflate_offset_slot[] array, which maps
+# 'offset - 1 => offset_slot' for offset <= 256.
 
 DEFLATE_OFFSET_SLOT_BASE = [
 	1    , 2    , 3    , 4     , 5     , 7     , 9     , 13    ,
@@ -10,26 +10,14 @@ DEFLATE_OFFSET_SLOT_BASE = [
 	4097 , 6145 , 8193 , 12289 , 16385 , 24577 ,
 ]
 
-DEFLATE_EXTRA_OFFSET_BITS = [
-	0    , 0    , 0    , 0     , 1     , 1     , 2     , 2     ,
-	3    , 3    , 4    , 4     , 5     , 5     , 6     , 6     ,
-	7    , 7    , 8    , 8     , 9     , 9     , 10    , 10    ,
-	11   , 11   , 12   , 12    , 13    , 13    ,
-]
+offset_slot_map = [0] * 256
+offset_slot = -1
+for offset in range(1, len(offset_slot_map) + 1):
+    if offset >= DEFLATE_OFFSET_SLOT_BASE[offset_slot + 1]:
+        offset_slot += 1
+    offset_slot_map[offset - 1] = offset_slot
 
-offset_slot_map = [0] * 512
-
-for offset_slot, offset_base in enumerate(DEFLATE_OFFSET_SLOT_BASE):
-    num_extra_bits = DEFLATE_EXTRA_OFFSET_BITS[offset_slot]
-    offset_end = offset_base + (1 << num_extra_bits)
-    if offset_base <= 256:
-        for offset in range(offset_base, offset_end):
-            offset_slot_map[offset] = offset_slot
-    else:
-        for offset in range(offset_base, offset_end, 128):
-            offset_slot_map[256 + ((offset - 1) >> 7)] = offset_slot
-
-print('static const u8 deflate_offset_slot_map[512] = {')
+print(f'static const u8 deflate_offset_slot[{len(offset_slot_map)}] = {{')
 for i in range(0, len(offset_slot_map), 16):
     print('\t', end='')
     for j, v in enumerate(offset_slot_map[i:i+16]):

scripts/gzip_tests.sh

@@ -15,9 +15,15 @@ set -eu -o pipefail
 
 export -n GZIP GUNZIP TESTDATA
 
+ORIG_PWD=$PWD
 TMPDIR="$(mktemp -d)"
 CURRENT_TEST=
 
+BSD_STAT=false
+if ! stat --version 2>&1 | grep -q coreutils; then
+	BSD_STAT=true
+fi
+
 cleanup() {
 	if [ -n "$CURRENT_TEST" ]; then
 		echo "TEST FAILED: \"$CURRENT_TEST\""
@@ -27,14 +33,13 @@ cleanup() {
 
 trap cleanup EXIT
 
-TESTDATA="$(readlink -f "$TESTDATA")"
-cd "$TMPDIR"
-
 begin_test() {
 	CURRENT_TEST="$1"
 	rm -rf -- "${TMPDIR:?}"/*
-	cp "$TESTDATA" file
-	chmod +w file
+	cd "$ORIG_PWD"
+	cp "$TESTDATA" "$TMPDIR/file"
+	chmod +w "$TMPDIR/file"
+	cd "$TMPDIR"
 }
 
 gzip() {
@@ -45,6 +50,40 @@ gunzip() {
 	$GUNZIP "$@"
 }
 
+get_filesize() {
+	local file=$1
+
+	if $BSD_STAT; then
+		stat -f %z "$file"
+	else
+		stat -c %s "$file"
+	fi
+}
+
+get_linkcount() {
+	local file=$1
+
+	if $BSD_STAT; then
+		stat -f %l "$file"
+	else
+		stat -c %h "$file"
+	fi
+}
+
+get_modeandtimestamps() {
+	local file=$1
+
+	if $BSD_STAT; then
+		stat -f "%p;%a;%m" "$file"
+	elif [ "$(uname -m)" = s390x ]; then
+		# Use seconds precision instead of nanoseconds.
+		# TODO: why is this needed?  QEMU user mode emulation bug?
+		stat -c "%a;%X;%Y" "$file"
+	else
+		stat -c "%a;%x;%y" "$file"
+	fi
+}
+
 assert_status() {
 	local expected_status="$1"
 	local expected_msg="$2"
@@ -94,26 +133,6 @@ assert_equals() {
 	fi
 }
 
-# Get the filesystem type.
-FSTYPE=$(df -T . | tail -1 | awk '{print $2}')
-
-# If gzip or gunzip is the GNU version, require that it supports the '-k'
-# option.  This option was added in v1.6, released in 2013.
-check_version_prereq() {
-	local prog=$1
-
-	if ! echo | { $prog -k || true; } |& grep -q 'invalid option'; then
-		return 0
-	fi
-	if ! $prog -V |& grep -q 'Free Software Foundation'; then
-		echo 1>&2 "Unexpected case: not GNU $prog, but -k option is invalid"
-		exit 1
-	fi
-	echo "GNU $prog is too old; skipping gzip/gunzip tests"
-	exit 0
-}
-check_version_prereq gzip
-check_version_prereq gunzip
 
 begin_test 'Basic compression and decompression works'
 cp file orig
@@ -179,7 +198,7 @@ gunzip -kfd file.gz
 
 
 begin_test 'Compression levels'
-if [ "$GZIP" = /bin/gzip ]; then
+if [ "$GZIP" = /bin/gzip ] || [ "$GZIP" = /usr/bin/gzip ]; then
 	assert_error '\<invalid option\>' gzip -10
 	max_level=9
 else
@@ -333,13 +352,13 @@ cmp <(echo b) b
 begin_test '(gzip) hard linked file skipped without -f or -c'
 cp file orig
 ln file link
-assert_equals 2 "$(stat -c %h file)"
+assert_equals 2 "$(get_linkcount file)"
 assert_skipped gzip file
 gzip -c file > /dev/null
-assert_equals 2 "$(stat -c %h file)"
+assert_equals 2 "$(get_linkcount file)"
 gzip -f file
-assert_equals 1 "$(stat -c %h link)"
-assert_equals 1 "$(stat -c %h file.gz)"
+assert_equals 1 "$(get_linkcount link)"
+assert_equals 1 "$(get_linkcount file.gz)"
 cmp link orig
 # XXX: GNU gzip skips hard linked files with -k, libdeflate's doesn't
 
@@ -348,13 +367,13 @@ begin_test '(gunzip) hard linked file skipped without -f or -c'
 gzip file
 ln file.gz link.gz
 cp file.gz orig.gz
-assert_equals 2 "$(stat -c %h file.gz)"
+assert_equals 2 "$(get_linkcount file.gz)"
 assert_skipped gunzip file.gz
 gunzip -c file.gz > /dev/null
-assert_equals 2 "$(stat -c %h file.gz)"
+assert_equals 2 "$(get_linkcount file.gz)"
 gunzip -f file
-assert_equals 1 "$(stat -c %h link.gz)"
-assert_equals 1 "$(stat -c %h file)"
+assert_equals 1 "$(get_linkcount link.gz)"
+assert_equals 1 "$(get_linkcount file)"
 cmp link.gz orig.gz
 
 
@@ -433,20 +452,12 @@ assert_error '\<invalid suffix\>' gunzip -S '""' file
 
 
 begin_test 'Timestamps and mode are preserved'
-if [ "$FSTYPE" = shiftfs ]; then
-	# In Travis CI, the filesystem (shiftfs) only supports seconds precision
-	# timestamps.  Nanosecond precision still sometimes seems to work,
-	# probably due to caching, but it is unreliable.
-	format='%a;%X;%Y'
-else
-	format='%a;%x;%y'
-fi
 chmod 777 file
-orig_stat="$(stat -c "$format" file)"
+orig_stat=$(get_modeandtimestamps file)
 gzip file
 sleep 1
 gunzip file.gz
-assert_equals "$orig_stat" "$(stat -c "$format" file)"
+assert_equals "$orig_stat" "$(get_modeandtimestamps file)"
 
 
 begin_test 'Decompressing multi-member gzip file'

scripts/libFuzzer/deflate_compress/fuzz.c

@@ -0,0 +1,95 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+static void
+alloc_guarded_buffer(size_t size, uint8_t **start_ret, uint8_t **end_ret)
+{
+	const size_t pagesize = sysconf(_SC_PAGESIZE);
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	uint8_t *base_addr, *start, *end;
+
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	assert(base_addr != (uint8_t *)MAP_FAILED);
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+
+	*start_ret = start;
+	*end_ret = end;
+}
+
+static void
+free_guarded_buffer(uint8_t *start, uint8_t *end)
+{
+	munmap(start, end - start);
+}
+
+/* Fuzz the DEFLATE compression and decompression round trip. */
+int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
+{
+	int level;
+	bool use_bound;
+	struct libdeflate_compressor *c;
+	struct libdeflate_decompressor *d;
+	size_t csize_avail;
+	uint8_t *ubuf_start, *ubuf_end, *ubuf;
+	uint8_t *cbuf_start, *cbuf_end, *cbuf;
+	uint8_t *dbuf_start, *dbuf_end, *dbuf;
+	size_t csize;
+	enum libdeflate_result res;
+
+	if (insize < 2)
+		return 0;
+
+	level = in[0] % 13;
+	use_bound = in[1] % 2;
+	in += 2;
+	insize -= 2;
+
+	c = libdeflate_alloc_compressor(level);
+	d = libdeflate_alloc_decompressor();
+
+	/* Use guard pages to make all input/output buffer overflows segfault */
+
+	alloc_guarded_buffer(insize, &ubuf_start, &ubuf_end);
+	ubuf = ubuf_end - insize;
+	memcpy(ubuf, in, insize);
+
+	csize_avail = use_bound ? libdeflate_deflate_compress_bound(c, insize) :
+				  insize;
+	alloc_guarded_buffer(csize_avail, &cbuf_start, &cbuf_end);
+	cbuf = cbuf_end - csize_avail;
+
+	alloc_guarded_buffer(insize, &dbuf_start, &dbuf_end);
+	dbuf = dbuf_end - insize;
+
+	csize = libdeflate_deflate_compress(c, ubuf, insize, cbuf, csize_avail);
+	if (csize != 0) {
+		assert(csize <= csize_avail);
+		memmove(cbuf_end - csize, cbuf, csize);
+		res = libdeflate_deflate_decompress(d, cbuf_end - csize, csize,
+						    dbuf, insize, NULL);
+		assert(res == LIBDEFLATE_SUCCESS);
+		assert(memcmp(in, dbuf, insize) == 0);
+	} else {
+		assert(!use_bound);
+	}
+
+	libdeflate_free_compressor(c);
+	libdeflate_free_decompressor(d);
+	free_guarded_buffer(ubuf_start, ubuf_end);
+	free_guarded_buffer(cbuf_start, cbuf_end);
+	free_guarded_buffer(dbuf_start, dbuf_end);
+	return 0;
+}

scripts/libFuzzer/deflate_decompress/fuzz.c

@@ -0,0 +1,62 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+static void
+alloc_guarded_buffer(size_t size, uint8_t **start_ret, uint8_t **end_ret)
+{
+	const size_t pagesize = sysconf(_SC_PAGESIZE);
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	uint8_t *base_addr, *start, *end;
+
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	assert(base_addr != (uint8_t *)MAP_FAILED);
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+
+	*start_ret = start;
+	*end_ret = end;
+}
+
+static void
+free_guarded_buffer(uint8_t *start, uint8_t *end)
+{
+	munmap(start, end - start);
+}
+
+/* Fuzz DEFLATE decompression. */
+int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
+{
+	size_t outsize_avail = 3 * insize;
+	uint8_t *cbuf_start, *cbuf_end, *cbuf;
+	uint8_t *dbuf_start, *dbuf_end, *dbuf;
+	struct libdeflate_decompressor *d;
+
+	/* Use guard pages to make all input/output buffer overflows segfault */
+
+	alloc_guarded_buffer(insize, &cbuf_start, &cbuf_end);
+	cbuf = cbuf_end - insize;
+	memcpy(cbuf, in, insize);
+
+	alloc_guarded_buffer(outsize_avail, &dbuf_start, &dbuf_end);
+	dbuf = dbuf_end - outsize_avail;
+
+	d = libdeflate_alloc_decompressor();
+	libdeflate_deflate_decompress(d, cbuf, insize, dbuf, outsize_avail,
+				      NULL);
+	libdeflate_free_decompressor(d);
+	free_guarded_buffer(cbuf_start, cbuf_end);
+	free_guarded_buffer(dbuf_start, dbuf_end);
+	return 0;
+}

scripts/libFuzzer/fuzz.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -e -u -o pipefail
+
+cd "$(dirname "$0")"
+
+read -r -a AVAILABLE_TARGETS < <(echo */fuzz.c | sed 's@/fuzz.c@@g')
+
+usage()
+{
+	cat << EOF
+Usage: $0 [OPTION]... FUZZ_TARGET
+
+Fuzz libdeflate with LLVM's libFuzzer.
+
+Options:
+   --asan          Enable AddressSanitizer
+   --max-len=LEN   Maximum length of generated inputs (default: $MAX_LEN)
+   --msan          Enable MemorySanitizer
+   --time=SECONDS  Stop after the given time has passed
+   --ubsan         Enable UndefinedBehaviorSanitizer
+
+Available fuzz targets: ${AVAILABLE_TARGETS[*]}
+EOF
+}
+
+die()
+{
+	echo "$*" 1>&2
+	exit 1
+}
+
+run_cmd()
+{
+	echo "$*"
+	"$@"
+}
+
+EXTRA_SANITIZERS=
+EXTRA_FUZZER_ARGS=()
+MAX_LEN=65536
+
+longopts_array=(
+asan
+help
+max-len:
+msan
+time:
+ubsan
+)
+longopts=$(echo "${longopts_array[@]}" | tr ' ' ',')
+
+if ! options=$(getopt -o "" -l "$longopts" -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+eval set -- "$options"
+while true; do
+	case "$1" in
+	--asan)
+		EXTRA_SANITIZERS+=",address"
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--max-len)
+		MAX_LEN=$2
+		shift
+		;;
+	--msan)
+		EXTRA_SANITIZERS+=",memory"
+		;;
+	--time)
+		EXTRA_FUZZER_ARGS+=("-max_total_time=$2")
+		shift
+		;;
+	--ubsan)
+		EXTRA_SANITIZERS+=",undefined"
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Invalid option '$1'"
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+EXTRA_FUZZER_ARGS+=("-max_len=$MAX_LEN")
+
+if (( $# != 1 )); then
+	echo 1>&2 "No fuzz target specified!"
+	usage 1>&2
+	exit 1
+fi
+TARGET=$1
+if [ ! -e "$TARGET/fuzz.c" ]; then
+	echo 1>&2 "'$TARGET' is not a valid fuzz target!"
+	usage 1>&2
+	exit 1
+fi
+run_cmd clang -g -O1 -fsanitize=fuzzer$EXTRA_SANITIZERS \
+	-Wall -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS=1 -I ../../ \
+	../../lib/*{,/*}.c "$TARGET/fuzz.c" -o "$TARGET/fuzz"
+run_cmd "$TARGET/fuzz" "${EXTRA_FUZZER_ARGS[@]}" "$TARGET/corpus"

scripts/libFuzzer/gzip_decompress/fuzz.c

@@ -0,0 +1,19 @@
+#include <libdeflate.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/* Fuzz gzip decompression. */
+int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
+{
+	size_t outsize_avail = 3 * insize;
+	uint8_t *out;
+	struct libdeflate_decompressor *d;
+
+	out = malloc(outsize_avail);
+
+	d = libdeflate_alloc_decompressor();
+	libdeflate_gzip_decompress(d, in, insize, out, outsize_avail, NULL);
+	libdeflate_free_decompressor(d);
+	free(out);
+	return 0;
+}

scripts/libFuzzer/zlib_decompress/fuzz.c

@@ -0,0 +1,19 @@
+#include <libdeflate.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/* Fuzz zlib decompression. */
+int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
+{
+	size_t outsize_avail = 3 * insize;
+	uint8_t *out;
+	struct libdeflate_decompressor *d;
+
+	out = malloc(outsize_avail);
+
+	d = libdeflate_alloc_decompressor();
+	libdeflate_zlib_decompress(d, in, insize, out, outsize_avail, NULL);
+	libdeflate_free_decompressor(d);
+	free(out);
+	return 0;
+}