Remove references to AVX10/256 and AVX10/512

Intel cancelled these.

Also add a bit more explanation about the attempts to avoid
downclocking.

.cirrus.yml

@@ -1,8 +1,7 @@
 task:
   freebsd_instance:
     matrix:
-      - image_family: freebsd-13-3
-      - image_family: freebsd-14-0
+      - image_family: freebsd-14-2
   install_script: pkg install -y cmake
   script:
     - cmake -B build -DLIBDEFLATE_BUILD_TESTS=1

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }})
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04]
+        os: [ubuntu-24.04, ubuntu-22.04]
         compiler: [gcc, clang]
     runs-on: ${{ matrix.os }}
     env:
@@ -17,7 +17,6 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y clang llvm libz-dev valgrind
-    - run: sudo sysctl kernel.randomize_va_space=0 # https://bugs.launchpad.net/ubuntu/+source/llvm-toolchain-14/+bug/2048768
     - run: scripts/run_tests.sh
     - name: Direct compilation without official build system
       run: $CC -O2 -Wall -Werror lib/*{,/*}.c programs/{gzip,prog_util,tgetopt}.c -o libdeflate-gzip
@@ -27,22 +26,22 @@ jobs:
     strategy:
       matrix:
         include:
-        - { arch: armv6, distro: bullseye, compiler: gcc }
-        - { arch: armv6, distro: bullseye, compiler: clang }
-        - { arch: armv7, distro: bullseye, compiler: gcc }
-        - { arch: armv7, distro: bullseye, compiler: clang }
-        - { arch: aarch64, distro: bullseye, compiler: gcc }
-        - { arch: aarch64, distro: bullseye, compiler: clang }
-        - { arch: s390x, distro: bullseye, compiler: gcc }
-        - { arch: s390x, distro: bullseye, compiler: clang }
-        - { arch: ppc64le, distro: bullseye, compiler: gcc }
-        - { arch: ppc64le, distro: bullseye, compiler: clang }
+        # - { arch: armv6, distro: bookworm, compiler: gcc }
+        # - { arch: armv6, distro: bookworm, compiler: clang }
+        - { arch: armv7, distro: bookworm, compiler: gcc }
+        - { arch: armv7, distro: bookworm, compiler: clang }
+        - { arch: aarch64, distro: bookworm, compiler: gcc }
+        - { arch: aarch64, distro: bookworm, compiler: clang }
+        - { arch: s390x, distro: bookworm, compiler: gcc }
+        - { arch: s390x, distro: bookworm, compiler: clang }
+        - { arch: ppc64le, distro: bookworm, compiler: gcc }
+        - { arch: ppc64le, distro: bookworm, compiler: clang }
         - { arch: riscv64, distro: ubuntu_latest, compiler: gcc }
         - { arch: riscv64, distro: ubuntu_latest, compiler: clang }
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: uraimo/run-on-arch-action@v2.5.0
+      - uses: uraimo/run-on-arch-action@v3.0.0
         with:
           arch: ${{ matrix.arch }}
           distro: ${{ matrix.distro }}
@@ -139,10 +138,10 @@ jobs:
     name: Build (Windows, Visual Studio ${{matrix.toolset}}, ${{matrix.platform}})
     strategy:
       matrix:
-        platform: [ARM64, ARM]
+        platform: [ARM64, ARM64EC]
         toolset: [v143, ClangCL]
         exclude: # Exclude unsupported combinations
-        - platform: ARM
+        - platform: ARM64EC
           toolset: ClangCL
     runs-on: windows-latest
     steps:
@@ -152,7 +151,7 @@ jobs:
     - run: >
         cmake -B build -G "Visual Studio 17 2022" -T ${{matrix.toolset}}
         -A ${{matrix.platform}} -DCMAKE_C_FLAGS="/W4 /WX"
-        -DCMAKE_INSTALL_PREFIX=build\install
+        -DCMAKE_INSTALL_PREFIX=build\install -DCMAKE_SYSTEM_VERSION=10
     - run: cmake --build build --verbose --config Release
     - run: cmake --install build --verbose --config Release
 
@@ -288,6 +287,5 @@ jobs:
         sudo apt-get install -y clang llvm
     - name: Fuzz
       run: |
-        sudo sysctl kernel.randomize_va_space=0 # https://bugs.launchpad.net/ubuntu/+source/llvm-toolchain-14/+bug/2048768
         scripts/libFuzzer/fuzz.sh --time=120 ${{matrix.sanitizer}} \
             ${{matrix.target}}

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.7)
+cmake_minimum_required(VERSION 3.10)
 
 # Default to a release build.
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
@@ -44,6 +44,9 @@ option(LIBDEFLATE_BUILD_TESTS "Build the test programs" OFF)
 option(LIBDEFLATE_USE_SHARED_LIB
        "Link the libdeflate-gzip and test programs to the shared library instead
        of the static library" OFF)
+if(APPLE)
+    option(LIBDEFLATE_APPLE_FRAMEWORK "Build as Apple Framework" OFF)
+endif()
 
 if(LIBDEFLATE_BUILD_TESTS)
     enable_testing()
@@ -90,6 +93,55 @@ if(LIBDEFLATE_FREESTANDING)
     add_definitions(-DFREESTANDING)
 endif()
 
+# Check for cases where the compiler supports an instruction set extension but
+# the assembler does not, and in those cases print a warning and add an
+# appropriate -DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_* flag.  libdeflate's C
+# source files already check the compiler version before using the corresponding
+# intrinsics, but in the rare case of gcc being paired with a binutils much
+# older than itself those checks are insufficient.  There is no way to check the
+# assembler version from C.  The proper fix for too-old binutils is for the user
+# to upgrade binutils.  Unfortunately, as libdeflate has started using newer
+# instructions, binutils incompatibilities have started being seen more
+# frequently.  Hence these checks for assembler support here in CMakeLists.txt
+# to provide a fallback for users who may be unable to fix their toolchain.
+# These don't solve the problem for users not using CMake, though such users can
+# add specific -DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_* flags they need.
+function(check_assembler_support  feature assembly_code)
+    execute_process(COMMAND echo "${assembly_code}"
+                    COMMAND ${CMAKE_C_COMPILER} -c -x assembler -o /dev/null -
+                    RESULT_VARIABLE result
+                    ERROR_QUIET)
+    if(NOT ${result} EQUAL 0)
+        add_definitions(-DLIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_${feature})
+        message(STATUS "Your gcc supports ${feature} instructions but it is paired with an assembler that does not.  Upgrading binutils is recommended.")
+    endif()
+endfunction()
+if(UNIX AND CMAKE_C_COMPILER_ID STREQUAL "GNU")
+    execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpmachine
+                    OUTPUT_VARIABLE machine)
+    if(${machine} MATCHES "^(x86_64|i[3-6]86)")
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI if needed.
+            check_assembler_support(AVX512VNNI "vpdpbusd %zmm0, %zmm0, %zmm0")
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ if needed.
+            check_assembler_support(VPCLMULQDQ "vpclmulqdq $0, %zmm0, %zmm0, %zmm0")
+        endif()
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI if needed.
+            check_assembler_support(AVX_VNNI "{vex} vpdpbusd %ymm0, %ymm0, %ymm0")
+        endif()
+    elseif(${machine} MATCHES "^aarch64")
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD if needed.
+            check_assembler_support(DOTPROD ".arch armv8.2-a+dotprod\nudot v0.4s, v0.16b, v0.16b")
+        endif()
+        if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 9.1)
+            # Set LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3 if needed.
+            check_assembler_support(SHA3 ".arch armv8.2-a+sha3\neor3 v0.16b, v0.16b, v0.16b, v0.16b")
+        endif()
+    endif()
+endif()
+
 # Determine the list of source files and the list of compiler options that will
 # be used for both the static library and the shared library.
 
@@ -185,6 +237,21 @@ if(LIBDEFLATE_BUILD_STATIC_LIB)
     set_target_properties(libdeflate_static PROPERTIES
                           OUTPUT_NAME ${STATIC_LIB_NAME}
                           PUBLIC_HEADER libdeflate.h)
+    if(LIBDEFLATE_APPLE_FRAMEWORK)
+        set_target_properties(libdeflate_static PROPERTIES
+            FRAMEWORK TRUE
+            FRAMEWORK_VERSION "${VERSION_STRING}"
+            PRODUCT_BUNDLE_IDENTIFIER "github.com/ebiggers/libdeflate"
+            XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
+            XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
+            XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+            XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
+            MACOSX_FRAMEWORK_IDENTIFIER "github.com/ebiggers/libdeflate"
+            MACOSX_FRAMEWORK_BUNDLE_VERSION "${VERSION_STRING}"
+            MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${PROJECT_VERSION}"
+            MACOSX_RPATH TRUE)
+    endif()
+
     target_include_directories(libdeflate_static PUBLIC ${LIB_INCLUDE_DIRS})
     target_compile_definitions(libdeflate_static PRIVATE ${LIB_COMPILE_DEFINITIONS})
     target_compile_options(libdeflate_static PRIVATE ${LIB_COMPILE_OPTIONS})
@@ -218,6 +285,8 @@ install(TARGETS ${LIB_TARGETS}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        FRAMEWORK DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT runtime OPTIONAL
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
         PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 # Generate and install the pkg-config file.  (Don't confuse this with the CMake

COPYING

@@ -1,4 +1,5 @@
 Copyright 2016 Eric Biggers
+Copyright 2024 Google LLC
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation files

NEWS.md

@@ -1,5 +1,39 @@
 # libdeflate release notes
 
+## Version 1.23
+
+* Fixed bug introduced in 1.20 where incorrect checksums could be calculated if
+  libdeflate was compiled with clang at -O0 and run on a CPU supporting AVX512.
+
+* Fixed bug introduced in 1.20 where incorrect checksums could be calculated in
+  rare cases on macOS computers that support AVX512 and are running an older
+  version of macOS that contains a bug that corrupts AVX512 registers.  This
+  could occur only if code outside libdeflate enabled AVX512 in the thread.
+
+* Fixed build error when using -mno-evex512 with clang 18+ or gcc 14+.
+
+* Increased the minimum CMake version to 3.10.
+
+* Further optimized the x86 CRC code.
+
+## Version 1.22
+
+* The CMake-based build system now implements a workaround for gcc being paired
+  with a too-old binutils version.  This can prevent build errors.
+
+## Version 1.21
+
+* Fixed build error on x86 with gcc 8.1 and gcc 8.2.
+
+* Fixed build error on x86 when gcc 11 is paired with a binutils version that
+  doesn't support AVX-VNNI, e.g. as it is on RHEL 9.
+
+* Fixed build error on arm64 with gcc 6.
+
+* Fixed build error on arm64 with gcc 13.1 and later with some -mcpu options.
+
+* Enabled detection of dotprod support in Windows ARM64 builds.
+
 ## Version 1.20
 
 * Improved CRC-32 performance on recent x86 CPUs by adding

README.md

@@ -10,7 +10,7 @@ The supported formats are:
 - gzip (a.k.a. DEFLATE with a gzip wrapper)
 
 libdeflate is heavily optimized.  It is significantly faster than the zlib
-library, both for compression and decompression, and especially on x86
+library, both for compression and decompression, and especially on x86 and ARM
 processors.  In addition, libdeflate provides optional high compression modes
 that provide a better compression ratio than the zlib's "level 9".
 
@@ -37,6 +37,7 @@ For the release notes, see the [NEWS file](NEWS.md).
 - [Building](#building)
   - [Using CMake](#using-cmake)
   - [Directly integrating the library sources](#directly-integrating-the-library-sources)
+  - [Supported compilers](#supported-compilers)
 - [API](#api)
 - [Bindings for other programming languages](#bindings-for-other-programming-languages)
 - [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip)
@@ -98,6 +99,19 @@ for release builds.  `-O3` is fine too, but often `-O2` actually gives better
 results.  It's unnecessary to add flags such as `-mavx2` or `/arch:AVX2`, though
 you can do so if you want to.  Most of the relevant optimized functions are
 built regardless of such flags, and appropriate ones are selected at runtime.
+For the same reason, flags like `-mno-avx2` do *not* cause all code using the
+corresponding instruction set extension to be omitted from the binary; this is
+working as intended due to the use of runtime CPU feature detection.
+
+If using gcc, your gcc should always be paired with a binutils version that is
+not much older than itself, to avoid problems where the compiler generates
+instructions the assembler cannot assemble.  Usually systems have their gcc and
+binutils paired properly, but rarely a mismatch can arise in cases such as the
+user installing a newer gcc version without a proper binutils alongside it.
+Since libdeflate v1.22, the CMake-based build system will detect incompatible
+binutils versions and disable some optimized code accordingly.  In older
+versions of libdeflate, or if CMake is not being used, a too-old binutils can
+cause build errors like "no such instruction" from the assembler.
 
 # API
 
@@ -130,6 +144,7 @@ libdeflate from a programming language other than C or C++, consider using the
 following bindings:
 
 * C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET)
+* Delphi: [libdeflate-pas](https://github.com/zedxxx/libdeflate-pas)
 * Go: [go-libdeflate](https://github.com/4kills/go-libdeflate)
 * Java: [libdeflate-java](https://github.com/astei/libdeflate-java)
 * Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl)

common_defs.h

@@ -67,7 +67,8 @@
 #undef ARCH_ARM32
 #undef ARCH_RISCV
 #ifdef _MSC_VER
-#  if defined(_M_X64)
+   /* Way too many things are broken in ARM64EC to pretend that it is x86_64. */
+#  if defined(_M_X64) && !defined(_M_ARM64EC)
 #    define ARCH_X86_64
 #  elif defined(_M_IX86)
 #    define ARCH_X86_32

lib/arm/adler32_impl.h

@@ -209,16 +209,25 @@ adler32_arm_neon(u32 adler, const u8 *p, size_t len)
 #endif /* Regular NEON implementation */
 
 /* NEON+dotprod implementation */
-#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN()
+#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN() && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD)
 #  define adler32_arm_neon_dotprod	adler32_arm_neon_dotprod
 #  ifdef __clang__
 #    define ATTRIBUTES	_target_attribute("dotprod")
    /*
-    * With gcc, arch=armv8.2-a is needed for dotprod intrinsics, unless the
-    * default target is armv8.3-a or later in which case it must be omitted.
-    * armv8.3-a or later can be detected by checking for __ARM_FEATURE_JCVT.
+    * Both gcc and binutils originally considered dotprod to depend on
+    * arch=armv8.2-a or later.  This was fixed in gcc 13.2 by commit
+    * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+    * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+    * dependencies from features").  Unfortunately, always using arch=armv8.2-a
+    * causes build errors with some compiler options because it may reduce the
+    * arch rather than increase it.  Therefore we try to omit the arch whenever
+    * possible.  If gcc is 14 or later, then both gcc and binutils are probably
+    * fixed, so we omit the arch.  We also omit the arch if a feature that
+    * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
     */
-#  elif defined(__ARM_FEATURE_JCVT)
+#  elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+			  || defined(__ARM_FEATURE_DOTPROD)
 #    define ATTRIBUTES	_target_attribute("+dotprod")
 #  else
 #    define ATTRIBUTES	_target_attribute("arch=armv8.2-a+dotprod")

lib/arm/cpu_features.c

@@ -170,6 +170,10 @@ static u32 query_arm_cpu_features(void)
 
 #include <windows.h>
 
+#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE /* added in Windows SDK 20348 */
+#  define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
+#endif
+
 static u32 query_arm_cpu_features(void)
 {
 	u32 features = ARM_CPU_FEATURE_NEON;
@@ -178,8 +182,10 @@ static u32 query_arm_cpu_features(void)
 		features |= ARM_CPU_FEATURE_PMULL;
 	if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
 		features |= ARM_CPU_FEATURE_CRC32;
+	if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_DOTPROD;
 
-	/* FIXME: detect SHA3 and DOTPROD support too. */
+	/* FIXME: detect SHA3 support too. */
 
 	return features;
 }

lib/arm/cpu_features.h

@@ -93,7 +93,7 @@ static inline u32 get_arm_cpu_features(void) { return 0; }
 #  define HAVE_PMULL(features)	((features) & ARM_CPU_FEATURE_PMULL)
 #endif
 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
-	(GCC_PREREQ(6, 1) || defined(__clang__) || defined(_MSC_VER)) && \
+	(GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
 	CPU_IS_LITTLE_ENDIAN() /* untested on big endian */
 #  define HAVE_PMULL_INTRIN	1
    /* Work around MSVC's vmull_p64() taking poly64x1_t instead of poly64_t */

lib/arm/crc32_impl.h

@@ -434,13 +434,11 @@ crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
 		{ CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */
 		{ CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */
 	};
-	static const u64 _aligned_attribute(16) final_mults[3][2] = {
-		{ CRC32_X63_MODG, 0 },
-		{ CRC32_BARRETT_CONSTANT_1, 0 },
-		{ CRC32_BARRETT_CONSTANT_2, 0 },
+	static const u64 _aligned_attribute(16) barrett_consts[3][2] = {
+		{ CRC32_X95_MODG, },
+		{ CRC32_BARRETT_CONSTANT_1, },
+		{ CRC32_BARRETT_CONSTANT_2, },
 	};
-	const uint8x16_t zeroes = vdupq_n_u8(0);
-	const uint8x16_t mask32 = vreinterpretq_u8_u64(vdupq_n_u64(0xFFFFFFFF));
 	const poly64x2_t multipliers_1 = load_multipliers(mults[0]);
 	uint8x16_t v0, v1, v2, v3;
 
@@ -497,24 +495,13 @@ crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
 	if (len)
 		v0 = fold_partial_vec(v0, p, len, multipliers_1);
 
-	/*
-	 * Fold 128 => 96 bits.  This also implicitly appends 32 zero bits,
-	 * which is equivalent to multiplying by x^32.  This is needed because
-	 * the CRC is defined as M(x)*x^32 mod G(x), not just M(x) mod G(x).
-	 */
-
-	v0 = veorq_u8(vextq_u8(v0, zeroes, 8),
-		      clmul_high(vextq_u8(zeroes, v0, 8), multipliers_1));
-
-	/* Fold 96 => 64 bits. */
-	v0 = veorq_u8(vextq_u8(v0, zeroes, 4),
-		      clmul_low(vandq_u8(v0, mask32),
-				load_multipliers(final_mults[0])));
-
-	/* Reduce 64 => 32 bits using Barrett reduction. */
-	v1 = clmul_low(vandq_u8(v0, mask32), load_multipliers(final_mults[1]));
-	v1 = clmul_low(vandq_u8(v1, mask32), load_multipliers(final_mults[2]));
-	return vgetq_lane_u32(vreinterpretq_u32_u8(veorq_u8(v0, v1)), 1);
+	/* Reduce to 32 bits, following lib/x86/crc32_pclmul_template.h */
+	v0 = veorq_u8(clmul_low(v0, load_multipliers(barrett_consts[0])),
+		      vextq_u8(v0, vdupq_n_u8(0), 8));
+	v1 = clmul_low(v0, load_multipliers(barrett_consts[1]));
+	v1 = clmul_low(v1, load_multipliers(barrett_consts[2]));
+	v0 = veorq_u8(v0, v1);
+	return vgetq_lane_u32(vreinterpretq_u32_u8(v0), 2);
 }
 #undef SUFFIX
 #undef ATTRIBUTES
@@ -545,17 +532,26 @@ crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
  * This like crc32_arm_pmullx12_crc(), but it adds the eor3 instruction (from
  * the sha3 extension) for even better performance.
  */
-#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN
+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3)
 #  define crc32_arm_pmullx12_crc_eor3	crc32_arm_pmullx12_crc_eor3
 #  define SUFFIX				 _pmullx12_crc_eor3
 #  ifdef __clang__
 #    define ATTRIBUTES	_target_attribute("aes,crc,sha3")
    /*
-    * With gcc, arch=armv8.2-a is needed for the sha3 intrinsics, unless the
-    * default target is armv8.3-a or later in which case it must be omitted.
-    * armv8.3-a or later can be detected by checking for __ARM_FEATURE_JCVT.
+    * Both gcc and binutils originally considered sha3 to depend on
+    * arch=armv8.2-a or later.  This was fixed in gcc 13.2 by commit
+    * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+    * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+    * dependencies from features").  Unfortunately, always using arch=armv8.2-a
+    * causes build errors with some compiler options because it may reduce the
+    * arch rather than increase it.  Therefore we try to omit the arch whenever
+    * possible.  If gcc is 14 or later, then both gcc and binutils are probably
+    * fixed, so we omit the arch.  We also omit the arch if a feature that
+    * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
     */
-#  elif defined(__ARM_FEATURE_JCVT)
+#  elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+			  || defined(__ARM_FEATURE_DOTPROD)
 #    define ATTRIBUTES	_target_attribute("+crypto,+crc,+sha3")
 #  else
 #    define ATTRIBUTES	_target_attribute("arch=armv8.2-a+crypto+crc+sha3")

lib/crc32_multipliers.h

@@ -1,7 +1,7 @@
 /*
  * crc32_multipliers.h - constants for CRC-32 folding
  *
- * THIS FILE WAS GENERATED BY gen_crc32_multipliers.c.  DO NOT EDIT.
+ * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.
  */
 
 #define CRC32_X159_MODG 0xae689191 /* x^159 mod G(x) */
@@ -100,10 +100,8 @@
 #define CRC32_X4127_MODG 0x1072db28 /* x^4127 mod G(x) */
 #define CRC32_X4063_MODG 0x0c30f51d /* x^4063 mod G(x) */
 
-#define CRC32_X63_MODG 0xb8bc6765 /* x^63 mod G(x) */
-#define CRC32_BARRETT_CONSTANT_1 0x00000001f7011641ULL /* floor(x^64 / G(x)) */
+#define CRC32_BARRETT_CONSTANT_1 0xb4e5b025f7011641ULL /* floor(x^95 / G(x)) */
 #define CRC32_BARRETT_CONSTANT_2 0x00000001db710641ULL /* G(x) */
-#define CRC32_BARRETT_CONSTANTS { CRC32_BARRETT_CONSTANT_1, CRC32_BARRETT_CONSTANT_2 }
 
 #define CRC32_NUM_CHUNKS 4
 #define CRC32_MIN_VARIABLE_CHUNK_LEN 128UL

lib/crc32_tables.h

@@ -1,7 +1,7 @@
 /*
  * crc32_tables.h - data tables for CRC-32 computation
  *
- * THIS FILE WAS GENERATED BY gen_crc32_tables.c.  DO NOT EDIT.
+ * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.
  */
 
 static const u32 crc32_slice1_table[] MAYBE_UNUSED = {

lib/deflate_compress.c

@@ -234,7 +234,7 @@ check_buildtime_parameters(void)
 /******************************************************************************/
 
 /* Table: length slot => length slot base value */
-static const unsigned deflate_length_slot_base[] = {
+static const u32 deflate_length_slot_base[] = {
 	3,    4,    5,    6,    7,    8,    9,    10,
 	11,   13,   15,   17,   19,   23,   27,   31,
 	35,   43,   51,   59,   67,   83,   99,   115,
@@ -250,7 +250,7 @@ static const u8 deflate_extra_length_bits[] = {
 };
 
 /* Table: offset slot => offset slot base value */
-static const unsigned deflate_offset_slot_base[] = {
+static const u32 deflate_offset_slot_base[] = {
 	1,     2,     3,     4,     5,     7,     9,     13,
 	17,    25,    33,    49,    65,    97,    129,   193,
 	257,   385,   513,   769,   1025,  1537,  2049,  3073,
@@ -470,13 +470,13 @@ struct libdeflate_compressor {
 	 * The maximum search depth: consider at most this many potential
 	 * matches at each position
 	 */
-	unsigned max_search_depth;
+	u32 max_search_depth;
 
 	/*
 	 * The "nice" match length: if a match of this length is found, choose
 	 * it immediately without further consideration
 	 */
-	unsigned nice_match_length;
+	u32 nice_match_length;
 
 	/* Frequency counters for the current block */
 	struct deflate_freqs freqs;
@@ -626,7 +626,7 @@ struct libdeflate_compressor {
 			 * early, before max_optim_passes has been reached.
 			 * Smaller values = more compression.
 			 */
-			unsigned min_improvement_to_continue;
+			u32 min_improvement_to_continue;
 
 			/*
 			 * The minimum number of bits that would need to be
@@ -636,7 +636,7 @@ struct libdeflate_compressor {
 			 * optimization pass actually increased the cost.
 			 * Smaller values = more compression.
 			 */
-			unsigned min_bits_to_use_nonfinal_path;
+			u32 min_bits_to_use_nonfinal_path;
 
 			/*
 			 * The maximum block length, in uncompressed bytes, at
@@ -653,7 +653,7 @@ struct libdeflate_compressor {
 			 * match/literal list as the optimized dynamic block
 			 * happens to be cheaper than the dynamic block itself.
 			 */
-			unsigned max_len_to_optimize_static_block;
+			u32 max_len_to_optimize_static_block;
 
 		} n; /* (n)ear-optimal */
 	#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
@@ -1639,7 +1639,7 @@ static void
 deflate_compute_full_len_codewords(struct libdeflate_compressor *c,
 				   const struct deflate_codes *codes)
 {
-	unsigned len;
+	u32 len;
 
 	STATIC_ASSERT(MAX_LITLEN_CODEWORD_LEN +
 		      DEFLATE_MAX_EXTRA_LENGTH_BITS <= 32);
@@ -1662,8 +1662,8 @@ deflate_compute_full_len_codewords(struct libdeflate_compressor *c,
 do {									\
 	const struct libdeflate_compressor *c__ = (c_);			\
 	const struct deflate_codes *codes__ = (codes_);			\
-	unsigned length__ = (length_);					\
-	unsigned offset__ = (offset_);					\
+	u32 length__ = (length_);					\
+	u32 offset__ = (offset_);					\
 	unsigned offset_slot__ = (offset_slot_);			\
 									\
 	/* Litlen symbol and extra length bits */			\
@@ -1936,9 +1936,9 @@ deflate_flush_block(struct libdeflate_compressor *c,
 		struct deflate_optimum_node * const end_node =
 			&c->p.n.optimum_nodes[block_length];
 		do {
-			unsigned length = cur_node->item & OPTIMUM_LEN_MASK;
-			unsigned offset = cur_node->item >>
-					  OPTIMUM_OFFSET_SHIFT;
+			u32 length = cur_node->item & OPTIMUM_LEN_MASK;
+			u32 offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
+
 			if (length == 1) {
 				/* Literal */
 				ADD_BITS(codes->codewords.litlen[offset],
@@ -1960,8 +1960,8 @@ deflate_flush_block(struct libdeflate_compressor *c,
 		for (seq = sequences; ; seq++) {
 			u32 litrunlen = seq->litrunlen_and_length &
 					SEQ_LITRUNLEN_MASK;
-			unsigned length = seq->litrunlen_and_length >>
-					  SEQ_LENGTH_SHIFT;
+			u32 length = seq->litrunlen_and_length >>
+				     SEQ_LENGTH_SHIFT;
 			unsigned lit;
 
 			/* Output a run of literals. */
@@ -2118,7 +2118,7 @@ observe_literal(struct block_split_stats *stats, u8 lit)
  * one observation type for "long match".
  */
 static forceinline void
-observe_match(struct block_split_stats *stats, unsigned length)
+observe_match(struct block_split_stats *stats, u32 length)
 {
 	stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES +
 				(length >= 9)]++;
@@ -2242,7 +2242,7 @@ deflate_choose_literal(struct libdeflate_compressor *c, unsigned literal,
 
 static forceinline void
 deflate_choose_match(struct libdeflate_compressor *c,
-		     unsigned length, unsigned offset, bool gather_split_stats,
+		     u32 length, u32 offset, bool gather_split_stats,
 		     struct deflate_sequence **seq_p)
 {
 	struct deflate_sequence *seq = *seq_p;
@@ -2254,7 +2254,7 @@ deflate_choose_match(struct libdeflate_compressor *c,
 	if (gather_split_stats)
 		observe_match(&c->split_stats, length);
 
-	seq->litrunlen_and_length |= (u32)length << SEQ_LENGTH_SHIFT;
+	seq->litrunlen_and_length |= length << SEQ_LENGTH_SHIFT;
 	seq->offset = offset;
 	seq->offset_slot = offset_slot;
 
@@ -2268,7 +2268,7 @@ deflate_choose_match(struct libdeflate_compressor *c,
  * the input buffer.
  */
 static forceinline void
-adjust_max_and_nice_len(unsigned *max_len, unsigned *nice_len, size_t remaining)
+adjust_max_and_nice_len(u32 *max_len, u32 *nice_len, size_t remaining)
 {
 	if (unlikely(remaining < DEFLATE_MAX_MATCH_LEN)) {
 		*max_len = remaining;
@@ -2292,8 +2292,8 @@ adjust_max_and_nice_len(unsigned *max_len, unsigned *nice_len, size_t remaining)
  * probably be worthwhile.  Conversely, if not many literals are used, then
  * probably literals will be cheap and short matches won't be worthwhile.
  */
-static unsigned
-choose_min_match_len(unsigned num_used_literals, unsigned max_search_depth)
+static u32
+choose_min_match_len(u32 num_used_literals, u32 max_search_depth)
 {
 	/* map from num_used_literals to min_len */
 	static const u8 min_lens[] = {
@@ -2304,7 +2304,7 @@ choose_min_match_len(unsigned num_used_literals, unsigned max_search_depth)
 		4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
 		/* The rest is implicitly 3. */
 	};
-	unsigned min_len;
+	u32 min_len;
 
 	STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN <= 3);
 	STATIC_ASSERT(ARRAY_LEN(min_lens) <= DEFLATE_NUM_LITERALS + 1);
@@ -2326,12 +2326,11 @@ choose_min_match_len(unsigned num_used_literals, unsigned max_search_depth)
 	return min_len;
 }
 
-static unsigned
-calculate_min_match_len(const u8 *data, size_t data_len,
-			unsigned max_search_depth)
+static u32
+calculate_min_match_len(const u8 *data, size_t data_len, u32 max_search_depth)
 {
 	u8 used[256] = { 0 };
-	unsigned num_used_literals = 0;
+	u32 num_used_literals = 0;
 	size_t i;
 
 	/*
@@ -2357,13 +2356,13 @@ calculate_min_match_len(const u8 *data, size_t data_len,
  * Recalculate the minimum match length for a block, now that we know the
  * distribution of literals that are actually being used (freqs->litlen).
  */
-static unsigned
+static u32
 recalculate_min_match_len(const struct deflate_freqs *freqs,
-			  unsigned max_search_depth)
+			  u32 max_search_depth)
 {
 	u32 literal_freq = 0;
 	u32 cutoff;
-	unsigned num_used_literals = 0;
+	u32 num_used_literals = 0;
 	int i;
 
 	for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
@@ -2457,8 +2456,8 @@ deflate_compress_fastest(struct libdeflate_compressor * restrict c,
 	const u8 *in_next = in;
 	const u8 *in_end = in_next + in_nbytes;
 	const u8 *in_cur_base = in_next;
-	unsigned max_len = DEFLATE_MAX_MATCH_LEN;
-	unsigned nice_len = MIN(c->nice_match_length, max_len);
+	u32 max_len = DEFLATE_MAX_MATCH_LEN;
+	u32 nice_len = MIN(c->nice_match_length, max_len);
 	u32 next_hash = 0;
 
 	ht_matchfinder_init(&c->p.f.ht_mf);
@@ -2534,8 +2533,8 @@ deflate_compress_greedy(struct libdeflate_compressor * restrict c,
 	const u8 *in_next = in;
 	const u8 *in_end = in_next + in_nbytes;
 	const u8 *in_cur_base = in_next;
-	unsigned max_len = DEFLATE_MAX_MATCH_LEN;
-	unsigned nice_len = MIN(c->nice_match_length, max_len);
+	u32 max_len = DEFLATE_MAX_MATCH_LEN;
+	u32 nice_len = MIN(c->nice_match_length, max_len);
 	u32 next_hashes[2] = {0, 0};
 
 	hc_matchfinder_init(&c->p.g.hc_mf);
@@ -2547,7 +2546,7 @@ deflate_compress_greedy(struct libdeflate_compressor * restrict c,
 		const u8 * const in_max_block_end = choose_max_block_end(
 				in_next, in_end, SOFT_MAX_BLOCK_LENGTH);
 		struct deflate_sequence *seq = c->p.g.sequences;
-		unsigned min_len;
+		u32 min_len;
 
 		init_block_split_stats(&c->split_stats);
 		deflate_begin_sequences(c, seq);
@@ -2610,8 +2609,8 @@ deflate_compress_lazy_generic(struct libdeflate_compressor * restrict c,
 	const u8 *in_next = in;
 	const u8 *in_end = in_next + in_nbytes;
 	const u8 *in_cur_base = in_next;
-	unsigned max_len = DEFLATE_MAX_MATCH_LEN;
-	unsigned nice_len = MIN(c->nice_match_length, max_len);
+	u32 max_len = DEFLATE_MAX_MATCH_LEN;
+	u32 nice_len = MIN(c->nice_match_length, max_len);
 	u32 next_hashes[2] = {0, 0};
 
 	hc_matchfinder_init(&c->p.g.hc_mf);
@@ -2625,7 +2624,7 @@ deflate_compress_lazy_generic(struct libdeflate_compressor * restrict c,
 		const u8 *next_recalc_min_len =
 			in_next + MIN(in_end - in_next, 10000);
 		struct deflate_sequence *seq = c->p.g.sequences;
-		unsigned min_len;
+		u32 min_len;
 
 		init_block_split_stats(&c->split_stats);
 		deflate_begin_sequences(c, seq);
@@ -2633,10 +2632,10 @@ deflate_compress_lazy_generic(struct libdeflate_compressor * restrict c,
 						  in_max_block_end - in_next,
 						  c->max_search_depth);
 		do {
-			unsigned cur_len;
-			unsigned cur_offset;
-			unsigned next_len;
-			unsigned next_offset;
+			u32 cur_len;
+			u32 cur_offset;
+			u32 next_len;
+			u32 next_offset;
 
 			/*
 			 * Recalculate the minimum match length if it hasn't
@@ -2849,8 +2848,8 @@ deflate_tally_item_list(struct libdeflate_compressor *c, u32 block_length)
 		&c->p.n.optimum_nodes[block_length];
 
 	do {
-		unsigned length = cur_node->item & OPTIMUM_LEN_MASK;
-		unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
+		u32 length = cur_node->item & OPTIMUM_LEN_MASK;
+		u32 offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
 
 		if (length == 1) {
 			/* Literal */
@@ -3111,7 +3110,7 @@ deflate_choose_default_litlen_costs(struct libdeflate_compressor *c,
 				    const u8 *block_begin, u32 block_length,
 				    u32 *lit_cost, u32 *len_sym_cost)
 {
-	unsigned num_used_literals = 0;
+	u32 num_used_literals = 0;
 	u32 literal_freq = block_length;
 	u32 match_freq = 0;
 	u32 cutoff;
@@ -3161,7 +3160,7 @@ deflate_choose_default_litlen_costs(struct libdeflate_compressor *c,
 }
 
 static forceinline u32
-deflate_default_length_cost(unsigned len, u32 len_sym_cost)
+deflate_default_length_cost(u32 len, u32 len_sym_cost)
 {
 	unsigned slot = deflate_length_slot[len];
 	u32 num_extra_bits = deflate_extra_length_bits[slot];
@@ -3188,7 +3187,7 @@ static void
 deflate_set_default_costs(struct libdeflate_compressor *c,
 			  u32 lit_cost, u32 len_sym_cost)
 {
-	unsigned i;
+	u32 i;
 
 	/* Literals */
 	for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
@@ -3224,7 +3223,7 @@ static forceinline void
 deflate_adjust_costs_impl(struct libdeflate_compressor *c,
 			  u32 lit_cost, u32 len_sym_cost, int change_amount)
 {
-	unsigned i;
+	u32 i;
 
 	/* Literals */
 	for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
@@ -3337,7 +3336,7 @@ deflate_find_min_cost_path(struct libdeflate_compressor *c,
 	cur_node->cost_to_end = 0;
 	do {
 		unsigned num_matches;
-		unsigned literal;
+		u32 literal;
 		u32 best_cost_to_end;
 
 		cur_node--;
@@ -3349,14 +3348,14 @@ deflate_find_min_cost_path(struct libdeflate_compressor *c,
 		/* It's always possible to choose a literal. */
 		best_cost_to_end = c->p.n.costs.literal[literal] +
 				   (cur_node + 1)->cost_to_end;
-		cur_node->item = ((u32)literal << OPTIMUM_OFFSET_SHIFT) | 1;
+		cur_node->item = (literal << OPTIMUM_OFFSET_SHIFT) | 1;
 
 		/* Also consider matches if there are any. */
 		if (num_matches) {
 			const struct lz_match *match;
-			unsigned len;
-			unsigned offset;
-			unsigned offset_slot;
+			u32 len;
+			u32 offset;
+			u32 offset_slot;
 			u32 offset_cost;
 			u32 cost_to_end;
 
@@ -3384,7 +3383,7 @@ deflate_find_min_cost_path(struct libdeflate_compressor *c,
 					if (cost_to_end < best_cost_to_end) {
 						best_cost_to_end = cost_to_end;
 						cur_node->item = len |
-							((u32)offset <<
+							(offset <<
 							 OPTIMUM_OFFSET_SHIFT);
 					}
 				} while (++len <= match->length);
@@ -3601,8 +3600,8 @@ deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
 	const u8 *in_cur_base = in_next;
 	const u8 *in_next_slide =
 		in_next + MIN(in_end - in_next, MATCHFINDER_WINDOW_SIZE);
-	unsigned max_len = DEFLATE_MAX_MATCH_LEN;
-	unsigned nice_len = MIN(c->nice_match_length, max_len);
+	u32 max_len = DEFLATE_MAX_MATCH_LEN;
+	u32 nice_len = MIN(c->nice_match_length, max_len);
 	struct lz_match *cache_ptr = c->p.n.match_cache;
 	u32 next_hashes[2] = {0, 0};
 	bool prev_block_used_only_literals = false;
@@ -3617,7 +3616,7 @@ deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
 		const u8 *prev_end_block_check = NULL;
 		bool change_detected = false;
 		const u8 *next_observation = in_next;
-		unsigned min_len;
+		u32 min_len;
 
 		/*
 		 * Use the minimum match length heuristic to improve the
@@ -3648,7 +3647,7 @@ deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
 		 */
 		for (;;) {
 			struct lz_match *matches;
-			unsigned best_len;
+			u32 best_len;
 			size_t remaining = in_end - in_next;
 
 			/* Slide the window forward if needed. */
@@ -3853,9 +3852,9 @@ deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
 static void
 deflate_init_offset_slot_full(struct libdeflate_compressor *c)
 {
-	unsigned offset_slot;
-	unsigned offset;
-	unsigned offset_end;
+	u32 offset_slot;
+	u32 offset;
+	u32 offset_end;
 
 	for (offset_slot = 0; offset_slot < ARRAY_LEN(deflate_offset_slot_base);
 	     offset_slot++) {

lib/matchfinder_common.h

@@ -175,11 +175,11 @@ lz_hash(u32 seq, unsigned num_bits)
  * Return the number of bytes at @matchptr that match the bytes at @strptr, up
  * to a maximum of @max_len.  Initially, @start_len bytes are matched.
  */
-static forceinline unsigned
+static forceinline u32
 lz_extend(const u8 * const strptr, const u8 * const matchptr,
-	  const unsigned start_len, const unsigned max_len)
+	  const u32 start_len, const u32 max_len)
 {
-	unsigned len = start_len;
+	u32 len = start_len;
 	machine_word_t v_word;
 
 	if (UNALIGNED_ACCESS_IS_FAST) {

lib/x86/adler32_impl.h

@@ -52,8 +52,16 @@
 /*
  * AVX-VNNI implementation.  This is used on CPUs that have AVX2 and AVX-VNNI
  * but don't have AVX-512, for example Intel Alder Lake.
+ *
+ * Unusually for a new CPU feature, gcc added support for the AVX-VNNI
+ * intrinsics (in gcc 11.1) slightly before binutils added support for
+ * assembling AVX-VNNI instructions (in binutils 2.36).  Distros can reasonably
+ * have gcc 11 with binutils 2.35.  Because of this issue, we check for gcc 12
+ * instead of gcc 11.  (libdeflate supports direct compilation without a
+ * configure step, so checking the binutils version is not always an option.)
  */
-#if GCC_PREREQ(11, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)
+#if (GCC_PREREQ(12, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI)
 #  define adler32_x86_avx2_vnni	adler32_x86_avx2_vnni
 #  define SUFFIX			   _avx2_vnni
 #  define ATTRIBUTES		_target_attribute("avx2,avxvnni")
@@ -63,17 +71,18 @@
 #  include "adler32_template.h"
 #endif
 
-#if GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)
+#if (GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI)
 /*
  * AVX512VNNI implementation using 256-bit vectors.  This is very similar to the
  * AVX-VNNI implementation but takes advantage of masking and more registers.
- * This is used on CPUs that support AVX-512 but where using 512-bit vectors
- * causes downclocking.  This should also be the optimal implementation on CPUs
- * that support AVX10/256 but not AVX10/512.
+ * This is used on certain older Intel CPUs, specifically Ice Lake and Tiger
+ * Lake, which support AVX512VNNI but downclock a bit too eagerly when ZMM
+ * registers are used.
  */
 #  define adler32_x86_avx512_vl256_vnni	adler32_x86_avx512_vl256_vnni
 #  define SUFFIX				   _avx512_vl256_vnni
-#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vl,avx512vnni")
+#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vl,avx512vnni" NO_EVEX512)
 #  define VL			32
 #  define USE_VNNI		1
 #  define USE_AVX512		1
@@ -81,12 +90,11 @@
 
 /*
  * AVX512VNNI implementation using 512-bit vectors.  This is used on CPUs that
- * have a good AVX-512 implementation including AVX512VNNI.  This should also be
- * the optimal implementation on CPUs that support AVX10/512.
+ * have a good AVX-512 implementation including AVX512VNNI.
  */
 #  define adler32_x86_avx512_vl512_vnni	adler32_x86_avx512_vl512_vnni
 #  define SUFFIX				   _avx512_vl512_vnni
-#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vnni")
+#  define ATTRIBUTES		_target_attribute("avx512bw,avx512vnni" EVEX512)
 #  define VL			64
 #  define USE_VNNI		1
 #  define USE_AVX512		1

lib/x86/cpu_features.c

@@ -88,9 +88,31 @@ static const struct cpu_feature x86_cpu_feature_table[] = {
 
 volatile u32 libdeflate_x86_cpu_features = 0;
 
+static inline bool
+os_supports_avx512(u64 xcr0)
+{
+#ifdef __APPLE__
+	/*
+	 * The Darwin kernel had a bug where it could corrupt the opmask
+	 * registers.  See
+	 * https://community.intel.com/t5/Software-Tuning-Performance/MacOS-Darwin-kernel-bug-clobbers-AVX-512-opmask-register-state/m-p/1327259
+	 * Darwin also does not initially set the XCR0 bits for AVX512, but they
+	 * are set if the thread tries to use AVX512 anyway.  Thus, to safely
+	 * and consistently use AVX512 on macOS we'd need to check the kernel
+	 * version as well as detect AVX512 support using a macOS-specific
+	 * method.  We don't bother with this, especially given Apple's
+	 * transition to arm64.
+	 */
+	return false;
+#else
+	return (xcr0 & 0xe6) == 0xe6;
+#endif
+}
+
 /*
- * Don't use 512-bit vectors on Intel CPUs before Rocket Lake and Sapphire
- * Rapids, due to the downclocking penalty.
+ * Don't use 512-bit vectors (ZMM registers) on Intel CPUs before Rocket Lake
+ * and Sapphire Rapids, due to the overly-eager downclocking which can reduce
+ * the performance of workloads that use ZMM registers only occasionally.
  */
 static inline bool
 allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
@@ -140,7 +162,12 @@ void libdeflate_init_x86_cpu_features(void)
 		family += (a >> 20) & 0xff;
 	if (d & (1 << 26))
 		features |= X86_CPU_FEATURE_SSE2;
-	if (c & (1 << 1))
+	/*
+	 * No known CPUs have pclmulqdq without sse4.1, so in practice code
+	 * targeting pclmulqdq can use sse4.1 instructions.  But to be safe,
+	 * explicitly check for both the pclmulqdq and sse4.1 bits.
+	 */
+	if ((c & (1 << 1)) && (c & (1 << 19)))
 		features |= X86_CPU_FEATURE_PCLMULQDQ;
 	if (c & (1 << 27))
 		xcr0 = read_xcr(0);
@@ -152,21 +179,24 @@ void libdeflate_init_x86_cpu_features(void)
 
 	/* EAX=7, ECX=0: Extended Features */
 	cpuid(7, 0, &a, &b, &c, &d);
-	if ((b & (1 << 5)) && ((xcr0 & 0x6) == 0x6))
-		features |= X86_CPU_FEATURE_AVX2;
 	if (b & (1 << 8))
 		features |= X86_CPU_FEATURE_BMI2;
-	if (((xcr0 & 0xe6) == 0xe6) &&
-	    allow_512bit_vectors(manufacturer, family, model))
-		features |= X86_CPU_FEATURE_ZMM;
-	if ((b & (1 << 30)) && ((xcr0 & 0xe6) == 0xe6))
-		features |= X86_CPU_FEATURE_AVX512BW;
-	if ((b & (1U << 31)) && ((xcr0 & 0xe6) == 0xe6))
-		features |= X86_CPU_FEATURE_AVX512VL;
-	if ((c & (1 << 10)) && ((xcr0 & 0x6) == 0x6))
-		features |= X86_CPU_FEATURE_VPCLMULQDQ;
-	if ((c & (1 << 11)) && ((xcr0 & 0xe6) == 0xe6))
-		features |= X86_CPU_FEATURE_AVX512VNNI;
+	if ((xcr0 & 0x6) == 0x6) {
+		if (b & (1 << 5))
+			features |= X86_CPU_FEATURE_AVX2;
+		if (c & (1 << 10))
+			features |= X86_CPU_FEATURE_VPCLMULQDQ;
+	}
+	if (os_supports_avx512(xcr0)) {
+		if (allow_512bit_vectors(manufacturer, family, model))
+			features |= X86_CPU_FEATURE_ZMM;
+		if (b & (1 << 30))
+			features |= X86_CPU_FEATURE_AVX512BW;
+		if (b & (1U << 31))
+			features |= X86_CPU_FEATURE_AVX512VL;
+		if (c & (1 << 11))
+			features |= X86_CPU_FEATURE_AVX512VNNI;
+	}
 
 	/* EAX=7, ECX=1: Extended Features */
 	cpuid(7, 1, &a, &b, &c, &d);

lib/x86/cpu_features.h

@@ -108,7 +108,8 @@ static inline u32 get_x86_cpu_features(void) { return 0; }
 #  define HAVE_SSE2_NATIVE		0
 #endif
 
-#if defined(__PCLMUL__) || (defined(_MSC_VER) && defined(__AVX2__))
+#if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
+	(defined(_MSC_VER) && defined(__AVX2__))
 #  define HAVE_PCLMULQDQ(features)	1
 #else
 #  define HAVE_PCLMULQDQ(features)	((features) & X86_CPU_FEATURE_PCLMULQDQ)
@@ -164,6 +165,15 @@ static inline u32 get_x86_cpu_features(void) { return 0; }
 #  define HAVE_AVXVNNI(features)	((features) & X86_CPU_FEATURE_AVXVNNI)
 #endif
 
+#if (GCC_PREREQ(14, 0) || CLANG_PREREQ(18, 0, 18000000)) \
+	&& !defined(__EVEX512__) /* avoid subtracting the evex512 feature */
+#  define EVEX512	",evex512"	/* needed to override potential -mno-evex512 */
+#  define NO_EVEX512	",no-evex512"
+#else
+#  define EVEX512	""
+#  define NO_EVEX512	""
+#endif
+
 #endif /* ARCH_X86_32 || ARCH_X86_64 */
 
 #endif /* LIB_X86_CPU_FEATURES_H */

lib/x86/crc32_impl.h

@@ -44,31 +44,26 @@ static const u8 MAYBE_UNUSED shift_tab[48] = {
 };
 
 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
-/* PCLMULQDQ implementation */
+/*
+ * PCLMULQDQ implementation.  This targets PCLMULQDQ+SSE4.1, since in practice
+ * all CPUs that support PCLMULQDQ also support SSE4.1.
+ */
 #  define crc32_x86_pclmulqdq	crc32_x86_pclmulqdq
 #  define SUFFIX			 _pclmulqdq
-#  define ATTRIBUTES		_target_attribute("pclmul")
+#  define ATTRIBUTES		_target_attribute("pclmul,sse4.1")
 #  define VL			16
-#  define USE_SSE4_1		0
 #  define USE_AVX512		0
 #  include "crc32_pclmul_template.h"
 
 /*
- * PCLMULQDQ/AVX implementation.  Compared to the regular PCLMULQDQ
- * implementation, this still uses 128-bit vectors, but it has two potential
- * benefits.  First, simply compiling against the AVX target can improve
- * performance significantly (e.g. 10100 MB/s to 16700 MB/s on Skylake) without
- * actually using any AVX intrinsics, probably due to the availability of
- * non-destructive VEX-encoded instructions.  Second, AVX support implies SSSE3
- * and SSE4.1 support, and we can use SSSE3 and SSE4.1 intrinsics for efficient
- * handling of partial blocks.  (We *could* compile a variant with
- * PCLMULQDQ+SSE4.1 without AVX, but for simplicity we currently don't bother.)
+ * PCLMULQDQ/AVX implementation.  Same as above, but this is compiled with AVX
+ * enabled so that the compiler can generate VEX-coded instructions which can be
+ * slightly more efficient.  It still uses 128-bit vectors.
  */
 #  define crc32_x86_pclmulqdq_avx	crc32_x86_pclmulqdq_avx
 #  define SUFFIX				 _pclmulqdq_avx
 #  define ATTRIBUTES		_target_attribute("pclmul,avx")
 #  define VL			16
-#  define USE_SSE4_1		1
 #  define USE_AVX512		0
 #  include "crc32_pclmul_template.h"
 #endif
@@ -80,43 +75,50 @@ static const u8 MAYBE_UNUSED shift_tab[48] = {
  * Currently this can't be enabled with MSVC because MSVC has a bug where it
  * incorrectly assumes that VPCLMULQDQ implies AVX-512:
  * https://developercommunity.visualstudio.com/t/Compiler-incorrectly-assumes-VAES-and-VP/10578785
+ *
+ * gcc 8.1 and 8.2 had a similar bug where they assumed that
+ * _mm256_clmulepi64_epi128() always needed AVX512.  It's fixed in gcc 8.3.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
  */
-#if GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000)
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
 #  define crc32_x86_vpclmulqdq_avx2	crc32_x86_vpclmulqdq_avx2
 #  define SUFFIX				 _vpclmulqdq_avx2
 #  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx2")
 #  define VL			32
-#  define USE_SSE4_1		1
 #  define USE_AVX512		0
 #  include "crc32_pclmul_template.h"
 #endif
 
-#if GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
 /*
  * VPCLMULQDQ/AVX512 implementation using 256-bit vectors.  This is very similar
  * to the VPCLMULQDQ/AVX2 implementation but takes advantage of the vpternlog
- * instruction and more registers.  This is used on CPUs that support AVX-512
- * but where using 512-bit vectors causes downclocking.  This should also be the
- * optimal implementation on CPUs that support AVX10/256 but not AVX10/512.
+ * instruction and more registers.  This is used on certain older Intel CPUs,
+ * specifically Ice Lake and Tiger Lake, which support VPCLMULQDQ and AVX512 but
+ * downclock a bit too eagerly when ZMM registers are used.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
  */
 #  define crc32_x86_vpclmulqdq_avx512_vl256  crc32_x86_vpclmulqdq_avx512_vl256
 #  define SUFFIX				      _vpclmulqdq_avx512_vl256
-#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl" NO_EVEX512)
 #  define VL			32
-#  define USE_SSE4_1		1
 #  define USE_AVX512		1
 #  include "crc32_pclmul_template.h"
 
 /*
  * VPCLMULQDQ/AVX512 implementation using 512-bit vectors.  This is used on CPUs
- * that have a good AVX-512 implementation including VPCLMULQDQ.  This should
- * also be the optimal implementation on CPUs that support AVX10/512.
+ * that have a good AVX-512 implementation including VPCLMULQDQ.
+ *
+ * _mm512_zextsi128_si512() requires gcc 10.
  */
 #  define crc32_x86_vpclmulqdq_avx512_vl512  crc32_x86_vpclmulqdq_avx512_vl512
 #  define SUFFIX				      _vpclmulqdq_avx512_vl512
-#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl" EVEX512)
 #  define VL			64
-#  define USE_SSE4_1		1
 #  define USE_AVX512		1
 #  include "crc32_pclmul_template.h"
 #endif

lib/x86/crc32_pclmul_template.h

@@ -34,17 +34,13 @@
  * ATTRIBUTES:
  *	Target function attributes to use.  Must satisfy the dependencies of the
  *	other parameters as follows:
- *	   VL=16 && USE_SSE4_1=0 && USE_AVX512=0: at least pclmul
- *	   VL=16 && USE_SSE4_1=1 && USE_AVX512=0: at least pclmul,sse4.1
- *	   VL=32 && USE_SSE4_1=1 && USE_AVX512=0: at least vpclmulqdq,pclmul,avx2
- *	   VL=32 && USE_SSE4_1=1 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
- *	   VL=64 && USE_SSE4_1=1 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
+ *	   VL=16 && USE_AVX512=0: at least pclmul,sse4.1
+ *	   VL=32 && USE_AVX512=0: at least vpclmulqdq,pclmul,avx2
+ *	   VL=32 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
+ *	   VL=64 && USE_AVX512=1: at least vpclmulqdq,pclmul,avx512bw,avx512vl
  *	   (Other combinations are not useful and have not been tested.)
  * VL:
  *	Vector length in bytes.  Must be 16, 32, or 64.
- * USE_SSE4_1:
- *	If 1, take advantage of SSE4.1 instructions such as pblendvb.
- *	If 0, assume that the CPU might not support SSE4.1.
  * USE_AVX512:
  *	If 1, take advantage of AVX-512 features such as masking and the
  *	vpternlog instruction.  This doesn't enable the use of 512-bit vectors;
@@ -55,7 +51,10 @@
  * instructions.  Note that the x86 crc32 instruction cannot be used, as it is
  * for a different polynomial, not the gzip one.  For an explanation of CRC
  * folding with carryless multiplication instructions, see
- * scripts/gen_crc32_multipliers.c and the following paper:
+ * scripts/gen-crc32-consts.py and the following blog posts and papers:
+ *
+ *	"An alternative exposition of crc32_4k_pclmulqdq"
+ *	https://www.corsix.org/content/alternative-exposition-crc32_4k_pclmulqdq
  *
  *	"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
  *	https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
@@ -81,7 +80,7 @@
 #  define fold_vec		fold_vec256
 #  define VLOADU(p)		_mm256_loadu_si256((const void *)(p))
 #  define VXOR(a, b)		_mm256_xor_si256((a), (b))
-#  define M128I_TO_VEC(a)	_mm256_castsi128_si256(a)
+#  define M128I_TO_VEC(a)	_mm256_zextsi128_si256(a)
 #  define MULTS(a, b)		_mm256_set_epi64x(a, b, a, b)
 #  define MULTS_8V		MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
 #  define MULTS_4V		MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
@@ -92,7 +91,7 @@
 #  define fold_vec		fold_vec512
 #  define VLOADU(p)		_mm512_loadu_si512((const void *)(p))
 #  define VXOR(a, b)		_mm512_xor_si512((a), (b))
-#  define M128I_TO_VEC(a)	_mm512_castsi128_si512(a)
+#  define M128I_TO_VEC(a)	_mm512_zextsi128_si512(a)
 #  define MULTS(a, b)		_mm512_set_epi64(a, b, a, b, a, b, a, b)
 #  define MULTS_8V		MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
 #  define MULTS_4V		MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
@@ -149,7 +148,6 @@ ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i /* __v8du */ mults)
 #define fold_vec512	ADD_SUFFIX(fold_vec512)
 #endif /* VL >= 64 */
 
-#if USE_SSE4_1
 /*
  * Given 'x' containing a 16-byte polynomial, and a pointer 'p' that points to
  * the next '1 <= len <= 15' data bytes, rearrange the concatenation of 'x' and
@@ -181,7 +179,6 @@ ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
 	return fold_vec128(x0, x1, mults_128b);
 }
 #define fold_lessthan16bytes	ADD_SUFFIX(fold_lessthan16bytes)
-#endif /* USE_SSE4_1 */
 
 static ATTRIBUTES u32
 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
@@ -192,15 +189,13 @@ ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
 	 * folding across 128 bits.  mults_128b differs from mults_1v when
 	 * VL != 16.  All multipliers are 64-bit, to match what pclmulqdq needs,
 	 * but since this is for CRC-32 only their low 32 bits are nonzero.
-	 * For more details, see scripts/gen_crc32_multipliers.c.
+	 * For more details, see scripts/gen-crc32-consts.py.
 	 */
 	const vec_t mults_8v = MULTS_8V;
 	const vec_t mults_4v = MULTS_4V;
 	const vec_t mults_2v = MULTS_2V;
 	const vec_t mults_1v = MULTS_1V;
 	const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
-	const __m128i final_mult = _mm_set_epi64x(0, CRC32_X63_MODG);
-	const __m128i mask32 = _mm_set_epi32(0, 0, 0, 0xFFFFFFFF);
 	const __m128i barrett_reduction_constants =
 		_mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
 	vec_t v0, v1, v2, v3, v4, v5, v6, v7;
@@ -273,7 +268,6 @@ ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
 			size_t align = -(uintptr_t)p & (VL-1);
 
 			len -= align;
-		#if USE_SSE4_1
 			x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
 			p += 16;
 			if (align & 15) {
@@ -296,11 +290,6 @@ ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
 			v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
 		#  endif
 			p -= 16;
-		#else
-			crc = crc32_slice1(crc, p, align);
-			p += align;
-			v0 = VXOR(VLOADU(p), M128I_TO_VEC(_mm_cvtsi32_si128(crc)));
-		#endif
 		} else {
 			v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
 		}
@@ -395,86 +384,27 @@ less_than_vl_remaining:
 less_than_16_remaining:
 	len &= 15;
 
-	/*
-	 * If fold_lessthan16bytes() is available, handle any remainder
-	 * of 1 to 15 bytes now, before reducing to 32 bits.
-	 */
-#if USE_SSE4_1
+	/* Handle any remainder of 1 to 15 bytes. */
 	if (len)
 		x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
-#endif
 #if USE_AVX512
 reduce_x0:
 #endif
-
 	/*
-	 * Fold 128 => 96 bits.  This also implicitly appends 32 zero bits,
-	 * which is equivalent to multiplying by x^32.  This is needed because
-	 * the CRC is defined as M(x)*x^32 mod G(x), not just M(x) mod G(x).
+	 * Multiply the remaining 128-bit message polynomial 'x0' by x^32, then
+	 * reduce it modulo the generator polynomial G.  This gives the CRC.
+	 *
+	 * This implementation matches that used in crc-pclmul-template.S from
+	 * https://lore.kernel.org/r/20250210174540.161705-4-ebiggers@kernel.org/
+	 * with the parameters n=32 and LSB_CRC=1 (what the gzip CRC uses).  See
+	 * there for a detailed explanation of the math used here.
 	 */
-	x0 = _mm_xor_si128(_mm_srli_si128(x0, 8),
-			   _mm_clmulepi64_si128(x0, mults_128b, 0x10));
-
-	/* Fold 96 => 64 bits. */
-	x0 = _mm_xor_si128(_mm_srli_si128(x0, 4),
-			   _mm_clmulepi64_si128(_mm_and_si128(x0, mask32),
-						final_mult, 0x00));
-
-	/*
-	 * Reduce 64 => 32 bits using Barrett reduction.
-	 *
-	 * Let M(x) = A(x)*x^32 + B(x) be the remaining message.  The goal is to
-	 * compute R(x) = M(x) mod G(x).  Since degree(B(x)) < degree(G(x)):
-	 *
-	 *	R(x) = (A(x)*x^32 + B(x)) mod G(x)
-	 *	     = (A(x)*x^32) mod G(x) + B(x)
-	 *
-	 * Then, by the Division Algorithm there exists a unique q(x) such that:
-	 *
-	 *	A(x)*x^32 mod G(x) = A(x)*x^32 - q(x)*G(x)
-	 *
-	 * Since the left-hand side is of maximum degree 31, the right-hand side
-	 * must be too.  This implies that we can apply 'mod x^32' to the
-	 * right-hand side without changing its value:
-	 *
-	 *	(A(x)*x^32 - q(x)*G(x)) mod x^32 = q(x)*G(x) mod x^32
-	 *
-	 * Note that '+' is equivalent to '-' in polynomials over GF(2).
-	 *
-	 * We also know that:
-	 *
-	 *	              / A(x)*x^32 \
-	 *	q(x) = floor (  ---------  )
-	 *	              \    G(x)   /
-	 *
-	 * To compute this efficiently, we can multiply the top and bottom by
-	 * x^32 and move the division by G(x) to the top:
-	 *
-	 *	              / A(x) * floor(x^64 / G(x)) \
-	 *	q(x) = floor (  -------------------------  )
-	 *	              \           x^32            /
-	 *
-	 * Note that floor(x^64 / G(x)) is a constant.
-	 *
-	 * So finally we have:
-	 *
-	 *	                          / A(x) * floor(x^64 / G(x)) \
-	 *	R(x) = B(x) + G(x)*floor (  -------------------------  )
-	 *	                          \           x^32            /
-	 */
-	x1 = _mm_clmulepi64_si128(_mm_and_si128(x0, mask32),
-				  barrett_reduction_constants, 0x00);
-	x1 = _mm_clmulepi64_si128(_mm_and_si128(x1, mask32),
-				  barrett_reduction_constants, 0x10);
+	x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
+			   _mm_bsrli_si128(x0, 8));
+	x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
+	x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
 	x0 = _mm_xor_si128(x0, x1);
-#if USE_SSE4_1
-	crc = _mm_extract_epi32(x0, 1);
-#else
-	crc = _mm_cvtsi128_si32(_mm_shuffle_epi32(x0, 0x01));
-	/* Process up to 15 bytes left over at the end. */
-	crc = crc32_slice1(crc, p, len);
-#endif
-	return crc;
+	return _mm_extract_epi32(x0, 2);
 }
 
 #undef vec_t
@@ -491,5 +421,4 @@ reduce_x0:
 #undef SUFFIX
 #undef ATTRIBUTES
 #undef VL
-#undef USE_SSE4_1
 #undef USE_AVX512

libdeflate.h

@@ -13,8 +13,8 @@ extern "C" {
 #endif
 
 #define LIBDEFLATE_VERSION_MAJOR	1
-#define LIBDEFLATE_VERSION_MINOR	20
-#define LIBDEFLATE_VERSION_STRING	"1.20"
+#define LIBDEFLATE_VERSION_MINOR	23
+#define LIBDEFLATE_VERSION_STRING	"1.23"
 
 /*
  * Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause

scripts/deflate_benchmarks.sh

@@ -72,7 +72,7 @@ single_file()
 	local include_old=false
 
 	usize=$(stat -c "%s" "$file")
-	: ${NUM_ITERATIONS:=3}
+	: "${NUM_ITERATIONS:=3}"
 
 	if [ -e "$topdir/benchmark-old" ]; then
 		include_old=true

scripts/gen-crc32-consts.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+#
+# This script generates constants for efficient computation of the gzip CRC-32.
+
+import sys
+
+# This is the generator polynomial G(x) of the gzip CRC-32, represented as an
+# int using the natural mapping between bits and polynomial coefficients.
+G = 0x104c11db7
+
+# XOR (add) an iterable of polynomials.
+def xor(iterable):
+    res = 0
+    for val in iterable:
+        res ^= val
+    return res
+
+# Multiply two polynomials.
+def clmul(a, b):
+    return xor(a << i for i in range(b.bit_length()) if (b & (1 << i)) != 0)
+
+# Polynomial division floor(a / b).
+def div(a, b):
+    q = 0
+    while a.bit_length() >= b.bit_length():
+        q ^= 1 << (a.bit_length() - b.bit_length())
+        a ^= b << (a.bit_length() - b.bit_length())
+    return q
+
+# Reduce the polynomial 'a' modulo the polynomial 'b'.
+def reduce(a, b):
+    return a ^ clmul(div(a, b), b)
+
+# Reverse the bits of a polynomial.
+def bitreverse(poly, num_bits):
+    return xor(1 << (num_bits - 1 - i) for i in range(num_bits)
+               if (poly & (1 << i)) != 0)
+
+# Compute x^d mod G.
+def x_to_the_d(d):
+    if d < G.bit_length() - 1:
+        return 1 << d
+    t = x_to_the_d(d//2)
+    t = clmul(t, t)
+    if d % 2 != 0:
+        t <<= 1
+    return reduce(t, G)
+
+def gen_tables():
+    print('/*')
+    print(' * crc32_tables.h - data tables for CRC-32 computation')
+    print(' *')
+    print(' * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.')
+    print(' */')
+    for n in [1, 8]:
+        print('')
+        print(f'static const u32 crc32_slice{n}_table[] MAYBE_UNUSED = {{')
+        # The i'th table entry is the CRC-32 of the message consisting of byte
+        # i % 256 followed by i // 256 zero bytes.
+        polys = [bitreverse(i % 256, 8) << (32 + 8*(i//256)) for i in range(256 * n)]
+        polys = [bitreverse(reduce(poly, G), 32) for poly in polys]
+        for i in range(0, len(polys), 4):
+            print(f'\t0x{polys[i+0]:08x}, 0x{polys[i+1]:08x}, 0x{polys[i+2]:08x}, 0x{polys[i+3]:08x},')
+        print('};')
+
+# Compute the constant multipliers needed for "folding" over various distances
+# with the gzip CRC-32.  Each such multiplier is x^d mod G(x) for some distance
+# d, in bits, over which the folding is occurring.
+#
+# Folding works as follows: let A(x) be a polynomial (possibly reduced partially
+# or fully mod G(x)) for part of the message, and let B(x) be a polynomial
+# (possibly reduced partially or fully mod G(x)) for a later part of the
+# message.  The unreduced combined polynomial is A(x)*x^d + B(x), where d is the
+# number of bits separating the two parts of the message plus len(B(x)).  Since
+# mod G(x) can be applied at any point, x^d mod G(x) can be precomputed and used
+# instead of x^d unreduced.  That allows the combined polynomial to be computed
+# relatively easily in a partially-reduced form A(x)*(x^d mod G(x)) + B(x), with
+# length max(len(A(x)) + 31, len(B(x))).  This does require doing a polynomial
+# multiplication (carryless multiplication).
+#
+# "Folding" in this way can be used for the entire CRC computation except the
+# final reduction to 32 bits; this works well when CPU support for carryless
+# multiplication is available.  It can also be used to combine CRCs of different
+# parts of the message that were computed using a different method.
+#
+# Note that the gzip CRC-32 uses bit-reversed polynomials.  I.e., the low order
+# bits are really the high order polynomial coefficients.
+def gen_multipliers():
+    print('/*')
+    print(' * crc32_multipliers.h - constants for CRC-32 folding')
+    print(' *')
+    print(' * THIS FILE WAS GENERATED BY gen-crc32-consts.py.  DO NOT EDIT.')
+    print(' */')
+    print('')
+
+    # Compute the multipliers needed for CRC-32 folding with carryless
+    # multiplication instructions that operate on the 64-bit halves of 128-bit
+    # segments.  Using the terminology from earlier, for each 64-bit fold
+    # len(A(x)) = 64, and len(B(x)) = 95 since a 64-bit polynomial multiplied by
+    # a 32-bit one produces a 95-bit one.  When A(x) is the low order polynomial
+    # half of a 128-bit segments (high order physical half), the separation
+    # between the message parts is the total length of the 128-bit segments
+    # separating the values.  When A(x) is the high order polynomial half, the
+    # separation is 64 bits greater.
+    for i in range(1, 33):
+        sep_lo = 128 * (i - 1)
+        sep_hi = sep_lo + 64
+        len_B = 95
+        for d in [sep_hi + len_B, # A(x) = high 64 polynomial bits (low 64 physical bits)
+                  sep_lo + len_B # A(x) = low 64 polynomial bits (high 64 physical bits)
+                  ]:
+            poly = bitreverse(x_to_the_d(d), 32)
+            print(f'#define CRC32_X{d}_MODG 0x{poly:08x} /* x^{d} mod G(x) */')
+        print('')
+
+    # Compute constants for the final 128 => 32 bit reduction.
+    poly = bitreverse(div(1 << 95, G), 64)
+    print(f'#define CRC32_BARRETT_CONSTANT_1 0x{poly:016x}ULL /* floor(x^95 / G(x)) */')
+    poly = bitreverse(G, 33)
+    print(f'#define CRC32_BARRETT_CONSTANT_2 0x{poly:016x}ULL /* G(x) */')
+
+    # Compute multipliers for combining the CRCs of separate chunks.
+    print('')
+    num_chunks = 4
+    table_len = 129
+    min_chunk_len = 128
+    print(f'#define CRC32_NUM_CHUNKS {num_chunks}')
+    print(f'#define CRC32_MIN_VARIABLE_CHUNK_LEN {min_chunk_len}UL')
+    print(f'#define CRC32_MAX_VARIABLE_CHUNK_LEN {(table_len-1) * min_chunk_len}UL')
+    print('')
+    print('/* Multipliers for implementations that use a variable chunk length */')
+    print('static const u32 crc32_mults_for_chunklen[][CRC32_NUM_CHUNKS - 1] MAYBE_UNUSED = {')
+    print('\t{ 0 /* unused row */ },')
+    for i in range(1, table_len):
+        chunk_len = i * min_chunk_len
+        print(f'\t/* chunk_len={chunk_len} */')
+        print('\t{ ', end='')
+        for j in range(num_chunks - 1, 0, -1):
+            d = (j * 8 * chunk_len) - 33
+            poly = bitreverse(x_to_the_d(d), 32)
+            print(f'0x{poly:08x} /* x^{d} mod G(x) */, ', end='')
+        print('},')
+    print('};')
+    fixed_chunk_len = 32768
+    print('')
+    print('/* Multipliers for implementations that use a large fixed chunk length */')
+    print(f'#define CRC32_FIXED_CHUNK_LEN {fixed_chunk_len}UL')
+    for j in range(1, num_chunks):
+        d = (j * 8 * fixed_chunk_len) - 33
+        poly = bitreverse(x_to_the_d(d), 32)
+        print(f'#define CRC32_FIXED_CHUNK_MULT_{j} 0x{poly:08x} /* x^{d} mod G(x) */')
+
+with open('lib/crc32_tables.h', 'w') as f:
+    sys.stdout = f
+    gen_tables()
+with open('lib/crc32_multipliers.h', 'w') as f:
+    sys.stdout = f
+    gen_multipliers()

scripts/gen_crc32_multipliers.c

@@ -1,199 +0,0 @@
-/*
- * gen_crc32_multipliers.c
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * This program computes the constant multipliers needed for "folding" over
- * various distances with the gzip CRC-32.  Each such multiplier is x^D mod G(x)
- * for some distance D, in bits, over which the folding is occurring.
- *
- * Folding works as follows: let A(x) be a polynomial (possibly reduced
- * partially or fully mod G(x)) for part of the message, and let B(x) be a
- * polynomial (possibly reduced partially or fully mod G(x)) for a later part of
- * the message.  The unreduced combined polynomial is A(x)*x^D + B(x), where D
- * is the number of bits separating the two parts of the message plus len(B(x)).
- * Since mod G(x) can be applied at any point, x^D mod G(x) can be precomputed
- * and used instead of x^D unreduced.  That allows the combined polynomial to be
- * computed relatively easily in a partially-reduced form A(x)*(x^D mod G(x)) +
- * B(x), with length max(len(A(x)) + 31, len(B(x))).  This does require doing a
- * polynomial multiplication (carryless multiplication).
- *
- * "Folding" in this way can be used for the entire CRC computation except the
- * final reduction to 32 bits; this works well when CPU support for carryless
- * multiplication is available.  It can also be used to combine CRCs of
- * different parts of the message that were computed using a different method.
- *
- * Note that the gzip CRC-32 uses bit-reversed polynomials.  I.e., the low order
- * bits are really the high order polynomial coefficients.
- */
-
-#include <inttypes.h>
-#include <stdio.h>
-
-#include "../common_defs.h"
-
-/* The generator polynomial G(x) for the gzip CRC-32 */
-#define CRCPOLY		0xEDB88320 /* G(x) without x^32 term */
-#define CRCPOLY_FULL	(((u64)CRCPOLY << 1) | 1) /* G(x) */
-
-/* Compute x^D mod G(x) */
-static u32
-compute_xD_modG(size_t D)
-{
-	/* Start with x^0 mod G(x) */
-	u32 remainder = 0x80000000;
-
-	/* Each iteration, 'remainder' becomes x^i mod G(x) */
-	for (size_t i = 1; i <= D; i++)
-		remainder = (remainder >> 1) ^ ((remainder & 1) ? CRCPOLY : 0);
-
-	/* Now 'remainder' is x^D mod G(x) */
-	return remainder;
-}
-
-/* Compute floor(x^64 / G(x)) */
-static u64
-compute_x64_div_G(void)
-{
-	u64 quotient = 0;
-	u64 dividend = 0x1;
-
-	for (int i = 0; i < 64 - 32 + 1; i++) {
-		if ((dividend >> i) & 1) {
-			quotient |= (u64)1 << i;
-			dividend ^= CRCPOLY_FULL << i;
-		}
-	}
-
-	return quotient;
-}
-
-static void
-gen_vec_folding_constants(void)
-{
-	/*
-	 * Compute the multipliers needed for CRC-32 folding with carryless
-	 * multiplication instructions that operate on the 64-bit halves of
-	 * 128-bit segments.  Using the terminology from earlier, for each 64-bit
-	 * fold len(A(x)) = 64, and len(B(x)) = 95 since a 64-bit polynomial
-	 * multiplied by a 32-bit one produces a 95-bit one.  When A(x) is the
-	 * low order polynomial half of a 128-bit segments (high order physical
-	 * half), the separation between the message parts is the total length
-	 * of the 128-bit segments separating the values.  When A(x) is the high
-	 * order polynomial half, the separation is 64 bits greater.
-	 */
-	for (int i = 1; i <= 32; i++) {
-		const int sep_lo = 128 * (i - 1);
-		const int sep_hi = sep_lo + 64;
-		const int len_B = 95;
-		int D;
-
-		/* A(x) = high 64 polynomial bits (low 64 physical bits) */
-		D = sep_hi + len_B;
-		printf("#define CRC32_X%d_MODG 0x%08"PRIx32" /* x^%d mod G(x) */\n",
-		       D, compute_xD_modG(D), D);
-
-		/* A(x) = low 64 polynomial bits (high 64 physical bits) */
-		D = sep_lo + len_B;
-		printf("#define CRC32_X%d_MODG 0x%08"PRIx32" /* x^%d mod G(x) */\n",
-		       D, compute_xD_modG(D), D);
-		printf("\n");
-	}
-
-	/* Multiplier for final 96 => 64 bit fold */
-	printf("#define CRC32_X63_MODG 0x%08"PRIx32" /* x^63 mod G(x) */\n",
-	       compute_xD_modG(63));
-
-	/*
-	 * Constants for final 64 => 32 bit reduction.  These constants are the
-	 * odd ones out, as this final reduction step can't use the regular CRC
-	 * folding described above.  It uses Barrett reduction instead.
-	 */
-	printf("#define CRC32_BARRETT_CONSTANT_1 0x%016"PRIx64"ULL /* floor(x^64 / G(x)) */\n",
-	       compute_x64_div_G());
-	printf("#define CRC32_BARRETT_CONSTANT_2 0x%016"PRIx64"ULL /* G(x) */\n",
-	       CRCPOLY_FULL);
-	printf("#define CRC32_BARRETT_CONSTANTS { CRC32_BARRETT_CONSTANT_1, CRC32_BARRETT_CONSTANT_2 }\n");
-}
-
-/* Multipliers for combining the CRCs of separate chunks */
-static void
-gen_chunk_constants(void)
-{
-	const size_t num_chunks = 4;
-	const size_t table_len = 129;
-	const size_t min_chunk_len = 128;
-
-	printf("#define CRC32_NUM_CHUNKS %zu\n", num_chunks);
-	printf("#define CRC32_MIN_VARIABLE_CHUNK_LEN %zuUL\n", min_chunk_len);
-	printf("#define CRC32_MAX_VARIABLE_CHUNK_LEN %zuUL\n",
-	       (table_len - 1) * min_chunk_len);
-	printf("\n");
-	printf("/* Multipliers for implementations that use a variable chunk length */\n");
-	printf("static const u32 crc32_mults_for_chunklen[][CRC32_NUM_CHUNKS - 1] MAYBE_UNUSED = {\n");
-	printf("\t{ 0 /* unused row */ },\n");
-	for (size_t i = 1; i < table_len; i++) {
-		const size_t chunk_len = i*min_chunk_len;
-
-		printf("\t/* chunk_len=%zu */\n", chunk_len);
-		printf("\t{ ");
-		for (size_t j = num_chunks - 1; j >= 1; j--) {
-			const size_t D = (j * 8 * chunk_len) - 33;
-
-			printf("0x%08"PRIx32" /* x^%zu mod G(x) */, ",
-			       compute_xD_modG(D), D);
-		}
-		printf("},\n");
-	}
-	printf("};\n");
-	printf("\n");
-
-	printf("/* Multipliers for implementations that use a large fixed chunk length */\n");
-	const size_t fixed_chunk_len = 32768;
-	printf("#define CRC32_FIXED_CHUNK_LEN %zuUL\n", fixed_chunk_len);
-	for (int j = 1; j < num_chunks; j++) {
-		const size_t D = (j * 8 * fixed_chunk_len) - 33;
-
-		printf("#define CRC32_FIXED_CHUNK_MULT_%d 0x%08"PRIx32" /* x^%zu mod G(x) */\n",
-		       j, compute_xD_modG(D), D);
-	}
-}
-
-int
-main(void)
-{
-	printf("/*\n"
-	       " * crc32_multipliers.h - constants for CRC-32 folding\n"
-	       " *\n"
-	       " * THIS FILE WAS GENERATED BY gen_crc32_multipliers.c.  DO NOT EDIT.\n"
-	       " */\n"
-	       "\n");
-
-	gen_vec_folding_constants();
-	printf("\n");
-	gen_chunk_constants();
-	return 0;
-}

scripts/gen_crc32_tables.c

@@ -1,105 +0,0 @@
-/*
- * gen_crc32_tables.c - a program for CRC-32 table generation
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <stdio.h>
-
-#include "../common_defs.h"
-
-#define CRCPOLY	0xEDB88320 /* G(x) without x^32 term */
-
-static u32
-crc32_update_bit(u32 remainder, u8 next_bit)
-{
-	return (remainder >> 1) ^ (((remainder ^ next_bit) & 1) ? CRCPOLY : 0);
-}
-
-static u32
-crc32_update_byte(u32 remainder, u8 next_byte)
-{
-	for (int j = 0; j < 8; j++, next_byte >>= 1)
-		remainder = crc32_update_bit(remainder, next_byte & 1);
-	return remainder;
-}
-
-static void
-print_256_entries(const u32 *entries)
-{
-	for (size_t i = 0; i < 256 / 4; i++) {
-		printf("\t");
-		for (size_t j = 0; j < 4; j++) {
-			printf("0x%08x,", entries[i * 4 + j]);
-			if (j != 3)
-				printf(" ");
-		}
-		printf("\n");
-	}
-}
-
-int
-main(void)
-{
-	u32 crc32_table[0x800];
-
-	/* crc32_table[i] for 0 <= i < 0x100 is the CRC-32 of byte i. */
-	for (int i = 0; i < 0x100; i++)
-		crc32_table[i] = crc32_update_byte(0, i);
-
-	/*
-	 * crc32_table[i] for 0x100 <= i < 0x800 is the CRC-32 of byte i % 0x100
-	 * followed by i / 0x100 zero bytes.
-	 */
-	for (int i = 0x100; i < 0x800; i++)
-		crc32_table[i] = crc32_update_byte(crc32_table[i - 0x100], 0);
-
-	printf("/*\n");
-	printf(" * crc32_tables.h - data tables for CRC-32 computation\n");
-	printf(" *\n");
-	printf(" * THIS FILE WAS GENERATED BY gen_crc32_tables.c.  DO NOT EDIT.\n");
-	printf(" */\n");
-	printf("\n");
-	/*
-	 * Although crc32_slice1_table is the same as the first 256 entries of
-	 * crc32_slice8_table, we output these tables separately because any
-	 * combo of (slice1, slice8, slice1 && slice8, nothing) might be needed,
-	 * and it's simplest to let the compiler optimize out any unused tables.
-	 */
-	printf("static const u32 crc32_slice1_table[] MAYBE_UNUSED = {\n");
-	print_256_entries(&crc32_table[0x000]);
-	printf("};\n");
-	printf("\n");
-	printf("static const u32 crc32_slice8_table[] MAYBE_UNUSED = {\n");
-	print_256_entries(&crc32_table[0x000]);
-	print_256_entries(&crc32_table[0x100]);
-	print_256_entries(&crc32_table[0x200]);
-	print_256_entries(&crc32_table[0x300]);
-	print_256_entries(&crc32_table[0x400]);
-	print_256_entries(&crc32_table[0x500]);
-	print_256_entries(&crc32_table[0x600]);
-	print_256_entries(&crc32_table[0x700]);
-	printf("};\n");
-	return 0;
-}

scripts/libFuzzer/deflate_compress/fuzz.c

@@ -4,6 +4,36 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+static void
+alloc_guarded_buffer(size_t size, uint8_t **start_ret, uint8_t **end_ret)
+{
+	const size_t pagesize = sysconf(_SC_PAGESIZE);
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	uint8_t *base_addr, *start, *end;
+
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	assert(base_addr != (uint8_t *)MAP_FAILED);
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+
+	*start_ret = start;
+	*end_ret = end;
+}
+
+static void
+free_guarded_buffer(uint8_t *start, uint8_t *end)
+{
+	munmap(start, end - start);
+}
 
 /* Fuzz the DEFLATE compression and decompression round trip. */
 int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
@@ -13,8 +43,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
 	struct libdeflate_compressor *c;
 	struct libdeflate_decompressor *d;
 	size_t csize_avail;
-	uint8_t *cbuf;
-	uint8_t *decompressed;
+	uint8_t *ubuf_start, *ubuf_end, *ubuf;
+	uint8_t *cbuf_start, *cbuf_end, *cbuf;
+	uint8_t *dbuf_start, *dbuf_end, *dbuf;
 	size_t csize;
 	enum libdeflate_result res;
 
@@ -29,24 +60,36 @@ int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
 	c = libdeflate_alloc_compressor(level);
 	d = libdeflate_alloc_decompressor();
 
+	/* Use guard pages to make all input/output buffer overflows segfault */
+
+	alloc_guarded_buffer(insize, &ubuf_start, &ubuf_end);
+	ubuf = ubuf_end - insize;
+	memcpy(ubuf, in, insize);
+
 	csize_avail = use_bound ? libdeflate_deflate_compress_bound(c, insize) :
 				  insize;
-	cbuf = malloc(csize_avail);
-	decompressed = malloc(insize);
+	alloc_guarded_buffer(csize_avail, &cbuf_start, &cbuf_end);
+	cbuf = cbuf_end - csize_avail;
 
-	csize = libdeflate_deflate_compress(c, in, insize, cbuf, csize_avail);
+	alloc_guarded_buffer(insize, &dbuf_start, &dbuf_end);
+	dbuf = dbuf_end - insize;
+
+	csize = libdeflate_deflate_compress(c, ubuf, insize, cbuf, csize_avail);
 	if (csize != 0) {
-		res = libdeflate_deflate_decompress(d, cbuf, csize, decompressed,
-						    insize, NULL);
+		assert(csize <= csize_avail);
+		memmove(cbuf_end - csize, cbuf, csize);
+		res = libdeflate_deflate_decompress(d, cbuf_end - csize, csize,
+						    dbuf, insize, NULL);
 		assert(res == LIBDEFLATE_SUCCESS);
-		assert(memcmp(in, decompressed, insize) == 0);
+		assert(memcmp(in, dbuf, insize) == 0);
 	} else {
 		assert(!use_bound);
 	}
 
 	libdeflate_free_compressor(c);
 	libdeflate_free_decompressor(d);
-	free(cbuf);
-	free(decompressed);
+	free_guarded_buffer(ubuf_start, ubuf_end);
+	free_guarded_buffer(cbuf_start, cbuf_end);
+	free_guarded_buffer(dbuf_start, dbuf_end);
 	return 0;
 }

scripts/libFuzzer/deflate_decompress/fuzz.c

@@ -1,19 +1,62 @@
+#include <assert.h>
 #include <libdeflate.h>
+#include <stdbool.h>
 #include <stdint.h>
 #include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+static void
+alloc_guarded_buffer(size_t size, uint8_t **start_ret, uint8_t **end_ret)
+{
+	const size_t pagesize = sysconf(_SC_PAGESIZE);
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	uint8_t *base_addr, *start, *end;
+
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	assert(base_addr != (uint8_t *)MAP_FAILED);
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+
+	*start_ret = start;
+	*end_ret = end;
+}
+
+static void
+free_guarded_buffer(uint8_t *start, uint8_t *end)
+{
+	munmap(start, end - start);
+}
 
 /* Fuzz DEFLATE decompression. */
 int LLVMFuzzerTestOneInput(const uint8_t *in, size_t insize)
 {
 	size_t outsize_avail = 3 * insize;
-	uint8_t *out;
+	uint8_t *cbuf_start, *cbuf_end, *cbuf;
+	uint8_t *dbuf_start, *dbuf_end, *dbuf;
 	struct libdeflate_decompressor *d;
 
-	out = malloc(outsize_avail);
+	/* Use guard pages to make all input/output buffer overflows segfault */
+
+	alloc_guarded_buffer(insize, &cbuf_start, &cbuf_end);
+	cbuf = cbuf_end - insize;
+	memcpy(cbuf, in, insize);
+
+	alloc_guarded_buffer(outsize_avail, &dbuf_start, &dbuf_end);
+	dbuf = dbuf_end - outsize_avail;
 
 	d = libdeflate_alloc_decompressor();
-	libdeflate_deflate_decompress(d, in, insize, out, outsize_avail, NULL);
+	libdeflate_deflate_decompress(d, cbuf, insize, dbuf, outsize_avail,
+				      NULL);
 	libdeflate_free_decompressor(d);
-	free(out);
+	free_guarded_buffer(cbuf_start, cbuf_end);
+	free_guarded_buffer(dbuf_start, dbuf_end);
 	return 0;
 }

scripts/libFuzzer/fuzz.sh

@@ -103,6 +103,6 @@ if [ ! -e "$TARGET/fuzz.c" ]; then
 	exit 1
 fi
 run_cmd clang -g -O1 -fsanitize=fuzzer$EXTRA_SANITIZERS \
-	-Wall -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS=1 \
+	-Wall -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS=1 -I ../../ \
 	../../lib/*{,/*}.c "$TARGET/fuzz.c" -o "$TARGET/fuzz"
 run_cmd "$TARGET/fuzz" "${EXTRA_FUZZER_ARGS[@]}" "$TARGET/corpus"