Add extra space to match linux kernel

It is no longer necessary to get good performance, there is only a small
speed difference between -O2 and -O3, so just stick to the default of
-O2.

This also fixes the stack space usage on parisc. The compiler was buggy
for -O3 and used ~3KB of stack space for several functions. With -O2 the
problem is completely resolved, and stack space is back to a few hundred
bytes.

Additionally, we get a large code size win on gcc:

| Compiler | Before (Bytes) | After (Bytes) | Delta (Bytes) |
|----------|----------------|---------------|---------------|
| gcc-11   |         952754 |        738954 |       -213800 |
| clang-12 |         976290 |        938826 |        -37464 |

.circleci/config.yml

@@ -7,21 +7,19 @@ jobs:
     #       preinstalled to reduce installation time.
     docker:
       - image: fbopensource/zstd-circleci-primary:0.0.1
-    # TODO: Re-enable aarch64 build:
-    #       make aarch64build     && make clean
     steps:
       - checkout
       - run:
           name: Test
           command: |
             ./tests/test-license.py
-            cc -v
-            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
-            make c99build; make clean
-            make c11build; make clean
+            cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
+            make c99build         ; make clean
+            make c11build         ; make clean
+            make aarch64build     ; make clean
             make -j regressiontest; make clean
-            make shortest; make clean
-            make cxxtest; make clean
+            make shortest         ; make clean
+            make cxxtest          ; make clean
   # the second half of the jobs are in this test
   short-tests-1:
     docker:
@@ -86,10 +84,31 @@ workflows:
   commit:
     jobs:
       # Run the tests in parallel
-      - short-tests-0
-      - short-tests-1
-      - regression-test
-
+      - short-tests-0:
+          filters:
+            tags:
+              only: /.*/
+      - short-tests-1:
+          filters:
+            tags:
+              only: /.*/
+      # Create a branch called regression and set it to dev to force a
+      # regression test run
+      - regression-test:
+          filters:
+            branches:
+              only:
+                - regression
+      # Only run on release tags.
+      - publish-github-release:
+          requires:
+            - short-tests-0
+            - short-tests-1
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
   nightly:
     triggers:
       - schedule:
@@ -101,7 +120,7 @@ workflows:
                 - dev
                 - master
     jobs:
-      # Run daily regression tests
+      # Run daily long regression tests
       - regression-test
 
 

.github/workflows/dev-long-tests.yml

@@ -1,240 +0,0 @@
-name: dev-long-tests
-# Tests longer than 10mn
-
-concurrency:
-  group: long-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  make-all:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make all
-      run: make all
-
-  # lasts ~24mn
-  make-test:
-    runs-on: ubuntu-latest
-    env:
-      DEVNULLRIGHTS: 1
-      READFROMBLOCKDEVICE: 1
-    steps:
-    - uses: actions/checkout@v2
-    - name: make test
-      run: make test
-
-  # lasts ~26mn
-  make-test-osx:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: OS-X test
-      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
-
-  no-intrinsics-fuzztest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: no intrinsics fuzztest
-      run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
-
-  tsan-zstreamtest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: thread sanitizer zstreamtest
-      run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
-
-  # lasts ~15mn
-  tsan-fuzztest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: thread sanitizer fuzztest
-      run: CC=clang make tsan-fuzztest
-
-  # lasts ~23mn
-  gcc-8-asan-ubsan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Test Zstd
-      run: |
-        sudo apt-get -qqq update
-        make gcc8install
-        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
-
-  gcc-asan-ubsan-testzstd-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Test Zstd, 32bit mode
-      run: |
-        make libc6install
-        make -j uasan-test-zstd32 V=1
-
-    # Note : external libraries must be turned off when using MSAN tests,
-    # because they are not msan-instrumented,
-    # so any data coming from these libraries is always considered "uninitialized"
-
-  gcc-8-asan-ubsan-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Fuzz Test
-      run: |
-        make gcc8install
-        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
-
-  gcc-asan-ubsan-fuzz32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Fuzz Test 32bit
-      run: |
-        make libc6install
-        CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
-
-  asan-ubsan-regression:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Regression Test
-      run: make -j uasanregressiontest
-
-  msan-regression:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: MSan + Regression Test
-      run: make -j msanregressiontest
-
-  clang-msan-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang + MSan + Fuzz Test
-      run: |
-        sudo apt-get -qqq update
-        sudo apt-get install clang
-        CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
-
-  # lasts ~24mn
-  clang-msan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang + MSan + Test Zstd
-      run: |
-        sudo apt-get update
-        sudo apt-get install clang
-        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
-
-  armfuzz:
-      runs-on: ubuntu-latest
-      steps:
-      - uses: actions/checkout@v2
-      - name: Qemu ARM emulation + Fuzz Test
-        run: |
-          sudo apt-get -qqq update
-          make arminstall
-          make armfuzz
-
-  valgrind-fuzz-test:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: valgrind + fuzz test stack mode    # ~ 7mn
-      shell: 'script -q -e -c "bash {0}"'
-      run: |
-        make valgrindinstall
-        make -C tests valgrindTest
-        make clean
-        make -C tests test-fuzzer-stackmode
-
-  mingw-long-test:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
-          { compiler: gcc, platform: x64, action: test, script: ""},
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Mingw long test
-      run: |
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        IF ("${{matrix.action}}" -eq "build")
-        {
-          make -v
-          sh -c "${{matrix.compiler}} -v"
-          ECHO "Building zlib to static link"
-          $env:CC = "${{matrix.compiler}}"
-          sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
-          sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
-          ECHO "Building zstd"
-          $env:CPPFLAGS = "-I../../zlib"
-          $env:LDFLAGS = "../../zlib/libz.a"
-          sh -c "${{matrix.script}}"
-        }
-        ELSEIF ("${{matrix.action}}" -eq "test")
-        {
-            ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-            $env:CC = "gcc"
-            $env:CXX = "g++"
-            MKDIR build\cmake\build
-            CD build\cmake\build
-            $env:FUZZERTEST = "-T2mn"
-            $env:ZSTREAM_TESTTIME = "-T2mn"
-            cmake -G "Visual Studio 14 2015 Win64" ..
-            cd ..\..\..
-            make clean
-        }
-
-  # lasts ~20mn
-  oss-fuzz:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        sanitizer: [address, undefined, memory]
-    steps:
-    - name: Build Fuzzers (${{ matrix.sanitizer }})
-      id: build
-      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Run Fuzzers (${{ matrix.sanitizer }})
-      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        fuzz-seconds: 600
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Upload Crash
-      uses: actions/upload-artifact@v1
-      if: failure() && steps.build.outcome == 'success'
-      with:
-        name: ${{ matrix.sanitizer }}-artifacts
-        path: ./out/artifacts

.github/workflows/dev-short-tests.yml

@@ -1,427 +0,0 @@
-name: dev-short-tests
-# Faster tests: mostly build tests, along with some other
-# misc tests
-
-concurrency:
-  group: fast-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  linux-kernel:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: linux kernel, library + build + test
-      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
-
-  benchmarking:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make benchmarking
-      run: make benchmarking
-
-  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make check on 32-bit
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
-
-  check-x32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
-      env:
-        CHECK_CONSTRAINED_MEM: true
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
-
-  gcc-7-libzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-7 + libzstdmt compilation
-      run: |
-        sudo apt-get -qqq update
-        make gcc7install
-        CC=gcc-7 CFLAGS=-Werror make -j all
-        make clean
-        LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
-
-    # candidate test (to check) : underlink test
-    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
-
-  cmake-build-and-test-check:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: cmake build and test check
-      run: |
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-        cp -r ./ "../zstd source"
-        cd "../zstd source"
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-
-  cpp-gnu90-c99-compatibility:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: C++, gnu90 and c99 compatibility
-      run: |
-        make cxxtest
-        make clean
-        make gnu90build
-        make clean
-        make c99build
-        make clean
-        make travis-install   # just ensures `make install` works
-
-  mingw-cross-compilation:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: mingw cross-compilation
-      run: |
-        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
-        sudo apt-get -qqq update
-        sudo apt-get install gcc-mingw-w64
-        CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
-
-  armbuild:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ARM Build Test
-      run: |
-        sudo apt-get -qqq update
-        make arminstall
-        make armbuild
-
-  bourne-shell:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Bourne shell compatibility (shellcheck)
-      run: |
-        wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
-        tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
-        shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
-
-  zlib-wrapper:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: zlib wrapper test
-      run: |
-        sudo apt-get -qqq update
-        make valgrindinstall
-        make -C zlibWrapper test
-        make -C zlibWrapper valgrindTest
-
-  lz4-threadpool-libs:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: LZ4, thread pool, and libs build testslib wrapper test
-      run: |
-        make lz4install
-        make -C tests test-lz4
-        make check < /dev/null | tee   # mess with lz4 console detection
-        make clean
-        make -C tests test-pool
-        make clean
-        bash tests/libzstd_builds.sh
-
-  gcc-make-tests-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Make all, 32bit mode
-      run: |
-        sudo apt-get -qqq update
-        make libc6install
-        CFLAGS="-Werror -m32" make -j all32
-
-  gcc-8-make:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: gcc-8 build
-        run: |
-          sudo apt-get -qqq update
-          make gcc8install
-          CC=gcc-8 CFLAGS="-Werror" make -j all
-
-  implicit-fall-through:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: -Wimplicit-fallthrough build
-        run: |
-          make clean
-          CC=gcc MOREFLAGS="-Werror -Wimplicit-fallthrough=2 -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
-          make clean
-          CC=clang MOREFLAGS="-Werror -Wimplicit-fallthrough -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
-
-  cmake-visual-2019:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        include:
-          - generator: "Visual Studio 16 2019"
-            flags: "-A x64"
-          - generator: "Visual Studio 16 2019"
-            flags: "-A Win32"
-          - generator: "MinGW Makefiles"
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      run: |
-        cd build\cmake
-        mkdir build
-        cd build
-        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
-        cmake.exe --build .
-
-  visual-2019:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Debug, Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-
-  visual-2015:
-    # only GH actions windows-2016 contains VS 2015
-    runs-on: windows-2016
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Debug, Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-
-  minimal-decompressor-macros:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: minimal decompressor macros
-      run: |
-        make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-        make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-
-  dynamic-bmi2:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: dynamic bmi2 tests
-      run: |
-        make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1 -mbmi2"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0 -mbmi2"
-
-  test-variants:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make all variants & validate
-      run: |
-        make -j -C programs allVariants MOREFLAGS=-O0
-        ./tests/test-variants.sh
-
-
-  qemu-consistency:
-    name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
-      matrix:
-        include: [
-          { name: ARM,      xcc_pkg: gcc-arm-linux-gnueabi,     xcc: arm-linux-gnueabi-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-arm-static     },
-          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-aarch64-static },
-          { name: PPC,      xcc_pkg: gcc-powerpc-linux-gnu,     xcc: powerpc-linux-gnu-gcc,     xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc-static     },
-          { name: PPC64LE,  xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc64le-static },
-          { name: S390X,    xcc_pkg: gcc-s390x-linux-gnu,       xcc: s390x-linux-gnu-gcc,       xemu_pkg: qemu-system-s390x,  xemu: qemu-s390x-static   },
-          { name: MIPS,     xcc_pkg: gcc-mips-linux-gnu,        xcc: mips-linux-gnu-gcc,        xemu_pkg: qemu-system-mips,   xemu: qemu-mips-static    },
-          { name: M68K,     xcc_pkg: gcc-m68k-linux-gnu,        xcc: m68k-linux-gnu-gcc,        xemu_pkg: qemu-system-m68k,   xemu: qemu-m68k-static    },
-        ]
-    env:                        # Set environment variables
-      XCC: ${{ matrix.xcc }}
-      XEMU: ${{ matrix.xemu }}
-    steps:
-    - uses: actions/checkout@v2 # https://github.com/actions/checkout
-    - name: apt update & install
-      run: |
-        sudo apt-get update
-        sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
-        sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
-    - name: Environment info
-      run: |
-        echo && which $XCC
-        echo && $XCC --version
-        echo && $XCC -v  # Show built-in specs
-        echo && which $XEMU
-        echo && $XEMU --version
-    - name: ARM
-      if: ${{ matrix.name == 'ARM' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: ARM64
-      if: ${{ matrix.name == 'ARM64' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: PPC
-      if: ${{ matrix.name == 'PPC' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: PPC64LE
-      if: ${{ matrix.name == 'PPC64LE' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: S390X
-      if: ${{ matrix.name == 'S390X' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: MIPS
-      if: ${{ matrix.name == 'MIPS' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: M68K
-      if: ${{ matrix.name == 'M68K' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-
-  mingw-short-test:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: gcc, platform: x64, script: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"},
-          { compiler: gcc, platform: x86, script: "CFLAGS=-Werror make -j allzstd"},
-          { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Mingw short test
-      run: |
-        ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        make -v
-        sh -c "${{matrix.compiler}} -v"
-        $env:CC = "${{matrix.compiler}}"
-        sh -c "${{matrix.script}}"
-        ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-        make clean
-        make check
-
-
-  visual-runtime-tests:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build and run tests
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      env:
-        ZSTD_BIN: ./zstd.exe
-        DATAGEN_BIN: ./datagen.exe
-      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      run: |
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142 /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-        COPY build\VS2010\bin\${{matrix.platform}}_${{matrix.configuration}}\*.exe tests\
-        CD tests
-        sh -e playTests.sh
-        DIR
-        .\fuzzer.exe -T2m
-
-# This test currently fails on Github Actions specifically.
-# Possible reason : TTY emulation.
-# Note that the same test works fine locally and on travisCI.
-# This will have to be fixed before transferring the test to GA.
-#  versions-compatibility:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v2
-#    - name: Versions Compatibility Test
-#      run: |
-#        make -C tests versionsTest
-
-
-# For reference : icc tests
-# icc tests are currently failing on Github Actions, likely to issues during installation stage
-# To be fixed later
-#
-#  icc:
-#    name: icc-check
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: install icc
-#      run: |
-#        export DEBIAN_FRONTEND=noninteractive
-#        sudo apt-get -qqq update
-#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-#        sudo apt-get update
-#        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v2
-#    - name: make check
-#      run: |
-#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

.github/workflows/generic-dev.yml

@@ -0,0 +1,214 @@
+name: generic-dev
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+
+jobs:
+
+# Dev PR jobs that still have to be migrated from travis
+#
+# versionTag (only on release tags)
+# valgrindTest (keeps failing for some reason. need investigation)
+# staticAnalyze (need trusty so need self-hosted)
+# pcc-fuzz: (need trusty so need self-hosted)
+# min-decomp-macros (flakey)
+#
+# setting up self-hosted is pretty straightforward, but
+# I need admins permissions to the repo for that it looks like
+# So I'm tabling that for now
+#
+# The release branch exclusive jobs will be in a separate
+# workflow file (the osx tests and meson build that is)
+
+  benchmarking:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make benchmarking
+      run: make benchmarking
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make test
+      run: make test
+
+  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make check on 32-bit
+      run: |
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
+        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
+
+  gcc-7-libzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-7 + libzstdmt compilation
+      run: |
+        make gcc7install
+        CC=gcc-7 CFLAGS=-Werror make -j all
+        make clean
+        LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
+        make -C tests zbufftest-dll
+
+    # candidate test (to check) : underlink test
+    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
+    # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
+
+  gcc-8-asan-ubsan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-8 + ASan + UBSan + Test Zstd
+      run: |
+        make gcc8install
+        CC=gcc-8 CFLAGS="-Werror" make -j all
+        make clean
+        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
+
+  gcc-asan-ubsan-testzstd-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + Test Zstd, 32bit mode
+      run: |
+        make libc6install
+        CFLAGS="-Werror -m32" make -j all32
+        make clean
+        make -j uasan-test-zstd32 V=1
+
+  clang-msan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + MSan + Test Zstd
+      run: |
+        sudo apt-get update
+        sudo apt-get install clang
+        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
+
+    # Note : external libraries must be turned off when using MSAN tests,
+    # because they are not msan-instrumented,
+    # so any data coming from these libraries is always considered "uninitialized"
+
+  cmake-build-and-test-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: cmake build and test check
+      run: |
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+        cp -r ./ "../zstd source"
+        cd "../zstd source"
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+
+  gcc-8-asan-ubsan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-8 + ASan + UBSan + Fuzz Test
+      run: |
+        make gcc8install
+        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
+  gcc-asan-ubsan-fuzz32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + Fuzz Test 32bit
+      run: |
+        make libc6install
+        CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
+  asan-ubsan-msan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + MSan + Regression Test
+      run: |
+        make -j uasanregressiontest
+        make clean
+        make -j msanregressiontest
+
+  cpp-gnu90-c99-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: C++, gnu90 and c99 compatibility
+      run: |
+        make cxxtest
+        make clean
+        make gnu90build
+        make clean
+        make c99build
+        make clean
+        make travis-install   # just ensures `make install` works
+
+  mingw-cross-compilation:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: mingw cross-compilation
+      run: |
+        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
+        sudo apt-get install gcc-mingw-w64
+        CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
+
+# TODO: Broken test - fix and uncomment
+#  armbuild:
+#    runs-on: ubuntu-16.04 # doesn't work on latest
+#    steps:
+#    - uses: actions/checkout@v2
+#    - name: ARM Build Test
+#      run: |
+#        make arminstall
+#        make armbuild
+
+# TODO: Broken test - fix and uncomment
+#  armfuzz:
+#    runs-on: ubuntu-16.04 # doesn't work on latest
+#    steps:
+#    - uses: actions/checkout@v2
+#    - name: Qemu ARM emulation + Fuzz Test
+#      run: |
+#        make arminstall
+#        make armfuzz
+
+  bourne-shell:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bourne shell compatibility (shellcheck)
+      run: |
+        wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
+        tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
+        shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
+
+# For reference : icc tests
+# icc tests are currently failing on Github Actions, likely to issues during installation stage
+# To be fixed later
+#
+#  icc:
+#    name: icc-check
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: install icc
+#      run: |
+#        export DEBIAN_FRONTEND=noninteractive
+#        sudo apt-get -qqq update
+#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
+#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+#        sudo apt-get update
+#        sudo apt-get install -y intel-basekit intel-hpckit
+#    - uses: actions/checkout@v2
+#    - name: make check
+#      run: |
+#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

.github/workflows/generic-release.yml

@@ -0,0 +1,67 @@
+name: generic-release
+
+on:
+  pull_request:
+    # This will eventually only be for pushes to release
+    # but for dogfooding purposes, I'm running it even
+    # on dev pushes
+    branches: [ dev, release, actionsTest ]
+
+jobs:
+  # missing jobs
+  #
+  # ppc64le + fuzz test
+  # Qemu PPC64 + Fuzz test
+  # Qemu aarch64 + Fuzz Test (on Xenial)
+  # versions comp
+  # meson test
+
+  osx:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/checkout@v2
+    - name: OS-X
+      run: |
+        make test
+        # make -c lib all (need to fix. not working right now)
+
+  zbuff:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: zbuff test
+      run: |
+        make -C tests test-zbuff
+
+  tsan:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: thread sanitizer
+      run: |
+        CC=clang make tsan-test-zstream
+        CC=clang make tsan-fuzztest
+
+  zlib-wrapper:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: zlib wrapper test
+      run: |
+        make valgrindinstall
+        make -C zlibWrapper test
+        make -C zlibWrapper valgrindTest
+
+  lz4-threadpool-partial-libs:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: LZ4, thread pool, and partial libs testslib wrapper test
+      run: |
+        make lz4install
+        make -C tests test-lz4
+        make check < /dev/null | tee   # mess with lz4 console detection
+        make clean
+        make -C tests test-pool
+        make clean
+        bash tests/libzstd_partial_builds.sh

.github/workflows/linux-kernel.yml

@@ -0,0 +1,13 @@
+name: linux-kernel
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Create linux kernel library + build + test
+      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"

.github/workflows/main.yml

@@ -0,0 +1,30 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address, undefined]
+    steps:
+    - name: Build Fuzzers (${{ matrix.sanitizer }})
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'zstd'
+        dry-run: false
+        sanitizer: ${{ matrix.sanitizer }}
+    - name: Run Fuzzers (${{ matrix.sanitizer }})
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'zstd'
+        fuzz-seconds: 600
+        dry-run: false
+        sanitizer: ${{ matrix.sanitizer }}
+    - name: Upload Crash
+      uses: actions/upload-artifact@v1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: ${{ matrix.sanitizer }}-artifacts
+        path: ./out/artifacts

.gitignore

@@ -29,7 +29,6 @@ dictionary
 NUL
 
 # Build artefacts
-contrib/linux-kernel/linux/
 projects/
 bin/
 .buckd/
@@ -51,5 +50,3 @@ googletest/
 *.code-workspace
 compile_commands.json
 .clangd
-perf.data
-perf.data.old

.travis.yml

@@ -1,6 +1,5 @@
-# Travis CI is used to test platforms that github-actions currently doesn't support
-# without either self-hosting or some finnicky work-around. Also, some tests
-# are troublesome to migrate since GH Actions runs tests not in a tty.
+# Medium Tests: Run on all commits/PRs to dev branch
+
 language: c
 
 git:
@@ -19,48 +18,47 @@ addons:
 
 env:
   global:
-    - FUZZERTEST=-T1mn
-      ZSTREAM_TESTTIME=-T1mn
+    - FUZZERTEST=-T2mn
+      ZSTREAM_TESTTIME=-T2mn
       DECODECORPUS_TESTTIME=-T1mn
 
+
 matrix:
   fast_finish: true
   include:
-    - name: S390X (big endian) + Fuzz test
-      dist: trusty
-      arch: s390x
-      script:
-        - FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
-    - name: S390X (big endian) + Fuzz test + no intrinsics
-      dist: trusty
-      arch: s390x
-      script:
-        - MOREFLAGS="-DZSTD_NO_INTRINSICS" FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
     - name: arm64    # ~2.5 mn
       os: linux
       arch: arm64
       script:
         - make check
 
-    - name: arm64fuzz
-      os: linux
-      arch: arm64
+    - name: Minimal Decompressor Macros    # ~5mn
       script:
-        - make -C tests fuzztest
+        - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
+        - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
+        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+        - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
 
-    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
-    - name: static analyzer scanbuild    # ~8mn
+    - name: static analyzer scanbuild    # ~26mn
       dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
       script:
         - make staticAnalyze
 
-    # GH actions can't run this command on OS-X, non-tty issues
-    - name: OS-X make all lib
-      os: osx
+    - name: Valgrind + Fuzz Test Stack Mode    # ~ 7mn
       script:
-        - make -C lib all
+        - make valgrindinstall
+        - make -C tests clean valgrindTest
+        - make clean
+        - make -C tests test-fuzzer-stackmode
+
+    - name: Qemu ARM emulation + Fuzz Test    # ~13.5mn
+      script:
+        - make arminstall
+        - make armfuzz
 
     # Introduced to check compat with old toolchains, to prevent e.g. #1872
     - name: ARM Build Test (on Trusty)
@@ -69,31 +67,65 @@ matrix:
         - make arminstall
         - make armbuild
 
-    # check release number (release/new tag only)
+    - name: Qemu PPC + Fuzz Test    # ~13mn
+      dist: trusty   # it seems ppc cross-compilation fails on "current"
+      script:
+        - make ppcinstall
+        - make ppcfuzz
+
+    # check release number (release only)
     - name: Tag-Specific Test
       if: tag =~ ^v[0-9]\.[0-9]
       script:
         - make -C tests checkTag
         - tests/checkTag "$TRAVIS_BRANCH"
 
-    - name: PPC64LE + Fuzz test  # ~13mn
-      arch: ppc64le
-      env:
-        - FUZZER_FLAGS=--no-big-tests
-        - MOREFLAGS="-static"
+    # tests for release branch and cron job only
+    - name: OS-X    # ~13mn
+      if: branch = release
+      os: osx
       script:
-        - cat /proc/cpuinfo
-        - make -C tests fuzztest
+        - make test
+        - make -C lib all
 
-    # This test currently fails on GA specifically, for no obvious reason
-    # (it works fine on travisCI, and on local test platforms).
-    - name: Versions Compatibility Test   # ~6mn
+    - name: Versions Compatibility Test   # 11.5mn
+      if: branch = release
       script:
         - make -C tests versionsTest
 
+    - name: thread sanitizer   # ~29mn
+      if: branch = release
+      script:
+        - make clang38install
+        - CC=clang-3.8 make tsan-test-zstream
+        - CC=clang-3.8 make tsan-fuzztest
+
+    - name: PPC64LE + Fuzz test  # ~13mn
+      if: branch = release
+      arch: ppc64le
+      script:
+        - cat /proc/cpuinfo
+        - make test
+
+    - name: Qemu PPC64 + Fuzz test  # ~13mn, presumed Big-Endian (?)
+      dist: trusty  # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
+      if: branch = release
+      script:
+        - make ppcinstall
+        - make ppc64fuzz
+
+    # note : we already have aarch64 tests on hardware
+    - name: Qemu aarch64 + Fuzz Test (on Xenial)    # ~14mn
+      if: branch = release
+      dist: xenial
+      script:
+        - make arminstall
+        - make aarch64fuzz
+
     # meson dedicated test
     - name: Xenial (Meson + clang)    # ~15mn
-      dist: bionic
+      if: branch = release
+      dist: xenial
       language: cpp
       compiler: clang
       install:

CHANGELOG

@@ -1,77 +1,3 @@
-v1.5.1 (Dec, 2021)
-perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42
-perf: faster huffman decoder, using x64 assembly, by @terrelln
-perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte
-perf: improved binary size and faster compilation times, by @terrelln
-perf: new row64 mode, used notably in level 12, by @senhuang42
-perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42
-perf: minor compression ratio improvements for small data at high levels, by @cyan4973
-perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln
-perf: faster compression speed on incompressible data, by @bindhvo
-perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo
-build: allows hiding static symbols in the dynamic library, using build macro, by @skitt
-build: support for m68k (Motorola 68000's), by @cyan4973
-build: improved AIX support, by @Helflym
-build: improved meson unofficial build, by @eli-schwartz
-cli : custom memory limit when training dictionary (#2925), by @embg
-cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB
-
-v1.5.0  (May 11, 2021)
-api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
-  `ZSTD_defaultCLevel()`
-  `ZSTD_getDictID_fromCDict()`
-api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42)
-  `ZSTD_compress_advanced()`
-  `ZSTD_compress_usingCDict_advanced()`
-  `ZSTD_compressBegin_advanced()`
-  `ZSTD_compressBegin_usingCDict_advanced()`
-  `ZSTD_initCStream_srcSize()`
-  `ZSTD_initCStream_usingDict()`
-  `ZSTD_initCStream_usingCDict()`
-  `ZSTD_initCStream_advanced()`
-  `ZSTD_initCStream_usingCDict_advanced()`
-  `ZSTD_resetCStream()`
-api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973)
-perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln)
-perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42)
-perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973)
-perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln)
-bug: Improve setting permissions of created files (#2525, @felixhandte)
-bug: Fix large dictionary non-determinism (#2607, @terrelln)
-bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln)
-bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte)
-bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42)
-bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink)
-bug: Fix UBSAN error in decompression (#2625, @terrelln)
-bug: Fix superblock compression divide by zero bug (#2592, @senhuang42)
-bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone)
-doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln)
-doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize)
-doc: Add style guide docs for open source contributors (#2626, @Cyan4973)
-tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42)
-tests: Better test coverage of index reduction (#2603, @terrelln)
-tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42)
-tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42)
-build: Dynamic library built multithreaded by default (#2584, @senhuang42)
-build: Move  `zstd_errors.h`  and  `zdict.h`  to  `lib/`  root (#2597, @terrelln)
-build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973)
-build: Single file library build script moved to `build/` directory (#2618, @felixhandte)
-build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42)
-build: Fixed Meson build (#2548, @SupervisedThinking @kloczek)
-build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson)
-build: Detect presence of `md5` on Darwin (#2609, @felixhandte)
-build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann)
-cli: `--progress` flag added to always display progress bar (#2595, @senhuang42)
-cli: Allow reading from block devices with `--force` (#2613, @felixhandte)
-cli: Fix CLI filesize display bug (#2550, @Cyan4973)
-cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973)
-contrib: Various fixes for linux kernel patch (#2539, @terrelln)
-contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42)
-contrib: Seekable format - New seek table-only API  (#2113 #2518, @mdittmer @Cyan4973)
-contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng)
-contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat)
-misc: Automatically published release tarballs available on Github (#2535, @felixhandte)
-
 v1.4.9  (Mar 1, 2021)
 bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
 bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)

CONTRIBUTING.md

@@ -47,7 +47,7 @@ Our contribution process works in three main stages:
     * Topic and development:
         * Make a new branch on your fork about the topic you're developing for
         ```
-        # branch names should be concise but sufficiently informative
+        # branch names should be consise but sufficiently informative
         git checkout -b <branch-name>
         git push origin <branch-name>
         ```
@@ -104,7 +104,7 @@ Our contribution process works in three main stages:
         issue at hand, then please indicate this by requesting that an issue be closed by commenting.
         * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
         that the change must make it to an official zstd release for it to be meaningful. We recommend
-        that contributors track the activity on their pull request and corresponding issue(s) page(s) until
+        that contributers track the activity on their pull request and corresponding issue(s) page(s) until
         their change makes it to the next release of zstd. Users will often discover bugs in your code or
         suggest ways to refine and improve your initial changes even after the pull request is merged.
 
@@ -270,15 +270,15 @@ for level 1 compression on Zstd. Typically this means, you have identified a sec
 code that you think can be made to run faster.
 
 The first thing you will want to do is make sure that the piece of code is actually taking up
-a notable amount of time to run. It is usually not worth optimizing something which accounts for less than
+a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
 0.0001% of the total running time. Luckily, there are tools to help with this.
 Profilers will let you see how much time your code spends inside a particular function.
-If your target code snippet is only part of a function, it might be worth trying to
-isolate that snippet by moving it to its own function (this is usually not necessary but
+If your target code snippit is only part of a function, it might be worth trying to
+isolate that snippit by moving it to its own function (this is usually not necessary but
 might be).
 
-Most profilers (including the profilers discussed below) will generate a call graph of
-functions for you. Your goal will be to find your function of interest in this call graph
+Most profilers (including the profilers dicusssed below) will generate a call graph of
+functions for you. Your goal will be to find your function of interest in this call grapch
 and then inspect the time spent inside of it. You might also want to to look at the
 annotated assembly which most profilers will provide you with.
 
@@ -301,16 +301,16 @@ $ zstd -b1 -i5 <my-data> # this will run for 5 seconds
 5. Once you run your benchmarking script, switch back over to instruments and attach your
 process to the time profiler. You can do this by:
     * Clicking on the `All Processes` drop down in the top left of the toolbar.
-    * Selecting your process from the dropdown. In my case, it is just going to be labeled
+    * Selecting your process from the dropdown. In my case, it is just going to be labled
     `zstd`
     * Hitting the bright red record circle button on the top left of the toolbar
-6. You profiler will now start collecting metrics from your benchmarking script. Once
+6. You profiler will now start collecting metrics from your bencharking script. Once
 you think you have collected enough samples (usually this is the case after 3 seconds of
 recording), stop your profiler.
 7. Make sure that in toolbar of the bottom window, `profile` is selected.
 8. You should be able to see your call graph.
     * If you don't see the call graph or an incomplete call graph, make sure you have compiled
-    zstd and your benchmarking script using debug flags. On mac and linux, this just means
+    zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
     you will have to supply the `-g` flag alone with your build script. You might also
     have to provide the `-fno-omit-frame-pointer` flag
 9. Dig down the graph to find your function call and then inspect it by double clicking
@@ -329,7 +329,7 @@ Some general notes on perf:
 counter statistics. Perf uses a high resolution timer and this is likely one
 of the first things your team will run when assessing your PR.
 * Perf has a long list of hardware counters that can be viewed with `perf --list`.
-When measuring optimizations, something worth trying is to make sure the hardware
+When measuring optimizations, something worth trying is to make sure the handware
 counters you expect to be impacted by your change are in fact being so. For example,
 if you expect the L1 cache misses to decrease with your change, you can look at the
 counter `L1-dcache-load-misses`
@@ -368,7 +368,7 @@ Follow these steps to link travis-ci with your github fork of zstd
 TODO
 
 ### appveyor
-Follow these steps to link circle-ci with your github fork of zstd
+Follow these steps to link circle-ci with your girhub fork of zstd
 
 1. Make sure you are logged into your github account
 2. Go to https://www.appveyor.com/
@@ -399,105 +399,7 @@ disclosure of security bugs. In those cases, please go through the process
 outlined on that page and do not file a public issue.
 
 ## Coding Style
-It's a pretty long topic, which is difficult to summarize in a single paragraph.
-As a rule of thumbs, try to imitate the coding style of
-similar lines of codes around your contribution.
-The following is a non-exhaustive list of rules employed in zstd code base:
-
-### C90
-This code base is following strict C90 standard,
-with 2 extensions : 64-bit `long long` types, and variadic macros.
-This rule is applied strictly to code within `lib/` and `programs/`.
-Sub-project in `contrib/` are allowed to use other conventions.
-
-### C++ direct compatibility : symbol mangling
-All public symbol declarations must be wrapped in `extern “C” { … }`,
-so that this project can be compiled as C++98 code,
-and linked into C++ applications.
-
-### Minimal Frugal
-This design requirement is fundamental to preserve the portability of the code base.
-#### Dependencies
-- Reduce dependencies to the minimum possible level.
-  Any dependency should be considered “bad” by default,
-  and only tolerated because it provides a service in a better way than can be achieved locally.
-  The only external dependencies this repository tolerates are
-  standard C libraries, and in rare cases, system level headers.
-- Within `lib/`, this policy is even more drastic.
-  The only external dependencies allowed are `<assert.h>`, `<stdlib.h>`, `<string.h>`,
-  and even then, not directly.
-  In particular, no function shall ever allocate on heap directly,
-  and must use instead `ZSTD_malloc()` and equivalent.
-  Other accepted non-symbol headers are `<stddef.h>` and `<limits.h>`.
-- Within the project, there is a strict hierarchy of dependencies that must be respected.
-  `programs/` is allowed to depend on `lib/`, but only its public API.
-  Within `lib/`, `lib/common` doesn't depend on any other directory.
-  `lib/compress` and `lib/decompress` shall not depend on each other.
-  `lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`.
-#### Resources
-- Functions in `lib/` must use very little stack space,
-  several dozens of bytes max.
-  Everything larger must use the heap allocator,
-  or require a scratch buffer to be emplaced manually.
-
-### Naming
-* All public symbols are prefixed with `ZSTD_`
-  + private symbols, with a scope limited to their own unit, are free of this restriction.
-    However, since `libzstd` source code can be amalgamated,
-    each symbol name must attempt to be (and remain) unique.
-    Avoid too generic names that could become ground for future collisions.
-    This generally implies usage of some form of prefix.
-* For symbols (functions and variables), naming convention is `PREFIX_camelCase`.
-  + In some advanced cases, one can also find :
-    - `PREFIX_prefix2_camelCase`
-    - `PREFIX_camelCase_extendedQualifier`
-* Multi-words names generally consist of an action followed by object:
-  - for example : `ZSTD_createCCtx()`
-* Prefer positive actions
-  - `goBackward` rather than `notGoForward`
-* Type names (`struct`, etc.) follow similar convention,
-  except that they are allowed and even invited to start by an Uppercase letter.
-  Example : `ZSTD_CCtx`, `ZSTD_CDict`
-* Macro names are all Capital letters.
-  The same composition rules (`PREFIX_NAME_QUALIFIER`) apply.
-* File names are all lowercase letters.
-  The convention is `snake_case`.
-  File names **must** be unique across the entire code base,
-  even when they stand in clearly separated directories.
-
-### Qualifiers
-* This code base is `const` friendly, if not `const` fanatical.
-  Any variable that can be `const` (aka. read-only) **must** be `const`.
-  Any pointer which content will not be modified must be `const`.
-  This property is then controlled at compiler level.
-  `const` variables are an important signal to readers that this variable isn’t modified.
-  Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function.
-* If a function must be inlined, mention it explicitly,
-  using project's own portable macros, such as `FORCE_INLINE_ATTR`,
-  defined in `lib/common/compiler.h`.
-
-### Debugging
-* **Assertions** are welcome, and should be used very liberally,
-  to control any condition the code expects for its correct execution.
-  These assertion checks will be run in debug builds, and disabled in production.
-* For traces, this project provides its own debug macros,
-  in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`.
-
-### Code documentation
-* Avoid code documentation that merely repeats what the code is already stating.
-  Whenever applicable, prefer employing the code as the primary way to convey explanations.
-  Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`.
-  Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`.
-* At declaration level, the documentation explains how to use the function or variable
-  and when applicable why it's needed, of the scenarios where it can be useful.
-* At implementation level, the documentation explains the general outline of the algorithm employed,
-  and when applicable why this specific choice was preferred.
-
-### General layout
 * 4 spaces for indentation rather than tabs
-* Code documentation shall directly precede function declaration or implementation
-* Function implementations and its code documentation should be preceded and followed by an empty line
-
 
 ## License
 By contributing to Zstandard, you agree that your contributions will be licensed

Makefile

@@ -29,7 +29,6 @@ VOID = /dev/null
 # fail on other tested distros (ubuntu, debian) even
 # without manually specifying the TARGET_SYSTEM.
 TARGET_SYSTEM ?= $(OS)
-CP ?= cp
 
 ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
   EXT =.exe
@@ -58,8 +57,8 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release lib-mt lib-nomt
-lib lib-release lib-mt lib-nomt:
+.PHONY: lib lib-release
+lib lib-release :
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
@@ -70,7 +69,7 @@ zstd zstd-release:
 .PHONY: zstdmt
 zstdmt:
 	$(Q)$(MAKE) -C $(PRGDIR) $@
-	$(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+	$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
 zlibwrapper: lib
@@ -123,8 +122,8 @@ contrib: lib
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/seekable_format/tests test
 	$(MAKE) -C contrib/largeNbDicts all
-	cd build/single_file_libs/ ; ./build_decoder_test.sh
-	cd build/single_file_libs/ ; ./build_library_test.sh
+	cd contrib/single_file_libs/ ; ./build_decoder_test.sh
+	cd contrib/single_file_libs/ ; ./build_library_test.sh
 
 .PHONY: cleanTabs
 cleanTabs:
@@ -149,11 +148,10 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
 
 HOST_OS = POSIX
-
-MKDIR ?= mkdir -p
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
 HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 EGREP_OPTIONS ?=
@@ -181,7 +179,7 @@ list:
 	    done \
 	} | column -t -s $$'\t'
 
-.PHONY: install armtest usan asan uasan msan asan32
+.PHONY: install armtest usan asan uasan
 install:
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 	$(Q)$(MAKE) -C $(PRGDIR) $@
@@ -195,19 +193,22 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
-.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
+.PHONY: gcc5build
 gcc5build: clean
 	gcc-5 -v
 	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: gcc6build
 gcc6build: clean
 	gcc-6 -v
 	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: gcc7build
 gcc7build: clean
 	gcc-7 -v
 	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: clangbuild
 clangbuild: clean
 	clang -v
 	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
@@ -220,7 +221,7 @@ armbuild: clean
 	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 aarch64build: clean
-	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 ppcbuild: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
@@ -228,7 +229,6 @@ ppcbuild: clean
 ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
 
-.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
@@ -242,7 +242,7 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
-.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
+.PHONY: cxxtest
 cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
 cxxtest: clean
 	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
@@ -271,7 +271,6 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
 
-.PHONY: arm-ppc-compilation
 arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
@@ -287,24 +286,9 @@ uasanregressiontest:
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
 
-update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
-update_regressionResults:
-	$(MAKE) -C programs zstd
-	$(MAKE) -C tests/regression test
-	$(RM) -rf $(REGRESS_RESULTS_DIR)
-	$(MKDIR) $(REGRESS_RESULTS_DIR)
-	./tests/regression/test                         \
-        --cache  tests/regression/cache             \
-        --output $(REGRESS_RESULTS_DIR)/results.csv \
-        --zstd   programs/zstd
-	echo "Showing results differences"
-	! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv
-	echo "Updating results.csv"
-	$(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv
-
-
 # run UBsan with -fsanitize-recover=pointer-overflow
 # this only works with recent compilers such as gcc 8+
+
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
 
@@ -332,16 +316,13 @@ uasan-%: clean
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
 
-.PHONY: apt-install
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
 
-.PHONY: apt-add-repo
 apt-add-repo:
 	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 	sudo apt-get update -y -qq
 
-.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
 ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
 
@@ -376,22 +357,20 @@ lz4install:
 endif
 
 
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
-
 ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
 CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
+
 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
-	$(MKDIR) $(BUILDIR)/cmake/build
+	mkdir $(BUILDIR)/cmake/build
 	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
 	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
 	$(MAKE) -C $(BUILDIR)/cmake/build install;
@@ -400,23 +379,23 @@ cmakebuild:
 
 c89build: clean
 	$(CC) -v
-	CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c89 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
 
 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
 
 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
 
 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
 
 bmix64build: clean
 	$(CC) -v
@@ -435,5 +414,5 @@ bmi32build: clean
 staticAnalyze: SCANBUILD ?= scan-build
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
 endif

Package.swift

@@ -1,36 +0,0 @@
-// swift-tools-version:5.0
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
-    name: "zstd",
-    platforms: [
-        .macOS(.v10_10), .iOS(.v9), .tvOS(.v9)
-    ],
-    products: [
-        // Products define the executables and libraries a package produces, and make them visible to other packages.
-        .library(
-            name: "libzstd",
-            targets: [ "libzstd" ])
-    ],
-    dependencies: [
-        // Dependencies declare other packages that this package depends on.
-        // .package(url: /* package url */, from: "1.0.0"),
-    ],
-    targets: [
-        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
-        // Targets can depend on other targets in this package, and on products in packages this package depends on.
-        .target(
-            name: "libzstd",
-            path: "lib",
-            sources: [ "common", "compress", "decompress", "dictBuilder" ],
-            publicHeadersPath: ".",
-            cSettings: [
-                .headerSearchPath(".")
-            ])
-    ],
-    swiftLanguageVersions: [.v5],
-    cLanguageStandard: .gnu11,
-    cxxLanguageStandard: .gnucxx14
-)

README.md

@@ -4,8 +4,7 @@ __Zstandard__, or `zstd` as short version, is a fast lossless compression algori
 targeting real-time compression scenarios at zlib-level and better compression ratios.
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
 
-Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
-This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
+The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
@@ -18,8 +17,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 [![Build status][CirrusDevBadge]][CirrusLink]
 [![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
 
-[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
-[travisLink]: https://travis-ci.com/facebook/zstd
+[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
+[travisLink]: https://travis-ci.org/facebook/zstd
 [AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
@@ -32,8 +31,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 ## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
-with a Core i7-9700K CPU @ 4.9GHz,
+on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
+with a Core i9-9900K CPU @ 5.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
 compiled with [gcc] 9.3.0,
 on the [Silesia compression corpus].
@@ -44,24 +43,25 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
-| [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
-| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
-| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
-| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
-| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
-| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
-| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
+| **zstd 1.4.5 -1**       | 2.884 |   500 MB/s |  1660 MB/s |
+| zlib 1.2.11 -1          | 2.743 |    90 MB/s |   400 MB/s |
+| brotli 1.0.7 -0         | 2.703 |   400 MB/s |   450 MB/s |
+| **zstd 1.4.5 --fast=1** | 2.434 |   570 MB/s |  2200 MB/s |
+| **zstd 1.4.5 --fast=3** | 2.312 |   640 MB/s |  2300 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   560 MB/s |   710 MB/s |
+| **zstd 1.4.5 --fast=5** | 2.178 |   700 MB/s |  2420 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   690 MB/s |   820 MB/s |
+| lz4 1.9.2               | 2.101 |   740 MB/s |  4530 MB/s |
+| **zstd 1.4.5 --fast=7** | 2.096 |   750 MB/s |  2480 MB/s |
+| lzf 3.6 -1              | 2.077 |   410 MB/s |   860 MB/s |
+| snappy 1.1.8            | 2.073 |   560 MB/s |  1790 MB/s |
 
 [zlib]: http://www.zlib.net/
-[lz4]: http://www.lz4.org/
+[LZ4]: http://www.lz4.org/
 
 The negative compression levels, specified with `--fast=#`,
-offer faster compression and decompression speed
-at the cost of compression ratio (compared to level 1).
+offer faster compression and decompression speed in exchange for some loss in
+compression ratio compared to level 1, as seen in the table above.
 
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.

TESTING.md

@@ -40,4 +40,5 @@ They consist of the following tests:
 - Versions test (ensuring `zstd` can decode files from all previous versions)
 - `pzstd` with asan and tsan, as well as in 32-bits mode
 - Testing `zstd` with legacy mode off
+- Testing `zbuff` (old streaming API)
 - Entire test suite and make install on macOS

appveyor.yml

@@ -23,15 +23,34 @@
       SCRIPT:   "make allzstd MOREFLAGS=-static"
       ARTIFACT: "true"
       BUILD:    "true"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
+      BUILD:    "true"
 
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   ""
+      TEST:     "cmake"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
       PLATFORM: "x64"
       CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
 
   install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -85,14 +104,55 @@
           appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
       )
     )
-  - if [%HOST%]==[cmake-visual] (
+  - if [%HOST%]==[visual] (
       ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
+      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
+      ECHO *** &&
+      msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
 
   test_script:
@@ -108,6 +168,18 @@
       cd ..\..\.. &&
       make clean
     )
+  - SET "FUZZERTEST=-T30s"
+  - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
+      CD tests &&
+      SET ZSTD_BIN=./zstd.exe&&
+      SET DATAGEN_BIN=./datagen.exe&&
+      sh -e playTests.sh --test-large-data &&
+      fullbench.exe -i1 &&
+      fullbench.exe -i1 -P0 &&
+      fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
+    )
 
 
 # The following tests are for regular pushes
@@ -118,35 +190,38 @@
   version: 1.0.{build}
   environment:
     matrix:
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Release"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Release"
-
     - COMPILER: "gcc"
       HOST:     "cygwin"
       PLATFORM: "x64"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "CFLAGS=-Werror make -j allzstd"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
 
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
       PLATFORM: "x64"
       CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
 
   install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -154,10 +229,17 @@
   - if [%HOST%]==[cygwin] (
       ECHO Installing Cygwin Packages &&
       C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
+        gcc-g++,^
         gcc,^
         cmake,^
         make
     )
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+    )
   - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
       SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
     )
@@ -170,36 +252,41 @@
       C:\cygwin64\bin\bash --login -c "
         set -e;
         cd build/cmake;
-        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .;
-        make VERBOSE=1 -j;
+        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
+        make -j4;
         ctest -V -L Medium;
       "
     )
-  - if [%HOST%]==[cmake-visual] (
-      ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) ) &&
+      make -v &&
+      sh -c "%COMPILER% -v" &&
+      set "CC=%COMPILER%" &&
+      sh -c "%SCRIPT%"
     )
   - if [%HOST%]==[visual] (
       ECHO *** &&
-      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
 
 
   test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET "FUZZERTEST=-T10s"
   - if [%HOST%]==[mingw] (
       set "CC=%COMPILER%" &&
       make clean &&
       make check
-    )
+    )

build/VS2008/fullbench/fullbench.vcproj

@@ -463,7 +463,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File

build/VS2008/fuzzer/fuzzer.vcproj

@@ -483,7 +483,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -511,7 +511,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

build/VS2008/zstd/zstd.vcproj

@@ -559,7 +559,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File
@@ -575,7 +575,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File

build/VS2008/zstdlib/zstdlib.vcproj

@@ -495,7 +495,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -523,7 +523,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

build/VS2010/fullbench/fullbench.vcxproj

@@ -190,7 +190,7 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\common\pool.h" />
     <ClInclude Include="..\..\..\lib\common\threading.h" />

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -196,7 +196,7 @@
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
@@ -211,7 +211,7 @@
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
     <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
-    <ClInclude Include="..\..\..\lib\zdict.h" />
+    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -44,6 +44,9 @@
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -61,11 +64,12 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\mem.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
+    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />

build/VS2010/libzstd/libzstd.vcxproj

@@ -44,6 +44,9 @@
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -61,11 +64,12 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\mem.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
+    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />

build/VS2010/zstd/zstd.vcxproj

@@ -70,14 +70,14 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\zdict.h" />
+    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />

build/VS_scripts/build.generic.cmd

@@ -19,10 +19,10 @@ GOTO build
 :display_help
 
 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
+echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
-echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
+echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141)
 
 EXIT /B 1
 
@@ -43,16 +43,6 @@ IF %msbuild_version% == VS2017 (
 	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
 )
 
-:: VS2019
-SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2019 (
-	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
-	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
-	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
-)
-
 SET project="%~p0\..\VS2010\zstd.sln"
 
 SET msbuild_params=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%msbuild_platform% /p:Configuration=%msbuild_configuration%

build/cmake/CMakeLists.txt

@@ -92,14 +92,8 @@ else ()
     add_definitions(-DZSTD_LEGACY_SUPPORT=0)
 endif ()
 
-if (ANDROID)
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
-else()
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
-endif()
-
 # Multi-threading support
-option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
+option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
 
 if (ZSTD_MULTITHREAD_SUPPORT)
     message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -26,12 +26,7 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
         EnableCompilerFlag("-std=c++11" false true)
         #Set c99 by default
         EnableCompilerFlag("-std=c99" true false)
-        if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
-            # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true)
-        else ()
-            EnableCompilerFlag("-Wall" true true)
-        endif ()
+        EnableCompilerFlag("-Wall" true true)
         EnableCompilerFlag("-Wextra" true true)
         EnableCompilerFlag("-Wundef" true true)
         EnableCompilerFlag("-Wshadow" true true)

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -21,16 +21,10 @@ add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/O
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 
-if (ZSTD_BUILD_SHARED)
-    set(ZSTD_LIB libzstd_shared)
-else()
-    set(ZSTD_LIB libzstd_static)
-endif()
-
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 if (CMAKE_USE_PTHREADS_INIT)
-  target_link_libraries(pzstd ${ZSTD_LIB} ${CMAKE_THREAD_LIBS_INIT})
+    target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
 else()
     message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
 endif()

build/cmake/lib/CMakeLists.txt

@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(libzstd C ASM)
+project(libzstd C)
 
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
@@ -22,26 +22,30 @@ include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
 file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
 file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
-file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
+file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
 file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
+file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
 
 set(Sources
         ${CommonSources}
         ${CompressSources}
         ${DecompressSources}
-        ${DictBuilderSources})
+        ${DictBuilderSources}
+        ${DeprecatedSources})
 
 file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
 file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
 file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
 file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
+file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
 
 set(Headers
         ${LIBRARY_DIR}/zstd.h
         ${CommonHeaders}
         ${CompressHeaders}
         ${DecompressHeaders}
-        ${DictBuilderHeaders})
+        ${DictBuilderHeaders}
+        ${DeprecatedHeaders})
 
 if (ZSTD_LEGACY_SUPPORT)
     set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@@ -106,7 +110,7 @@ if (MSVC)
 endif ()
 
 # With MSVC static library needs to be renamed to avoid conflict with import library
-if (MSVC OR (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT MINGW))
+if (MSVC)
     set(STATIC_LIBRARY_BASE_NAME zstd_static)
 else ()
     set(STATIC_LIBRARY_BASE_NAME zstd)
@@ -158,8 +162,8 @@ endif ()
 # install target
 install(FILES
     "${LIBRARY_DIR}/zstd.h"
-    "${LIBRARY_DIR}/zdict.h"
-    "${LIBRARY_DIR}/zstd_errors.h"
+    "${LIBRARY_DIR}/dictBuilder/zdict.h"
+    "${LIBRARY_DIR}/common/zstd_errors.h"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
 install(TARGETS ${library_targets}
@@ -168,7 +172,6 @@ install(TARGETS ${library_targets}
     ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
     )
 
 # uninstall target

build/cmake/programs/CMakeLists.txt

@@ -37,9 +37,7 @@ target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
 endif ()
-install(TARGETS zstd
-  RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}")
+install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
 
 if (UNIX)
     add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")

build/cmake/tests/.gitignore

@@ -3,4 +3,5 @@ datagen
 fullbench
 fuzzer
 paramgrill
+zbufftest
 

build/cmake/tests/CMakeLists.txt

@@ -57,15 +57,13 @@ target_link_libraries(datagen libzstd_static)
 # fullbench
 #
 add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
-set_property(TARGET fullbench APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
+add_test(NAME fullbench COMMAND fullbench)
 
 #
 # fuzzer
 #
 add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
-set_property(TARGET fuzzer APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(fuzzer libzstd_static)
 AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
@@ -78,7 +76,6 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
 # zstreamtest
 #
 add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
-set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(zstreamtest libzstd_static)
 AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")

build/meson/contrib/pzstd/meson.build

@@ -18,7 +18,7 @@ pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
   join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
 pzstd = executable('pzstd',
   pzstd_sources,
-  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-Wno-deprecated-declarations' ],
+  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
   include_directories: pzstd_includes,
   dependencies: [ libzstd_dep, thread_dep ],
   install: true)

build/meson/lib/meson.build

@@ -14,13 +14,15 @@ libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
   join_paths(zstd_rootdir, 'lib/common'),
   join_paths(zstd_rootdir, 'lib/compress'),
   join_paths(zstd_rootdir, 'lib/decompress'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder'))]
+  join_paths(zstd_rootdir, 'lib/dictBuilder'),
+  join_paths(zstd_rootdir, 'lib/deprecated'))]
 
 libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
   join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
   join_paths(zstd_rootdir, 'lib/common/threading.c'),
   join_paths(zstd_rootdir, 'lib/common/pool.c'),
   join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_trace.c'),
   join_paths(zstd_rootdir, 'lib/common/error_private.c'),
   join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
   join_paths(zstd_rootdir, 'lib/compress/hist.c'),
@@ -43,16 +45,10 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
   join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
   join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
   join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')]
-
-# really we need anything that defines __GNUC__ as that is what ZSTD_ASM_SUPPORTED is gated on
-# but these are the two compilers that are supported in tree and actually handle this correctly
-# Otherwise, explicitly disable assmebly.
-if [compiler_gcc, compiler_clang].contains(cc_id)
-  libzstd_sources += join_paths(zstd_rootdir, 'lib/decompress/huf_decompress_amd64.S')
-else
-  add_project_arguments('-DZSTD_DISABLE_ASM', language: 'c')
-endif
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
 
 # Explicit define legacy support
 add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
@@ -117,7 +113,6 @@ libzstd = library('zstd',
   libzstd_sources,
   include_directories: libzstd_includes,
   c_args: libzstd_c_args,
-  gnu_symbol_visibility: 'hidden',
   dependencies: libzstd_deps,
   install: true,
   version: zstd_libversion)
@@ -133,5 +128,5 @@ pkgconfig.generate(libzstd,
   url: 'http://www.zstd.net/')
 
 install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
-  join_paths(zstd_rootdir, 'lib/zdict.h'),
-  join_paths(zstd_rootdir, 'lib/zstd_errors.h'))
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))

build/meson/meson.build

@@ -14,14 +14,10 @@ project('zstd',
   default_options : [
     'c_std=gnu99',
     'cpp_std=c++11',
-    'buildtype=release',
-    'warning_level=3',
-    # -Wdocumentation does not actually pass, nor do the test binaries,
-    # so this isn't safe
-    #'werror=true'
+    'buildtype=release'
   ],
   version: 'DUMMY',
-  meson_version: '>=0.48.0')
+  meson_version: '>=0.47.0')
 
 cc = meson.get_compiler('c')
 cxx = meson.get_compiler('cpp')
@@ -110,8 +106,10 @@ use_lz4 = lz4_dep.found()
 add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
 
 if [compiler_gcc, compiler_clang].contains(cc_id)
-  common_warning_flags = [ '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
+  common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
   if cc_id == compiler_clang
+    # Should use Meson's own --werror build option
+    #common_warning_flags += '-Werror'
     common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
   endif
   cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])

build/meson/meson_options.txt

@@ -10,7 +10,7 @@
 
 # Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
 
-option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
+option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
   description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
 option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
   description: 'Enable run-time debug. See lib/common/debug.h')

build/meson/programs/meson.build

@@ -18,9 +18,7 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/dibio.c'),
-  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c'),
-  # needed due to use of private symbol + -fvisibility=hidden
-  join_paths(zstd_rootdir, 'lib/common/xxhash.c')]
+  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
 
 zstd_c_args = libzstd_debug_cflags
 if use_multi_thread

build/meson/tests/meson.build

@@ -29,62 +29,64 @@ ZSTDRTTEST = ['--test-large-data']
 
 test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
 
-testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/benchfn.c'),
-  join_paths(zstd_rootdir, 'programs/benchzstd.c')]
-
-testcommon = static_library('testcommon',
-  testcommon_sources,
-  # needed due to use of private symbol + -fvisibility=hidden
-  objects: libzstd.extract_all_objects(recursive: false))
-
-testcommon_dep = declare_dependency(link_with: testcommon,
-  dependencies: libzstd_deps,
-  include_directories: libzstd_includes)
-
-datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
+datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'tests/datagencli.c')]
 datagen = executable('datagen',
   datagen_sources,
   c_args: [ '-DNDEBUG' ],
   include_directories: test_includes,
-  dependencies: testcommon_dep,
+  dependencies: libzstd_dep,
   install: false)
 
-fullbench_sources = [join_paths(zstd_rootdir, 'tests/fullbench.c')]
+fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'tests/fullbench.c')]
 fullbench = executable('fullbench',
   fullbench_sources,
   include_directories: test_includes,
-  dependencies: testcommon_dep,
+  dependencies: libzstd_dep,
   install: false)
 
-fuzzer_sources = [join_paths(zstd_rootdir, 'tests/fuzzer.c')]
+fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/fuzzer.c')]
 fuzzer = executable('fuzzer',
   fuzzer_sources,
   include_directories: test_includes,
-  dependencies: [ testcommon_dep, thread_dep ],
+  dependencies: libzstd_dep,
   install: false)
 
-zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'),
+zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/seqgen.c'),
   join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
 zstreamtest = executable('zstreamtest',
   zstreamtest_sources,
   include_directories: test_includes,
-  dependencies: testcommon_dep,
+  dependencies: libzstd_dep,
   install: false)
 
-paramgrill_sources = [join_paths(zstd_rootdir, 'tests/paramgrill.c')]
+paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/paramgrill.c')]
 paramgrill = executable('paramgrill',
   paramgrill_sources,
   include_directories: test_includes,
-  dependencies: [ testcommon_dep, libm_dep ],
+  dependencies: [ libzstd_dep, libm_dep ],
   install: false)
 
 roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
 roundTripCrash = executable('roundTripCrash',
   roundTripCrash_sources,
-  dependencies: [ testcommon_dep ],
+  dependencies: [ libzstd_dep ],
   install: false)
 
 longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
@@ -109,14 +111,18 @@ if 0 < legacy_level and legacy_level <= 4
     install: false)
 endif
 
-decodecorpus_sources = [join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
+decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
 decodecorpus = executable('decodecorpus',
   decodecorpus_sources,
   include_directories: test_includes,
-  dependencies: [ testcommon_dep, libm_dep ],
+  dependencies: [ libzstd_dep, libm_dep ],
   install: false)
 
-poolTests_sources = [join_paths(zstd_rootdir, 'tests/poolTests.c'),
+poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/poolTests.c'),
   join_paths(zstd_rootdir, 'lib/common/pool.c'),
   join_paths(zstd_rootdir, 'lib/common/threading.c'),
   join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
@@ -124,7 +130,7 @@ poolTests_sources = [join_paths(zstd_rootdir, 'tests/poolTests.c'),
 poolTests = executable('poolTests',
   poolTests_sources,
   include_directories: test_includes,
-  dependencies: [ testcommon_dep, thread_dep ],
+  dependencies: [ libzstd_dep, thread_dep ],
   install: false)
 
 checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
@@ -180,6 +186,10 @@ test('test-zstream-1',
   zstreamtest,
   args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
   timeout: 240)
+test('test-zstream-2',
+  zstreamtest,
+  args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
 test('test-zstream-3',
   zstreamtest,
   args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,

contrib/VS2005/fullbench/fullbench.vcproj

@@ -390,7 +390,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File

contrib/VS2005/fuzzer/fuzzer.vcproj

@@ -426,7 +426,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -454,7 +454,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/VS2005/zstd/zstd.vcproj

@@ -454,7 +454,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -482,7 +482,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/VS2005/zstdlib/zstdlib.vcproj

@@ -371,6 +371,18 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
 				>
@@ -446,7 +458,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -474,7 +486,11 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\common\zbuff.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/freestanding_lib/freestanding.py

@@ -27,6 +27,7 @@ SKIPPED_FILES = [
     "common/pool.h",
     "common/threading.c",
     "common/threading.h",
+    "common/zstd_trace.c",
     "common/zstd_trace.h",
     "compress/zstdmt_compress.h",
     "compress/zstdmt_compress.c",
@@ -460,8 +461,7 @@ class Freestanding(object):
         print(*args, **kwargs)
 
     def _copy_file(self, lib_path):
-        suffixes = [".c", ".h", ".S"]
-        if not any((lib_path.endswith(suffix) for suffix in suffixes)):
+        if not (lib_path.endswith(".c") or lib_path.endswith(".h")):
             return
         if lib_path in SKIPPED_FILES:
             self._log(f"\tSkipping file: {lib_path}")
@@ -481,7 +481,6 @@ class Freestanding(object):
         assert os.path.exists(self._src_lib)
         os.makedirs(self._dst_lib, exist_ok=True)
         self._copy_file("zstd.h")
-        self._copy_file("zstd_errors.h")
         for subdir in INCLUDED_SUBDIRS:
             src_dir = os.path.join(self._src_lib, subdir)
             dst_dir = os.path.join(self._dst_lib, subdir)

contrib/linux-kernel/Makefile

@@ -23,7 +23,7 @@ libzstd:
 		--rewrite-include '<limits\.h>=<linux/limits.h>' \
 		--rewrite-include '<stddef\.h>=<linux/types.h>' \
 		--rewrite-include '"\.\./zstd.h"=<linux/zstd.h>' \
-		--rewrite-include '"(\.\./)?zstd_errors.h"=<linux/zstd_errors.h>' \
+		--rewrite-include '"(\.\./common/)?zstd_errors.h"=<linux/zstd_errors.h>' \
 		--sed 's,/\*\*\*,/* *,g' \
 		--sed 's,/\*\*,/*,g' \
 		-DZSTD_NO_INTRINSICS \
@@ -35,11 +35,9 @@ libzstd:
 		-DXXH_STATIC_LINKING_ONLY \
 		-DMEM_FORCE_MEMORY_ACCESS=0 \
 		-D__GNUC__ \
-		-D__linux__=1 \
 		-DSTATIC_BMI2=0 \
 		-DZSTD_ADDRESS_SANITIZER=0 \
 		-DZSTD_MEMORY_SANITIZER=0 \
-		-DZSTD_DATAFLOW_SANITIZER=0 \
 		-DZSTD_COMPRESS_HEAPMODE=1 \
 		-UNO_PREFETCH \
 		-U__cplusplus \
@@ -57,7 +55,7 @@ libzstd:
 		-DZSTD_NO_TRACE \
 		-DZSTD_LINUX_KERNEL
 	mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
-	mv linux/lib/zstd/zstd_errors.h linux/include/linux/
+	mv linux/lib/zstd/common/zstd_errors.h linux/include/linux/
 	cp linux_zstd.h linux/include/linux/zstd.h
 	cp zstd_compress_module.c linux/lib/zstd
 	cp zstd_decompress_module.c linux/lib/zstd
@@ -83,23 +81,16 @@ import-upstream:
 	cp -r ../../lib/common $(LINUX)/lib/zstd
 	cp -r ../../lib/compress $(LINUX)/lib/zstd
 	cp -r ../../lib/decompress $(LINUX)/lib/zstd
-	mv $(LINUX)/lib/zstd/zstd_errors.h $(LINUX)/include/linux
+	mv $(LINUX)/lib/zstd/common/zstd_errors.h $(LINUX)/include/linux
 	rm $(LINUX)/lib/zstd/common/threading.*
 	rm $(LINUX)/lib/zstd/common/pool.*
 	rm $(LINUX)/lib/zstd/common/xxhash.*
 	rm $(LINUX)/lib/zstd/compress/zstdmt_*
 
-DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wmissing-prototypes -Wc++-compat \
-	    -Wimplicit-fallthrough
-
 .PHONY: test
 test: libzstd
-	$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS) $(DEBUGFLAGS) -Werror" -j
+	$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS)" -j
 
 .PHONY: clean
 clean:
-	$(RM) -rf linux test/test test/static_test
+	$(RM) -rf linux

contrib/linux-kernel/decompress_sources.h

@@ -16,12 +16,6 @@
  * decompression.
  */
 
-/*
- * Disable the ASM Huffman implementation because we need to
- * include all the sources.
- */
-#define ZSTD_DISABLE_ASM 1
-
 #include "common/debug.c"
 #include "common/entropy_common.c"
 #include "common/error_private.c"

contrib/linux-kernel/linux.mk

@@ -11,8 +11,6 @@
 obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
 obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
 
-ccflags-y += -Wno-error=deprecated-declarations
-
 zstd_compress-y := \
 		zstd_compress_module.o \
 		common/debug.o \
@@ -41,7 +39,6 @@ zstd_decompress-y := \
 		common/fse_decompress.o \
 		common/zstd_common.o \
 		decompress/huf_decompress.o \
-		decompress/huf_decompress_amd64.o \
 		decompress/zstd_ddict.o \
 		decompress/zstd_decompress.o \
 		decompress/zstd_decompress_block.o \

contrib/linux-kernel/mem.h

@@ -30,8 +30,6 @@
 *  Basic Types
 *****************************************************************/
 typedef uint8_t  BYTE;
-typedef uint8_t  U8;
-typedef int8_t   S8;
 typedef uint16_t U16;
 typedef int16_t  S16;
 typedef uint32_t U32;

contrib/linux-kernel/test/Makefile

@@ -11,20 +11,16 @@
 LINUX := ../linux
 LINUX_ZSTDLIB := $(LINUX)/lib/zstd
 
-CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations
+CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG
 # Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse
 CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE
 
 LINUX_ZSTD_MODULE     := $(wildcard $(LINUX_ZSTDLIB)/*.c)
 LINUX_ZSTD_COMMON     := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
 LINUX_ZSTD_COMPRESS   := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
-LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c $(LINUX_ZSTDLIB)/decompress/*.S)
+LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c)
 LINUX_ZSTD_FILES      := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
-LINUX_ZSTD_OBJECTS0   := $(LINUX_ZSTD_FILES:.c=.o)
-LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_OBJECTS0:.S=.o)
-
-%.o: %.S
-	$(CC) -c $(CPPFLAGS) $(CFLAGS) $^ -o $@
+LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_FILES:.c=.o)
 
 liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
 	$(AR) $(ARFLAGS) $@ $^

contrib/linux-kernel/test/include/asm/unaligned.h

@@ -20,7 +20,6 @@ static unsigned _isLittleEndian(void)
 {
     const union { uint32_t u; uint8_t c[4]; } one = { 1 };
     assert(_IS_LITTLE_ENDIAN == one.c[0]);
-    (void)one;
     return _IS_LITTLE_ENDIAN;
 }
 

contrib/linux-kernel/test/include/linux/compiler.h

@@ -18,6 +18,8 @@
 #define noinline __attribute__((noinline))
 #endif
 
+#ifndef fallthrough
 #define fallthrough __attribute__((__fallthrough__))
+#endif
 
 #endif

contrib/linux-kernel/test/include/linux/xxhash.h

@@ -124,10 +124,11 @@ XXH_API uint64_t xxh64(const void *input, size_t length, uint64_t seed);
 static inline unsigned long xxhash(const void *input, size_t length,
 				   uint64_t seed)
 {
-	if (sizeof(size_t) == 8)
-		return xxh64(input, length, seed);
-	else
-		return xxh32(input, length, seed);
+#if BITS_PER_LONG == 64
+       return xxh64(input, length, seed);
+#else
+       return xxh32(input, length, seed);
+#endif
 }
 
 /*-****************************

contrib/linux-kernel/test/macro-test.sh

@@ -41,4 +41,3 @@ test_not_present "ZSTD_DLL_IMPORT"
 test_not_present "__ICCARM__"
 test_not_present "_MSC_VER"
 test_not_present "_WIN32"
-test_not_present "__linux__"

contrib/linux-kernel/test/static_test.c

@@ -28,19 +28,17 @@ static const char kEmptyZstdFrame[] = {
     0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x00, 0x01, 0x00, 0x00, 0x99, 0xe9, 0xd8, 0x51
 };
 
-static void test_decompress_unzstd(void) {
+static void test_decompress_unzstd() {
     fprintf(stderr, "Testing decompress unzstd... ");
     {
         size_t const wkspSize = zstd_dctx_workspace_bound();
         void* wksp = malloc(wkspSize);
-        ZSTD_DCtx* dctx = zstd_init_dctx(wksp, wkspSize);
         CONTROL(wksp != NULL);
+        ZSTD_DCtx* dctx = zstd_init_dctx(wksp, wkspSize);
         CONTROL(dctx != NULL);
-        {
-          size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame));
-          CONTROL(!zstd_is_error(dSize));
-          CONTROL(dSize == 0);
-        }
+        size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame));
+        CONTROL(!zstd_is_error(dSize));
+        CONTROL(dSize == 0);
         free(wksp);
     }
     fprintf(stderr, "Ok\n");

contrib/linux-kernel/test/test.c

@@ -30,15 +30,15 @@ typedef struct {
   size_t compSize;
 } test_data_t;
 
-static test_data_t create_test_data(void) {
+test_data_t create_test_data(void) {
   test_data_t data;
   data.dataSize = 128 * 1024;
-  data.data = (char*)malloc(data.dataSize);
+  data.data = malloc(data.dataSize);
   CONTROL(data.data != NULL);
-  data.data2 = (char*)malloc(data.dataSize);
+  data.data2 = malloc(data.dataSize);
   CONTROL(data.data2 != NULL);
   data.compSize = zstd_compress_bound(data.dataSize);
-  data.comp = (char*)malloc(data.compSize);
+  data.comp = malloc(data.compSize);
   CONTROL(data.comp != NULL);
   memset(data.data, 0, data.dataSize);
   return data;
@@ -54,27 +54,26 @@ static void free_test_data(test_data_t const *data) {
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 
 static void test_btrfs(test_data_t const *data) {
-  size_t const size = MIN(data->dataSize, 128 * 1024);
   fprintf(stderr, "testing btrfs use cases... ");
+  size_t const size = MIN(data->dataSize, 128 * 1024);
   for (int level = -1; level < 16; ++level) {
     zstd_parameters params = zstd_get_params(level, size);
+    CONTROL(params.cParams.windowLog <= 17);
     size_t const workspaceSize =
         MAX(zstd_cstream_workspace_bound(&params.cParams),
             zstd_dstream_workspace_bound(size));
     void *workspace = malloc(workspaceSize);
+    CONTROL(workspace != NULL);
 
     char const *ip = data->data;
     char const *iend = ip + size;
     char *op = data->comp;
     char *oend = op + data->compSize;
-
-    CONTROL(params.cParams.windowLog <= 17);
-    CONTROL(workspace != NULL);
     {
       zstd_cstream *cctx = zstd_init_cstream(&params, size, workspace, workspaceSize);
+      CONTROL(cctx != NULL);
       zstd_out_buffer out = {NULL, 0, 0};
       zstd_in_buffer in = {NULL, 0, 0};
-      CONTROL(cctx != NULL);
       for (;;) {
         if (in.pos == in.size) {
           in.src = ip;
@@ -109,9 +108,9 @@ static void test_btrfs(test_data_t const *data) {
     oend = op + size;
     {
       zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cParams.windowLog, workspace, workspaceSize);
+      CONTROL(dctx != NULL);
       zstd_out_buffer out = {NULL, 0, 0};
       zstd_in_buffer in = {NULL, 0, 0};
-      CONTROL(dctx != NULL);
       for (;;) {
         if (in.pos == in.size) {
           in.src = ip;
@@ -126,16 +125,15 @@ static void test_btrfs(test_data_t const *data) {
           out.pos = 0;
           op += out.size;
         }
-        {
-          size_t const ret = zstd_decompress_stream(dctx, &out, &in);
-          CONTROL(!zstd_is_error(ret));
-          if (ret == 0) {
-            break;
-          }
+
+        size_t const ret = zstd_decompress_stream(dctx, &out, &in);
+        CONTROL(!zstd_is_error(ret));
+        if (ret == 0) {
+          break;
         }
       }
     }
-    CONTROL((size_t)(op - data->data2) == data->dataSize);
+    CONTROL(op - data->data2 == data->dataSize);
     CONTROL(!memcmp(data->data, data->data2, data->dataSize));
     free(workspace);
   }
@@ -143,14 +141,14 @@ static void test_btrfs(test_data_t const *data) {
 }
 
 static void test_decompress_unzstd(test_data_t const *data) {
-    size_t cSize;
     fprintf(stderr, "Testing decompress unzstd... ");
+    size_t cSize;
     {
         zstd_parameters params = zstd_get_params(19, 0);
         size_t const wkspSize = zstd_cctx_workspace_bound(&params.cParams);
         void* wksp = malloc(wkspSize);
-        zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize);
         CONTROL(wksp != NULL);
+        zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize);
         CONTROL(cctx != NULL);
         cSize = zstd_compress_cctx(cctx, data->comp, data->compSize, data->data, data->dataSize, &params);
         CONTROL(!zstd_is_error(cSize));
@@ -159,21 +157,19 @@ static void test_decompress_unzstd(test_data_t const *data) {
     {
         size_t const wkspSize = zstd_dctx_workspace_bound();
         void* wksp = malloc(wkspSize);
-        zstd_dctx* dctx = zstd_init_dctx(wksp, wkspSize);
         CONTROL(wksp != NULL);
+        zstd_dctx* dctx = zstd_init_dctx(wksp, wkspSize);
         CONTROL(dctx != NULL);
-        {
-          size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize);
-          CONTROL(!zstd_is_error(dSize));
-          CONTROL(dSize == data->dataSize);
-        }
+        size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize);
+        CONTROL(!zstd_is_error(dSize));
+        CONTROL(dSize == data->dataSize);
         CONTROL(!memcmp(data->data, data->data2, data->dataSize));
         free(wksp);
     }
     fprintf(stderr, "Ok\n");
 }
 
-static void test_f2fs(void) {
+static void test_f2fs() {
   fprintf(stderr, "testing f2fs uses... ");
   CONTROL(zstd_min_clevel() < 0);
   CONTROL(zstd_max_clevel() == 22);
@@ -186,7 +182,7 @@ static void __attribute__((noinline)) use(void *x) {
   asm volatile("" : "+r"(x));
 }
 
-static void __attribute__((noinline)) set_stack(void) {
+static void __attribute__((noinline)) set_stack() {
 
   char stack[8192];
   g_stack = stack;
@@ -194,16 +190,14 @@ static void __attribute__((noinline)) set_stack(void) {
   use(g_stack);
 }
 
-static void __attribute__((noinline)) check_stack(void) {
+static void __attribute__((noinline)) check_stack() {
   size_t cleanStack = 0;
   while (cleanStack < 8192 && g_stack[cleanStack] == 0x33) {
     ++cleanStack;
   }
-  {
-    size_t const stackSize = 8192 - cleanStack;
-    fprintf(stderr, "Maximum stack size: %zu\n", stackSize);
-    CONTROL(stackSize <= 2048 + 512);
-  }
+  size_t const stackSize = 8192 - cleanStack;
+  fprintf(stderr, "Maximum stack size: %zu\n", stackSize);
+  CONTROL(stackSize <= 2048 + 512);
 }
 
 static void test_stack_usage(test_data_t const *data) {

contrib/pzstd/Makefile

@@ -30,9 +30,6 @@ CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
 CPPFLAGS ?=
 LDFLAGS  ?=
 
-# PZstd uses legacy APIs
-CFLAGS   += -Wno-deprecated-declarations
-
 # Include flags
 PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 GTEST_INC  = -isystem googletest/googletest/include
@@ -57,6 +54,19 @@ LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
 CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
 CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
 
+# Get a list of all zstd files so we rebuild the static library when we need to
+ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
+                    $(wildcard $(ZSTDDIR)/common/*.h)
+ZSTDCOMP_FILES   := $(wildcard $(ZSTDDIR)/compress/*.c) \
+                    $(wildcard $(ZSTDDIR)/compress/*.h)
+ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
+                    $(wildcard $(ZSTDDIR)/decompress/*.h)
+ZSTDPROG_FILES   := $(wildcard $(PROGDIR)/*.c) \
+                    $(wildcard $(PROGDIR)/*.h)
+ZSTD_FILES       := $(wildcard $(ZSTDDIR)/*.h) \
+                    $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
+                    $(ZSTDPROG_FILES)
+
 # List all the pzstd source files so we can determine their dependencies
 PZSTD_SRCS  := $(wildcard *.cpp)
 PZSTD_TESTS := $(wildcard test/*.cpp)
@@ -176,8 +186,7 @@ roundtrip: test/RoundTripTest$(EXT)
 
 # Use the static library that zstd builds for simplicity and
 # so we get the compiler options correct
-.PHONY: $(ZSTDDIR)/libzstd.a
-$(ZSTDDIR)/libzstd.a:
+$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
 	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
 
 # Rules to build the tests

contrib/pzstd/Options.cpp

@@ -87,7 +87,7 @@ void usage() {
   std::fprintf(stderr, "  -V, --version          : display version number and exit\n");
   std::fprintf(stderr, "  -v, --verbose          : verbose mode; specify multiple times to increase log level (default:2)\n");
   std::fprintf(stderr, "  -q, --quiet            : suppress warnings; specify twice to suppress errors too\n");
-  std::fprintf(stderr, "  -c, --stdout           : write to standard output (even if it is the console)\n");
+  std::fprintf(stderr, "  -c, --stdout           : force write to standard output, even if it is the console\n");
 #ifdef UTIL_HAS_CREATEFILELIST
   std::fprintf(stderr, "  -r                     : operate recursively on directories\n");
 #endif

contrib/pzstd/Options.h

@@ -9,9 +9,6 @@
 #pragma once
 
 #define ZSTD_STATIC_LINKING_ONLY
-#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
-                                         * and uses deprecated functions
-                                         */
 #include "zstd.h"
 #undef ZSTD_STATIC_LINKING_ONLY
 

contrib/pzstd/Pzstd.h

@@ -17,9 +17,6 @@
 #include "utils/ThreadPool.h"
 #include "utils/WorkQueue.h"
 #define ZSTD_STATIC_LINKING_ONLY
-#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
-                                         * and uses deprecated functions
-                                         */
 #include "zstd.h"
 #undef ZSTD_STATIC_LINKING_ONLY
 

contrib/pzstd/utils/Buffer.h

@@ -45,7 +45,7 @@ class Buffer {
       : buffer_(buffer), range_(data) {}
 
   Buffer(Buffer&&) = default;
-  Buffer& operator=(Buffer&&) = default;
+  Buffer& operator=(Buffer&&) & = default;
 
   /**
    * Splits the data into two pieces: [begin, begin + n), [begin + n, end).

contrib/pzstd/utils/Range.h

@@ -6,7 +6,7 @@
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  */
-
+ 
 /**
  * A subset of `folly/Range.h`.
  * All code copied verbatim modulo formatting
@@ -83,8 +83,8 @@ class Range {
   Range(const Range&) = default;
   Range(Range&&) = default;
 
-  Range& operator=(const Range&) = default;
-  Range& operator=(Range&&) = default;
+  Range& operator=(const Range&) & = default;
+  Range& operator=(Range&&) & = default;
 
   constexpr size_type size() const {
     return e_ - b_;

contrib/seekable_format/examples/Makefile

@@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
 
-CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
+CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
 
 CFLAGS ?= -O3
 CFLAGS += -g

contrib/seekable_format/examples/parallel_compression.c

@@ -21,6 +21,7 @@
 #  define SLEEP(x) usleep(x * 1000)
 #endif
 
+#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"
 
 #include "pool.h"      // use zstd thread pool for demo

contrib/seekable_format/examples/seekable_decompression.c

@@ -99,9 +99,6 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end
 
     while (startOffset < endOffset) {
         size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
-        if (!result) {
-            break;
-        }
 
         if (ZSTD_isError(result)) {
             fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",

contrib/seekable_format/examples/seekable_decompression_mem.c

@@ -104,9 +104,6 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end
 
     while (startOffset < endOffset) {
         size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
-        if (!result) {
-            break;
-        }
 
         if (ZSTD_isError(result)) {
             fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",

contrib/seekable_format/tests/Makefile

@@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
 
-CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
+CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
 
 CFLAGS ?= -O3
 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \

contrib/seekable_format/zstdseek_compress.c

@@ -12,6 +12,7 @@
 #include <assert.h>
 
 #define XXH_STATIC_LINKING_ONLY
+#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"
 
 #define ZSTD_STATIC_LINKING_ONLY
@@ -82,7 +83,7 @@ static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
 
 ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
 {
-    ZSTD_frameLog* const fl = (ZSTD_frameLog*)malloc(sizeof(ZSTD_frameLog));
+    ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog));
     if (fl == NULL) return NULL;
 
     if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
@@ -107,7 +108,7 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl)
 
 ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void)
 {
-    ZSTD_seekable_CStream* const zcs = (ZSTD_seekable_CStream*)malloc(sizeof(ZSTD_seekable_CStream));
+    ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream));
     if (zcs == NULL) return NULL;
 
     memset(zcs, 0, sizeof(*zcs));
@@ -176,7 +177,7 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
     if (fl->size == fl->capacity) {
         /* exponential size increase for constant amortized runtime */
         size_t const newCapacity = fl->capacity * 2;
-        framelogEntry_t* const newEntries = (framelogEntry_t*)realloc(fl->entries,
+        framelogEntry_t* const newEntries = realloc(fl->entries,
                 sizeof(framelogEntry_t) * newCapacity);
 
         if (newEntries == NULL) return ERROR(memory_allocation);

contrib/seekable_format/zstdseek_decompress.c

@@ -60,6 +60,7 @@
 #include <assert.h>
 
 #define XXH_STATIC_LINKING_ONLY
+#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"
 
 #define ZSTD_STATIC_LINKING_ONLY
@@ -175,7 +176,7 @@ struct ZSTD_seekable_s {
 
 ZSTD_seekable* ZSTD_seekable_create(void)
 {
-    ZSTD_seekable* const zs = (ZSTD_seekable*)malloc(sizeof(ZSTD_seekable));
+    ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable));
     if (zs == NULL) return NULL;
 
     /* also initializes stage to zsds_init */
@@ -201,7 +202,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
 
 ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
 {
-    ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
+    ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable));
     if (st==NULL) return NULL;
 
     st->checksumFlag = zs->seekTable.checksumFlag;
@@ -432,11 +433,6 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr
 
 size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset)
 {
-    unsigned long long const eos = zs->seekTable.entries[zs->seekTable.tableLen].dOffset;
-    if (offset + len > eos) {
-        len = eos - offset;
-    }
-
     U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset);
     U32 noOutputProgressCount = 0;
     size_t srcBytesRead = 0;
@@ -453,7 +449,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
             zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
             XXH64_reset(&zs->xxhState, 0);
             ZSTD_DCtx_reset(zs->dstream, ZSTD_reset_session_only);
-            if (zs->buffWrapper.size && srcBytesRead > zs->buffWrapper.size) {
+            if (srcBytesRead > zs->buffWrapper.size) {
                 return ERROR(seekableIO);
             }
         }
@@ -506,8 +502,6 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
                 if (zs->decompressedOffset < offset + len) {
                     /* go back to the start and force a reset of the stream */
                     targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, zs->decompressedOffset);
-                    /* in this case it will fail later with corruption_detected, since last block does not have checksum */
-                    assert(targetFrame != zs->seekTable.tableLen);
                 }
                 break;
             }

contrib/single_file_libs/README.md

@@ -11,7 +11,7 @@ This is the most common use case. The decompression library is small, adding, fo
 
 Create `zstddeclib.c` from the Zstd source using:
 ```
-cd zstd/build/single_file_libs
+cd zstd/contrib/single_file_libs
 ./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
 ```
 Then add the resulting file to your project (see the [example files](examples)).
@@ -25,7 +25,7 @@ The same tool can amalgamate the entire Zstd library for ease of adding both com
 
 Create `zstd.c` from the Zstd source using:
 ```
-cd zstd/build/single_file_libs
+cd zstd/contrib/single_file_libs
 ./combine.sh -r ../../lib -o zstd.c zstd-in.c
 ```
 It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`).

contrib/single_file_libs/zstd-in.c

@@ -25,7 +25,7 @@
  * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
  * defined in mem.h (breaking C99 compatibility).
  *
- * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
+ * Note: the undefs for xxHash allow Zstd's implementation to coinside with with
  * standalone xxHash usage (with global defines).
  *
  * Note: multithreading is enabled for all platforms apart from Emscripten.
@@ -43,8 +43,6 @@
 #define ZSTD_MULTITHREAD
 #endif
 #define ZSTD_TRACE 0
-/* TODO: Can't amalgamate ASM function */
-#define ZSTD_DISABLE_ASM 1
 
 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC

contrib/single_file_libs/zstddeclib-in.c

@@ -25,7 +25,7 @@
  * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
  * defined in mem.h (breaking C99 compatibility).
  *
- * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
+ * Note: the undefs for xxHash allow Zstd's implementation to coinside with with
  * standalone xxHash usage (with global defines).
  */
 #define DEBUGLEVEL 0
@@ -39,8 +39,6 @@
 #define ZSTD_LEGACY_SUPPORT 0
 #define ZSTD_STRIP_ERROR_STRINGS
 #define ZSTD_TRACE 0
-/* TODO: Can't amalgamate ASM function */
-#define ZSTD_DISABLE_ASM 1
 
 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC

doc/educational_decoder/zstd_decompress.c

@@ -2145,7 +2145,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable,
 
     // "All remaining symbols are sorted in their natural order. Starting from
     // symbol 0 and table position 0, each symbol gets attributed as many cells
-    // as its probability. Cell allocation is spread, not linear."
+    // as its probability. Cell allocation is spreaded, not linear."
     // Place the rest in the table
     const u16 step = (size >> 1) + (size >> 3) + 3;
     const u16 mask = size - 1;

doc/zstd_compression_format.md

@@ -1124,7 +1124,7 @@ These symbols define a full state reset, reading `Accuracy_Log` bits.
 Then, all remaining symbols, sorted in natural order, are allocated cells.
 Starting from symbol `0` (if it exists), and table position `0`,
 each symbol gets allocated as many cells as its probability.
-Cell allocation is spread, not linear :
+Cell allocation is spreaded, not linear :
 each successor position follows this rule :
 
 ```
@@ -1669,7 +1669,7 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
-- 0.3.7 : clarifications for Repeat_Offsets, matching RFC8878
+- 0.3.7 : clarifications for Repeat_Offsets
 - 0.3.6 : clarifications for Dictionary_ID
 - 0.3.5 : clarifications for Block_Maximum_Size
 - 0.3.4 : clarifications for FSE decoding table

doc/zstd_manual.html

@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.5.1 Manual</title>
+<title>zstd 1.4.9 Manual</title>
 </head>
 <body>
-<h1>zstd 1.5.1 Manual</h1>
+<h1>zstd 1.4.9 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -12,15 +12,15 @@
 <li><a href="#Chapter2">Version</a></li>
 <li><a href="#Chapter3">Simple API</a></li>
 <li><a href="#Chapter4">Explicit context</a></li>
-<li><a href="#Chapter5">Advanced compression API (Requires v1.4.0+)</a></li>
-<li><a href="#Chapter6">Advanced decompression API (Requires v1.4.0+)</a></li>
+<li><a href="#Chapter5">Advanced compression API</a></li>
+<li><a href="#Chapter6">Advanced decompression API</a></li>
 <li><a href="#Chapter7">Streaming</a></li>
 <li><a href="#Chapter8">Streaming compression - HowTo</a></li>
 <li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
 <li><a href="#Chapter10">Simple dictionary API</a></li>
 <li><a href="#Chapter11">Bulk processing dictionary API</a></li>
 <li><a href="#Chapter12">Dictionary helper functions</a></li>
-<li><a href="#Chapter13">Advanced dictionary and prefix API (Requires v1.4.0+)</a></li>
+<li><a href="#Chapter13">Advanced dictionary and prefix API</a></li>
 <li><a href="#Chapter14">experimental API (static linking only)</a></li>
 <li><a href="#Chapter15">Frame size functions</a></li>
 <li><a href="#Chapter16">Memory management</a></li>
@@ -141,9 +141,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
+int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed */<b>
 int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
-int         ZSTD_defaultCLevel(void);           </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
 </pre></b><BR>
 <a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
 
@@ -158,7 +157,7 @@ int         ZSTD_defaultCLevel(void);           </b>/*!< default compression lev
  
 </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* accept NULL pointer */<b>
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 </pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
                          void* dst, size_t dstCapacity,
@@ -180,7 +179,7 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* accept NULL pointer */<b>
   Use one context per thread for parallel execution. 
 </pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
-size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
+size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </pre></b><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
                            void* dst, size_t dstCapacity,
@@ -191,7 +190,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
  
 </p></pre><BR>
 
-<a name="Chapter5"></a><h2>Advanced compression API (Requires v1.4.0+)</h2><pre></pre>
+<a name="Chapter5"></a><h2>Advanced compression API</h2><pre></pre>
 
 <pre><b>typedef enum { ZSTD_fast=1,
                ZSTD_dfast=2,
@@ -271,6 +270,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression.
                               * Special: value 0 means "use default strategy". */
+
     </b>/* LDM mode parameters */<b>
     ZSTD_c_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
                                      * This parameter is designed to improve compression ratio
@@ -327,7 +327,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
     ZSTD_c_jobSize=401,      </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b>
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
-                              * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+                              * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
                               * The minimum size is automatically and transparently enforced. */
     ZSTD_c_overlapLog=402,   </b>/* Control the overlap size, as a fraction of window size.<b>
                               * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
@@ -357,8 +357,6 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_c_stableOutBuffer
      * ZSTD_c_blockDelimiters
      * ZSTD_c_validateSequences
-     * ZSTD_c_useBlockSplitter
-     * ZSTD_c_useRowMatchFinder
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -374,10 +372,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam9=1006,
      ZSTD_c_experimentalParam10=1007,
      ZSTD_c_experimentalParam11=1008,
-     ZSTD_c_experimentalParam12=1009,
-     ZSTD_c_experimentalParam13=1010,
-     ZSTD_c_experimentalParam14=1011,
-     ZSTD_c_experimentalParam15=1012
+     ZSTD_c_experimentalParam12=1009
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -461,7 +456,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
  
 </p></pre><BR>
 
-<a name="Chapter6"></a><h2>Advanced decompression API (Requires v1.4.0+)</h2><pre></pre>
+<a name="Chapter6"></a><h2>Advanced decompression API</h2><pre></pre>
 
 <pre><b>typedef enum {
 
@@ -592,7 +587,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
 <pre><b>typedef ZSTD_CCtx ZSTD_CStream;  </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b>
 </b></pre><BR>
 <h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  </b>/* accept NULL pointer */<b>
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 </pre></b><BR>
 <h3>Streaming compression functions</h3><pre></pre><b><pre>typedef enum {
     ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
@@ -686,7 +681,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 <pre><b>typedef ZSTD_DCtx ZSTD_DStream;  </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b>
 </b></pre><BR>
 <h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 </pre></b><BR>
 <h3>Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
@@ -702,7 +697,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
                                int compressionLevel);
 </b><p>  Compression at an explicit compression level using a Dictionary.
   A dictionary can be any arbitrary data segment (also called a prefix),
-  or a buffer with specified information (see zdict.h).
+  or a buffer with specified information (see dictBuilder/zdict.h).
   Note : This function loads the dictionary, resulting in significant startup delay.
          It's intended for a dictionary used only once.
   Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. 
@@ -737,8 +732,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-</b><p>  Function frees memory allocated by ZSTD_createCDict().
-  If a NULL pointer is passed, no operation is performed. 
+</b><p>  Function frees memory allocated by ZSTD_createCDict(). 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -757,8 +751,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
-</b><p>  Function frees memory allocated with ZSTD_createDDict()
-  If a NULL pointer is passed, no operation is performed. 
+</b><p>  Function frees memory allocated with ZSTD_createDDict() 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
@@ -777,12 +770,6 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
   It can still be loaded, but as a content-only dictionary. 
 </p></pre><BR>
 
-<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
-</b><p>  Provides the dictID of the dictionary loaded into `cdict`.
-  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
-  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
-</p></pre><BR>
-
 <pre><b>unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
 </b><p>  Provides the dictID of the dictionary loaded into `ddict`.
   If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
@@ -801,9 +788,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
   When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 </p></pre><BR>
 
-<a name="Chapter13"></a><h2>Advanced dictionary and prefix API (Requires v1.4.0+)</h2><pre>
+<a name="Chapter13"></a><h2>Advanced dictionary and prefix API</h2><pre>
  This API allows dictionaries to be used with ZSTD_compress2(),
- ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
+ ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
  only reset with the context is reset with ZSTD_reset_parameters or
  ZSTD_reset_session_and_parameters. Prefixes are single-use.
 <BR></pre>
@@ -1071,19 +1058,9 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
   ZSTD_lcm_uncompressed = 2   </b>/**< Always emit uncompressed literals. */<b>
 } ZSTD_literalCompressionMode_e;
 </b></pre><BR>
-<pre><b>typedef enum {
-  </b>/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final<b>
-   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
-   * or ZSTD_ps_disable allow for a force enable/disable the feature.
-   */
-  ZSTD_ps_auto = 0,         </b>/* Let the library automatically determine whether the feature shall be enabled */<b>
-  ZSTD_ps_enable = 1,       </b>/* Force-enable the feature */<b>
-  ZSTD_ps_disable = 2       </b>/* Do not use the feature */<b>
-} ZSTD_paramSwitch_e;
-</b></pre><BR>
 <a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
   `srcSize` must be the _exact_ size of this series
        (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1106,7 +1083,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
             however it does mean that all frame data must be present and valid. 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+<pre><b>unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
   `srcSize` must be the _exact_ size of this series
        (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1121,7 +1098,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+<pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 </b><p>  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
  @return : size of the Frame Header,
            or an error code (if srcSize is too small) 
@@ -1148,7 +1125,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+<pre><b>size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
 </b><p> Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
  by merging them into into the literals of the next sequence.
 
@@ -1161,7 +1138,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
+<pre><b>size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
                       const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
                       const void* src, size_t srcSize);
 </b><p> Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
@@ -1193,7 +1170,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+<pre><b>size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
                                 const void* src, size_t srcSize, unsigned magicVariant);
 </b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
 
@@ -1209,31 +1186,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 </p></pre><BR>
 
-<pre><b>size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
-                                const void* src, size_t srcSize);
-</b><p> Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
-
- The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
- i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
- in the magicVariant.
-
- Returns an error if destination buffer is not large enough, or if the frame is not skippable.
-
- @return : number of bytes written or a ZSTD error.
- 
-</p></pre><BR>
-
-<pre><b>unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
-</b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
- 
-</p></pre><BR>
-
 <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
+<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
+size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+size_t ZSTD_estimateDCtxSize(void);
 </b><p>  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
 
@@ -1258,11 +1216,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+<pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
+size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+size_t ZSTD_estimateDStreamSize(size_t windowSize);
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 </b><p>  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
   It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
@@ -1277,17 +1235,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, si
          In this case, get total size by adding ZSTD_estimate?DictSize 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+<pre><b>size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
 </b><p>  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
   ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
   Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
-ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
+<pre><b>ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
 </b><p>  Initialize an object using a pre-allocated fixed-size buffer.
   workspace: The memory area to emplace the object into.
              Provided pointer *must be 8-bytes aligned*.
@@ -1310,7 +1268,7 @@ ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticDCtx() */<b>
+<pre><b>ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticDCtx() */<b>
 </b></pre><BR>
 <pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
@@ -1326,24 +1284,9 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this con
  
 </p></pre><BR>
 
-<pre><b>typedef struct POOL_ctx_s ZSTD_threadPool;
-ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
-ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  </b>/* accept NULL pointer */<b>
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
-</b><p>  These prototypes make it possible to share a thread pool among multiple compression contexts.
-  This can limit resources for applications with multiple threads where each one uses
-  a threaded compression mode (via ZSTD_c_nbWorkers parameter).
-  ZSTD_createThreadPool creates a new thread pool with a given number of threads.
-  Note that the lifetime of such pool must exist while being used.
-  ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
-  to use an internal thread pool).
-  ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
- 
-</p></pre><BR>
-
 <a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 </b><p>  Create a digested dictionary for compression
   Dictionary content is just referenced, not duplicated.
   As a consequence, `dictBuffer` **must** outlive CDict,
@@ -1351,22 +1294,28 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
   note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+</b><p>  Provides the dictID of the dictionary loaded into `cdict`.
+  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
+</p></pre><BR>
+
+<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 </b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
  `estimatedSrcSize` value is optional, select 0 if not known 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 </b><p>  same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
   All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+<pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 </b><p>  Ensure param values remain within authorized range.
  @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+<pre><b>ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 </b><p>  optimize params for a given `srcSize` and `dictSize`.
  `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
  `dictSize` must be `0` when there is no dictionary.
@@ -1374,53 +1323,51 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
   This function never fails (wide contract) 
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2")
-size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+<pre><b>size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
                               void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                         const void* dict,size_t dictSize,
                               ZSTD_parameters params);
 </b><p>  Note : this function is now DEPRECATED.
          It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
-  This prototype will generate compilation warnings. 
+  This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x 
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
-size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                   void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
                             const ZSTD_CDict* cdict,
                                   ZSTD_frameParameters fParams);
-</b><p>  Note : this function is now DEPRECATED.
+</b><p>  Note : this function is now REDUNDANT.
          It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
-  This prototype will generate compilation warnings. 
+  This prototype will be marked as deprecated and generate compilation warning in some future version 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+<pre><b>size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 </b><p>  Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
   It saves some memory, but also requires that `dict` outlives its usage within `cctx` 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+<pre><b>size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
   how to load the dictionary (by copy ? by reference ?)
   and how to interpret it (automatic ? force raw mode ? full mode only ?) 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+<pre><b>size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_CCtx_refPrefix(), but gives finer control over
   how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+<pre><b>size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
 </b><p>  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
   and store it into int* value.
  @return : 0, or an error code (which can be tested with ZSTD_isError()).
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
-ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/* accept NULL pointer */<b>
+<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 </b><p>  Quick howto :
   - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
   - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
@@ -1432,31 +1379,31 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
                                     These parameters will be applied to
                                     all subsequent frames.
   - ZSTD_compressStream2() : Do compression using the CCtx.
-  - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
+  - ZSTD_freeCCtxParams() : Free the memory.
 
   This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
   for static allocation of CCtx for single-threaded compression.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+<pre><b>size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
 </b><p>  Reset params to default values.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+<pre><b>size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
 </b><p>  Initializes the compression parameters of cctxParams according to
   compression level. All other parameters are reset to their default values.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+<pre><b>size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
 </b><p>  Initializes the compression and frame parameters of cctxParams according to
   params. All other parameters are reset to their default values.
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+<pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 </b><p>  Similar to ZSTD_CCtx_setParameter.
   Set one compression parameter, selected by enum ZSTD_cParameter.
   Parameters must be applied to a ZSTD_CCtx using
@@ -1466,14 +1413,14 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+<pre><b>size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 </b><p> Similar to ZSTD_CCtx_getParameter.
  Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+<pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
         ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
 </b><p>  Apply a set of ZSTD_CCtx_params to the compression context.
   This can be done even after compression is started,
@@ -1483,7 +1430,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
+<pre><b>size_t ZSTD_compressStream2_simpleArgs (
                 ZSTD_CCtx* cctx,
                 void* dst, size_t dstCapacity, size_t* dstPos,
           const void* src, size_t srcSize, size_t* srcPos,
@@ -1497,40 +1444,40 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
 
 <a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier.
   Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
   Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
   Note 3 : Skippable Frame Identifiers are considered valid. 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 </b><p>  Create a digested dictionary, ready to start decompression operation without startup delay.
   Dictionary content is referenced, and therefore stays in dictBuffer.
   It is important that dictBuffer outlives DDict,
   it must remain read accessible throughout the lifetime of DDict 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+<pre><b>size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 </b><p>  Same as ZSTD_DCtx_loadDictionary(),
   but references `dict` content instead of copying it into `dctx`.
   This saves memory if `dict` remains around.,
   However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+<pre><b>size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_DCtx_loadDictionary(),
   but gives direct control over
   how to load the dictionary (by copy ? by reference ?)
   and how to interpret it (automatic ? force raw mode ? full mode only ?). 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+<pre><b>size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_DCtx_refPrefix(), but gives finer control over
   how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+<pre><b>size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
 </b><p>  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
   This protects a decoder context from reserving too much memory for itself (potential attack scenario).
   This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
@@ -1539,23 +1486,21 @@ ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/*
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+<pre><b>size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
 </b><p>  Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
   and store it into int* value.
  @return : 0, or an error code (which can be tested with ZSTD_isError()).
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
-size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
-</b><p>  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
-  Instruct the decoder context about what kind of data to decode next.
+<pre><b>size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+</b><p>  Instruct the decoder context about what kind of data to decode next.
   This instruction is mandatory to decode data without a fully-formed header,
   such ZSTD_f_zstd1_magicless for example.
  @return : 0, or an error code (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
+<pre><b>size_t ZSTD_decompressStream_simpleArgs (
                 ZSTD_DCtx* dctx,
                 void* dst, size_t dstCapacity, size_t* dstPos,
           const void* src, size_t srcSize, size_t* srcPos);
@@ -1572,11 +1517,11 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 <BR></pre>
 
 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+<pre><b>size_t
+ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
              int compressionLevel,
              unsigned long long pledgedSrcSize);
-</b><p> This function is DEPRECATED, and equivalent to:
+</b><p> This function is deprecated, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@@ -1585,15 +1530,15 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  pledgedSrcSize must be correct. If it is not known at init time, use
  ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
  "0" also disables frame content size field. It may be enabled in the future.
- This prototype will generate compilation warnings.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+<pre><b>size_t
+ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
          const void* dict, size_t dictSize,
                int compressionLevel);
-</b><p> This function is DEPRECATED, and is equivalent to:
+</b><p> This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
      ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1602,16 +1547,16 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
  dict == NULL or dictSize < 8, in which case no dict is used.
  Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
- This prototype will generate compilation warnings.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+<pre><b>size_t
+ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         const void* dict, size_t dictSize,
               ZSTD_parameters params,
               unsigned long long pledgedSrcSize);
-</b><p> This function is DEPRECATED, and is approximately equivalent to:
+</b><p> This function is deprecated, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd parameter and leave the rest as-is.
      for ((param, value) : params) {
@@ -1623,23 +1568,22 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
  dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
  pledgedSrcSize must be correct.
  If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
- This prototype will generate compilation warnings.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
-</b><p> This function is DEPRECATED, and equivalent to:
+<pre><b>size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+</b><p> This function is deprecated, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, cdict);
 
  note : cdict will just be referenced, and must outlive compression session
- This prototype will generate compilation warnings.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+<pre><b>size_t
+ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                    const ZSTD_CDict* cdict,
                          ZSTD_frameParameters fParams,
                          unsigned long long pledgedSrcSize);
@@ -1655,18 +1599,14 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
  pledgedSrcSize must be correct. If srcSize is not known at init time, use
  value ZSTD_CONTENTSIZE_UNKNOWN.
- This prototype will generate compilation warnings.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
-</b><p> This function is DEPRECATED, and is equivalent to:
+<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+</b><p> This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
-       ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
-       explicitly specified.
 
   start a new frame, using same parameters from previous frame.
   This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
@@ -1676,7 +1616,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
   For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
   but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
  @return : 0, or an error code (which can be tested using ZSTD_isError())
-  This prototype will generate compilation warnings.
+  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 </p></pre><BR>
 
@@ -1689,7 +1629,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
     unsigned nbActiveWorkers;      </b>/* MT only : nb of workers actively compressing at probe time */<b>
 } ZSTD_frameProgression;
 </b></pre><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+<pre><b>size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 </b><p>  Tell how many bytes are ready to be flushed immediately.
   Useful for multithreading scenarios (nbWorkers >= 1).
   Probe the oldest active job, defined as oldest job not yet entirely flushed,
@@ -1705,7 +1645,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </p></pre><BR>
 
 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+<pre><b>size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
@@ -1715,7 +1655,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+<pre><b>size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
@@ -1725,7 +1665,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+<pre><b>size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
@@ -1747,7 +1687,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
 
   Start by initializing a context.
-  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
+  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
+  or ZSTD_compressBegin_advanced(), for finer parameter control.
   It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
@@ -1769,13 +1710,13 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
   `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
 <BR></pre>
 
-<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
+<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </pre></b><BR>
-<pre><b>size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
-</b></pre><BR>
 <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
@@ -1858,12 +1799,12 @@ typedef struct {
     unsigned checksumFlag;
 } ZSTD_frameHeader;
 </pre></b><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
+<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
 </b>/*! ZSTD_getFrameHeader_advanced() :<b>
  *  same as ZSTD_getFrameHeader(),
  *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </b><p>  decode Frame Header, or requires larger `srcSize`.
  @return : 0, `zfhPtr` is correctly filled,
           >0, `srcSize` is too small, value is wanted `srcSize` amount,
@@ -1899,10 +1840,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowS
         Use ZSTD_insertBlock() for such a case.
 </p></pre><BR>
 
-<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
+<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
+size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
 </pre></b><BR>
 </html>
 </body>

examples/common.h

@@ -57,7 +57,7 @@ typedef enum {
  * Check the zstd error code and die if an error occurred after printing a
  * message.
  */
-#define CHECK_ZSTD(fn)                                           \
+#define CHECK_ZSTD(fn, ...)                                      \
     do {                                                         \
         size_t const err = (fn);                                 \
         CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \

examples/streaming_compression.c

@@ -15,12 +15,9 @@
 #include <zstd.h>      // presumes zstd library is installed
 #include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
-static void compressFile_orDie(const char* fname, const char* outName, int cLevel,
-                               int nbThreads)
-{
-    fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n",
-             fname, cLevel, nbThreads);
 
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
+{
     /* Open the input and output files. */
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
@@ -42,7 +39,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
      */
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 4);
 
     /* This loop read from the input file, compresses that entire chunk,
      * and writes all output produced to the output file.
@@ -109,32 +106,19 @@ int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];
 
-    if (argc < 2) {
+    if (argc!=2) {
         printf("wrong arguments\n");
         printf("usage:\n");
-        printf("%s FILE [LEVEL] [THREADS]\n", exeName);
+        printf("%s FILE\n", exeName);
         return 1;
     }
 
-    int cLevel = 1;
-    int nbThreads = 4;
-
-    if (argc >= 3) {
-      cLevel = atoi (argv[2]);
-      CHECK(cLevel != 0, "can't parse LEVEL!");
-    }
-
-    if (argc >= 4) {
-      nbThreads = atoi (argv[3]);
-      CHECK(nbThreads != 0, "can't parse THREADS!");
-    }
-
     const char* const inFilename = argv[1];
 
     char* const outFilename = createOutFilename_orDie(inFilename);
-    compressFile_orDie(inFilename, outFilename, cLevel, nbThreads);
+    compressFile_orDie(inFilename, outFilename, 1);
 
     free(outFilename);   /* not strictly required, since program execution stops there,
-                          * but some static analyzer may complain otherwise */
+                          * but some static analyzer main complain otherwise */
     return 0;
 }

examples/streaming_compression_thread_pool.c

@@ -28,10 +28,8 @@ typedef struct compress_args
 
 static void *compressFile_orDie(void *data)
 {
-    const int nbThreads = 16;
-
     compress_args_t *args = (compress_args_t *)data;
-    fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", args->fname, args->cLevel, nbThreads);
+    fprintf (stderr, "Starting compression of %s with level %d\n", args->fname, args->cLevel);
     /* Open the input and output files. */
     FILE* const fin  = fopen_orDie(args->fname, "rb");
     FILE* const fout = fopen_orDie(args->outName, "wb");
@@ -58,9 +56,9 @@ static void *compressFile_orDie(void *data)
      */
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) );
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 16);
 
-    /* This loop reads from the input file, compresses that entire chunk,
+    /* This loop read from the input file, compresses that entire chunk,
      * and writes all output produced to the output file.
      */
     size_t const toRead = buffInSize;

lib/BUCK

@@ -65,7 +65,9 @@ cxx_library(
     name='zdict',
     header_namespace='',
     visibility=['PUBLIC'],
-    exported_headers=['zdict.h'],
+    exported_headers=subdir_glob([
+        ('dictBuilder', 'zdict.h'),
+    ]),
     headers=subdir_glob([
         ('dictBuilder', 'divsufsort.h'),
         ('dictBuilder', 'cover.h'),
@@ -129,10 +131,10 @@ cxx_library(
     name='errors',
     header_namespace='',
     visibility=['PUBLIC'],
-    exported_headers=[
-        'zstd_errors.h',
-        'common/error_private.h',
-    ]
+    exported_headers=subdir_glob([
+        ('common', 'error_private.h'),
+        ('common', 'zstd_errors.h'),
+    ]),
     srcs=['common/error_private.c'],
 )
 

lib/Makefile

@@ -8,13 +8,103 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
+.PHONY: default
+default: lib-release
+
+# define silent mode as default (verbose mode with V=1 or VERBOSE=1)
+$(V)$(VERBOSE).SILENT:
+
+# When cross-compiling from linux to windows,
+# one might need to specify TARGET_SYSTEM as "Windows."
+# Building from Fedora fails without it.
+# (but Ubuntu and Debian don't need to set anything)
+TARGET_SYSTEM ?= $(OS)
+
+# Version numbers
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
+LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
+LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
+LIBVER := $(shell echo $(LIBVER_SCRIPT))
+VERSION?= $(LIBVER)
+CCVER := $(shell $(CC) --version)
+
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+  ZSTD_LIB_DEPRECATED ?= 0
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+ifneq ($(HAVE_CC_OZ), 0)
+    # Some compilers (clang) support an even more space-optimized setting.
+    CFLAGS += -Oz
+else
+    CFLAGS += -Os
+endif
+  CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
+            -DDYNAMIC_BMI2=0 -DNDEBUG
+else
+  CFLAGS += -O3
+endif
+
+DEBUGLEVEL ?= 0
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
+ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
+  CPPFLAGS += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
+endif
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
+CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS)
+
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+  GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
+SED_ERE_OPT ?= -E
+
+ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
+ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
+ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
+ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
+ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
+ZSTD_FILES := $(ZSTDCOMMON_FILES)
+
+ifeq ($(findstring GCC,$(CCVER)),GCC)
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
+endif
+
 # Modules
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
-ZSTD_LIB_DEPRECATED ?= 0
+ZSTD_LIB_DEPRECATED ?= 1
+
+# Legacy support
+ZSTD_LEGACY_SUPPORT ?= 5
+ZSTD_LEGACY_MULTITHREADED_API ?= 0
+
+# Build size optimizations
+HUF_FORCE_DECOMPRESS_X1 ?= 0
+HUF_FORCE_DECOMPRESS_X2 ?= 0
+ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
+ZSTD_FORCE_DECOMPRESS_LONG ?= 0
+ZSTD_NO_INLINE ?= 0
+ZSTD_STRIP_ERROR_STRINGS ?= 0
 
-# Input variables for libzstd.mk
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
   ZSTD_LIB_DICTBUILDER = 0
   ZSTD_LIB_DEPRECATED = 0
@@ -25,46 +115,84 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
   ZSTD_LIB_DEPRECATED = 0
 endif
 
-include libzstd.mk
-
-ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES)
-
 ifneq ($(ZSTD_LIB_COMPRESSION), 0)
-  ZSTD_FILES += $(ZSTD_COMPRESS_FILES)
+  ZSTD_FILES += $(ZSTDCOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
-  ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES)
+  ZSTD_FILES += $(ZSTDDECOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DEPRECATED), 0)
-  ZSTD_FILES += $(ZSTD_DEPRECATED_FILES)
+  ZSTD_FILES += $(ZDEPR_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
-  ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES)
+  ZSTD_FILES += $(ZDICT_FILES)
 endif
 
+ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+endif
+
+ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
+endif
+
+ifneq ($(ZSTD_NO_INLINE), 0)
+  CFLAGS += -DZSTD_NO_INLINE
+endif
+
+ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
+  CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+endif
+
+ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
+  CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+endif
+
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+  ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+endif
+CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+
 ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
-ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o)
-ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o)
+ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
 
-VERSION := $(ZSTD_VERSION)
+ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
+vpath %.c $(ZSTD_SUBDIR)
 
-# Note: by default, the static library is built single-threaded and dynamic library is built
-# multi-threaded. It is possible to force multi or single threaded builds by appending
-# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
-.PHONY: default
-default: lib-release
+UNAME := $(shell uname)
 
-CPPFLAGS_DYNLIB  += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
-LDFLAGS_DYNLIB   += -pthread
-CPPFLAGS_STATLIB +=                    # static library build defaults to single-threaded
-
-
-ifeq ($(findstring GCC,$(CCVER)),GCC)
-decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
+ifndef BUILD_DIR
+ifeq ($(UNAME), Darwin)
+  HASH ?= md5
+else ifeq ($(UNAME), FreeBSD)
+  HASH ?= gmd5sum
+else ifeq ($(UNAME), NetBSD)
+  HASH ?= md5 -n
+else ifeq ($(UNAME), OpenBSD)
+  HASH ?= md5
 endif
+HASH ?= md5sum
+
+HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " )
+HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
+ifeq ($(HAVE_HASH),0)
+  $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
+  BUILD_DIR := obj/generic_noconf
+endif
+endif # BUILD_DIR
 
 
 # macOS linker doesn't support -soname, and use different extension
@@ -75,24 +203,12 @@ ifeq ($(UNAME), Darwin)
   SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
   SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
-  ifeq ($(UNAME), AIX)
-    SONAME_FLAGS =
-  else
-    SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
-  endif
+  SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
   SHARED_EXT = so
   SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
   SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
-
-.PHONY: all
-all: lib
-
-
-.PHONY: libzstd.a  # must be run every time
-libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB)
-
 SET_CACHE_DIRECTORY = \
    +$(MAKE) --no-print-directory $@ \
     BUILD_DIR=obj/$(HASH_DIR) \
@@ -100,6 +216,13 @@ SET_CACHE_DIRECTORY = \
     CFLAGS="$(CFLAGS)" \
     LDFLAGS="$(LDFLAGS)"
 
+
+.PHONY: all
+all: lib
+
+
+.PHONY: libzstd.a  # must be run every time
+
 ifndef BUILD_DIR
 # determine BUILD_DIR from compilation flags
 
@@ -115,10 +238,7 @@ ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
 $(ZSTD_STATLIB): ARFLAGS = rcs
 $(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
 $(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
-  # Check for multithread flag at target execution time
-	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
-    @echo compiling multi-threaded static library $(LIBVER),\
-    @echo compiling single-threaded static library $(LIBVER))
+	@echo compiling static library
 	$(AR) $(ARFLAGS) $@ $^
 
 libzstd.a: $(ZSTD_STATLIB)
@@ -137,9 +257,8 @@ else  # not Windows
 
 LIBZSTD = libzstd.$(SHARED_EXT_VER)
 .PHONY: $(LIBZSTD)  # must be run every time
-$(LIBZSTD): CPPFLAGS += $(CPPFLAGS_DYNLIB)
-$(LIBZSTD): CFLAGS   += -fPIC -fvisibility=hidden
-$(LIBZSTD): LDFLAGS  += -shared $(LDFLAGS_DYNLIB)
+$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden
+$(LIBZSTD): LDFLAGS += -shared
 
 ifndef BUILD_DIR
 # determine BUILD_DIR from compilation flags
@@ -156,10 +275,7 @@ ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
 
 $(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR)
 $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
-# Check for multithread flag at target execution time
-	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
-    @echo compiling multi-threaded dynamic library $(LIBVER),\
-    @echo compiling single-threaded dynamic library $(LIBVER))
+	@echo compiling dynamic library $(LIBVER)
 	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
 	ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
@@ -181,17 +297,10 @@ lib : libzstd.a libzstd
 # note : do not define lib-mt or lib-release as .PHONY
 # make does not consider implicit pattern rule for .PHONY target
 
-%-mt : CPPFLAGS_DYNLIB  := -DZSTD_MULTITHREAD
-%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD
-%-mt : LDFLAGS_DYNLIB   := -pthread
+%-mt : CPPFLAGS += -DZSTD_MULTITHREAD
+%-mt : LDFLAGS  += -pthread
 %-mt : %
-	@echo multi-threaded build completed
-
-%-nomt : CPPFLAGS_DYNLIB  :=
-%-nomt : LDFLAGS_DYNLIB   :=
-%-nomt : CPPFLAGS_STATLIB :=
-%-nomt : %
-	@echo single-threaded build completed
+	@echo multi-threading build completed
 
 %-release : DEBUGFLAGS :=
 %-release : %
@@ -210,14 +319,6 @@ $(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
 	@echo CC $@
 	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
 
-$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
-	@echo AS $@
-	$(COMPILE.c) $(OUTPUT_OPTION) $<
-
-$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR)
-	@echo AS $@
-	$(COMPILE.c) $(OUTPUT_OPTION) $<
-
 MKDIR ?= mkdir
 $(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
 	$(MKDIR) -p $@
@@ -231,8 +332,7 @@ include $(wildcard $(DEPFILES))
 # Special case : building library in single-thread mode _and_ without zstdmt_compress.c
 ZSTDMT_FILES = compress/zstdmt_compress.c
 ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
-libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
-libzstd-nomt: LDFLAGS += -shared
+libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
 libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo compiling single-thread dynamic library $(LIBVER)
 	@echo files : $(ZSTD_NOMT_FILES)
@@ -249,9 +349,9 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
-lib: libzstd.pc
+all: libzstd.pc
 
 HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,)
 
@@ -296,27 +396,25 @@ INSTALL_PROGRAM ?= $(INSTALL)
 INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
+libzstd.pc:
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
 	@sed $(SED_ERE_OPT) \
 	        -e 's|@PREFIX@|$(PREFIX)|' \
 	        -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
-	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
-	        -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
-	        -e 's|@VERSION@|$(VERSION)|' \
-	        -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
-	        $< >$@
+          -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
+          -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
+          -e 's|@VERSION@|$(VERSION)|' \
+          $< >$@
 
 .PHONY: install
 install: install-pc install-static install-shared install-includes
 	@echo zstd static and shared library installed
 
-.PHONY: install-pc
 install-pc: libzstd.pc
 	[ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
 	$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
 
-.PHONY: install-static
 install-static:
 	# only generate libzstd.a if it's not already present
 	[ -e libzstd.a ] || $(MAKE) libzstd.a-release
@@ -324,7 +422,6 @@ install-static:
 	@echo Installing static library
 	$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
 
-.PHONY: install-shared
 install-shared:
 	# only generate libzstd.so if it's not already present
 	[ -e $(LIBZSTD) ] || $(MAKE) libzstd-release
@@ -334,13 +431,12 @@ install-shared:
 	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
 	ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 
-.PHONY: install-includes
 install-includes:
 	[ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
 	@echo Installing includes
 	$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
-	$(INSTALL_DATA) zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
-	$(INSTALL_DATA) zdict.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+	$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
 
 .PHONY: uninstall
 uninstall:

lib/README.md

@@ -19,16 +19,12 @@ The scope can be reduced on demand (see paragraph _modular build_).
 
 #### Multithreading support
 
-When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons).
-
+Multithreading is disabled by default when building with `make`.
 Enabling multithreading requires 2 conditions :
 - set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
 - for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
 
-For convenience, we provide a build target to generate multi and single threaded libraries:
-- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
-- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
-- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.
+Both conditions are automatically applied when invoking `make lib-mt` target.
 
 When linking a POSIX program with a multithreaded version of `libzstd`,
 note that it's necessary to invoke the `-pthread` flag during link stage.
@@ -46,8 +42,8 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
 
 Optional advanced features are exposed via :
 
-- `lib/zstd_errors.h` : translates `size_t` function results
-                        into a `ZSTD_ErrorCode`, for accurate error handling.
+- `lib/common/zstd_errors.h` : translates `size_t` function results
+                               into a `ZSTD_ErrorCode`, for accurate error handling.
 
 - `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
                           it unlocks access to the experimental API,
@@ -125,7 +121,7 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`).
 
   Finally, when integrating into your application, make sure you're doing link-
-  time optimization and unused symbol garbage collection (via some combination of,
+  time optimation and unused symbol garbage collection (via some combination of,
   e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`,
   `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`,
   `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands
@@ -155,12 +151,6 @@ The file structure is designed to make this selection manually achievable for an
 - The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics.
   Compiler builtins are still used.
 
-- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control
-  the amount of extra memory used during decompression to store literals.
-  This defaults to 64kB.  Reducing this value reduces the memory footprint of
-  `ZSTD_DCtx` decompression contexts,
-  but might also result in a small decompression speed cost.
-
 
 #### Windows : using MinGW+MSYS to create DLL
 

lib/common/bitstream.h

@@ -143,16 +143,10 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
     {
 #   if defined(_MSC_VER)   /* Visual */
 #       if STATIC_BMI2 == 1
-            return _lzcnt_u32(val) ^ 31;
+		return _lzcnt_u32(val) ^ 31;
 #       else
-            if (val != 0) {
-                unsigned long r;
-                _BitScanReverse(&r, val);
-                return (unsigned)r;
-            } else {
-                /* Should not reach this code path */
-                __assume(0);
-            }
+		unsigned long r = 0;
+		return _BitScanReverse(&r, val) ? (unsigned)r : 0;
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
         return __builtin_clz (val) ^ 31;
@@ -338,16 +332,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
     U32 const regMask = sizeof(bitContainer)*8 - 1;
     /* if start > regMask, bitstream is corrupted, and result is undefined */
     assert(nbBits < BIT_MASK_SIZE);
-    /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
-     * than accessing memory. When bmi2 instruction is not present, we consider
-     * such cpus old (pre-Haswell, 2013) and their performance is not of that
-     * importance.
-     */
-#if defined(__x86_64__) || defined(_M_X86)
-    return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
-#else
     return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
-#endif
 }
 
 MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)

lib/common/compiler.h

@@ -11,8 +11,6 @@
 #ifndef ZSTD_COMPILER_H
 #define ZSTD_COMPILER_H
 
-#include "portability_macros.h"
-
 /*-*******************************************************
 *  Compiler specifics
 *********************************************************/
@@ -42,7 +40,7 @@
 
 /**
   On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
-  This explicitly marks such functions as __cdecl so that the code will still compile
+  This explictly marks such functions as __cdecl so that the code will still compile
   if a CC other than __cdecl has been made the default.
 */
 #if  defined(_MSC_VER)
@@ -94,17 +92,29 @@
 
 
 /* target attribute */
+#ifndef __has_attribute
+  #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
+#endif
 #if defined(__GNUC__) || defined(__ICCARM__)
 #  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
 #else
 #  define TARGET_ATTRIBUTE(target)
 #endif
 
-/* Target attribute for BMI2 dynamic dispatch.
- * Enable lzcnt, bmi, and bmi2.
- * We test for bmi1 & bmi2. lzcnt is included in bmi1.
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
  */
-#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
+#ifndef DYNAMIC_BMI2
+  #if ((defined(__clang__) && __has_attribute(__target__)) \
+      || (defined(__GNUC__) \
+          && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
+      && (defined(__x86_64__) || defined(_M_X86)) \
+      && !defined(__BMI2__)
+  #  define DYNAMIC_BMI2 1
+  #else
+  #  define DYNAMIC_BMI2 0
+  #endif
+#endif
 
 /* prefetch
  * can be disabled, by declaring NO_PREFETCH build macro */
@@ -140,9 +150,8 @@
 }
 
 /* vectorization
- * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
- * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
-#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
+ * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */
+#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__)
 #  if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
 #    define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
 #  else
@@ -188,20 +197,14 @@
     #define STATIC_BMI2 0
 #endif
 
-/* compile time determination of SIMD support */
-#if !defined(ZSTD_NO_INTRINSICS)
-#  if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
-#    define ZSTD_ARCH_X86_SSE2
-#  endif
-#  if defined(__ARM_NEON) || defined(_M_ARM64)
-#    define ZSTD_ARCH_ARM_NEON
-#  endif
-#
-#  if defined(ZSTD_ARCH_X86_SSE2)
-#    include <emmintrin.h>
-#  elif defined(ZSTD_ARCH_ARM_NEON)
-#    include <arm_neon.h>
-#  endif
+/* compat. with non-clang compilers */
+#ifndef __has_builtin
+#  define __has_builtin(x) 0
+#endif
+
+/* compat. with non-clang compilers */
+#ifndef __has_feature
+#  define __has_feature(x) 0
 #endif
 
 /* C-language Attributes are added in C23. */
@@ -231,47 +234,20 @@
 # elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
 #  define ZSTD_FALLTHROUGH [[fallthrough]]
 # elif __has_attribute(__fallthrough__)
-/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
- * gcc complains about: a label can only be part of a statement and a declaration is not a statement.
- */
-#  define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
+#  define ZSTD_FALLTHROUGH __attribute__((__fallthrough__))
 # else
 #  define ZSTD_FALLTHROUGH
 # endif
 #endif
 
-/*-**************************************************************
-*  Alignment check
-*****************************************************************/
-
-/* this test was initially positioned in mem.h,
- * but this file is removed (or replaced) for linux kernel
- * so it's now hosted in compiler.h,
- * which remains valid for both user & kernel spaces.
- */
-
-#ifndef ZSTD_ALIGNOF
-# if defined(__GNUC__) || defined(_MSC_VER)
-/* covers gcc, clang & MSVC */
-/* note : this section must come first, before C11,
- * due to a limitation in the kernel source generator */
-#  define ZSTD_ALIGNOF(T) __alignof(T)
-
-# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
-/* C11 support */
-#  include <stdalign.h>
-#  define ZSTD_ALIGNOF(T) alignof(T)
-
-# else
-/* No known support for alignof() - imperfect backup */
-#  define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
-
-# endif
-#endif /* ZSTD_ALIGNOF */
-
-/*-**************************************************************
-*  Sanitizer
-*****************************************************************/
+/* detects whether we are being compiled under msan */
+#ifndef ZSTD_MEMORY_SANITIZER
+#  if __has_feature(memory_sanitizer)
+#    define ZSTD_MEMORY_SANITIZER 1
+#  else
+#    define ZSTD_MEMORY_SANITIZER 0
+#  endif
+#endif
 
 #if ZSTD_MEMORY_SANITIZER
 /* Not all platforms that support msan provide sanitizers/msan_interface.h.
@@ -294,6 +270,17 @@ void __msan_poison(const volatile void *a, size_t size);
 intptr_t __msan_test_shadow(const volatile void *x, size_t size);
 #endif
 
+/* detects whether we are being compiled under asan */
+#ifndef ZSTD_ADDRESS_SANITIZER
+#  if __has_feature(address_sanitizer)
+#    define ZSTD_ADDRESS_SANITIZER 1
+#  elif defined(__SANITIZE_ADDRESS__)
+#    define ZSTD_ADDRESS_SANITIZER 1
+#  else
+#    define ZSTD_ADDRESS_SANITIZER 0
+#  endif
+#endif
+
 #if ZSTD_ADDRESS_SANITIZER
 /* Not all platforms that support asan provide sanitizers/asan_interface.h.
  * We therefore declare the functions we need ourselves, rather than trying to

lib/common/entropy_common.c

@@ -43,14 +43,8 @@ static U32 FSE_ctz(U32 val)
     assert(val != 0);
     {
 #   if defined(_MSC_VER)   /* Visual */
-        if (val != 0) {
-            unsigned long r;
-            _BitScanForward(&r, val);
-            return (unsigned)r;
-        } else {
-            /* Should not reach this code path */
-            __assume(0);
-        }
+        unsigned long r=0;
+        return _BitScanForward(&r, val) ? (unsigned)r : 0;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
         return __builtin_ctz(val);
 #   elif defined(__ICCARM__)    /* IAR Intrinsic */
@@ -223,7 +217,7 @@ static size_t FSE_readNCount_body_default(
 }
 
 #if DYNAMIC_BMI2
-BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
+TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2(
         short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
         const void* headerBuffer, size_t hbSize)
 {
@@ -305,7 +299,7 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
     ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
     weightTotal = 0;
     {   U32 n; for (n=0; n<oSize; n++) {
-            if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
+            if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
             rankStats[huffWeight[n]]++;
             weightTotal += (1 << huffWeight[n]) >> 1;
     }   }
@@ -343,7 +337,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r
 }
 
 #if DYNAMIC_BMI2
-static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      U32* nbSymbolsPtr, U32* tableLogPtr,
                      const void* src, size_t srcSize,
                      void* workSpace, size_t wkspSize)

lib/common/error_private.h

@@ -21,10 +21,8 @@ extern "C" {
 /* ****************************************
 *  Dependencies
 ******************************************/
-#include "../zstd_errors.h"  /* enum list */
-#include "compiler.h"
-#include "debug.h"
-#include "zstd_deps.h"       /* size_t */
+#include "zstd_deps.h"    /* size_t */
+#include "zstd_errors.h"  /* enum list */
 
 
 /* ****************************************
@@ -75,83 +73,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
     return ERR_getErrorString(ERR_getErrorCode(code));
 }
 
-/**
- * Ignore: this is an internal helper.
- *
- * This is a helper function to help force C99-correctness during compilation.
- * Under strict compilation modes, variadic macro arguments can't be empty.
- * However, variadic function arguments can be. Using a function therefore lets
- * us statically check that at least one (string) argument was passed,
- * independent of the compilation flags.
- */
-static INLINE_KEYWORD UNUSED_ATTR
-void _force_has_format_string(const char *format, ...) {
-  (void)format;
-}
-
-/**
- * Ignore: this is an internal helper.
- *
- * We want to force this function invocation to be syntactically correct, but
- * we don't want to force runtime evaluation of its arguments.
- */
-#define _FORCE_HAS_FORMAT_STRING(...) \
-  if (0) { \
-    _force_has_format_string(__VA_ARGS__); \
-  }
-
-#define ERR_QUOTE(str) #str
-
-/**
- * Return the specified error if the condition evaluates to true.
- *
- * In debug modes, prints additional information.
- * In order to do that (particularly, printing the conditional that failed),
- * this can't just wrap RETURN_ERROR().
- */
-#define RETURN_ERROR_IF(cond, err, ...) \
-  if (cond) { \
-    RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
-           __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
-    _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
-    RAWLOG(3, ": " __VA_ARGS__); \
-    RAWLOG(3, "\n"); \
-    return ERROR(err); \
-  }
-
-/**
- * Unconditionally return the specified error.
- *
- * In debug modes, prints additional information.
- */
-#define RETURN_ERROR(err, ...) \
-  do { \
-    RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
-           __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
-    _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
-    RAWLOG(3, ": " __VA_ARGS__); \
-    RAWLOG(3, "\n"); \
-    return ERROR(err); \
-  } while(0);
-
-/**
- * If the provided expression evaluates to an error code, returns that error code.
- *
- * In debug modes, prints additional information.
- */
-#define FORWARD_IF_ERROR(err, ...) \
-  do { \
-    size_t const err_code = (err); \
-    if (ERR_isError(err_code)) { \
-      RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
-             __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
-      _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
-      RAWLOG(3, ": " __VA_ARGS__); \
-      RAWLOG(3, "\n"); \
-      return err_code; \
-    } \
-  } while(0);
-
 #if defined (__cplusplus)
 }
 #endif

lib/common/fse.h

@@ -336,9 +336,8 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 /* FSE_buildCTable_wksp() :
  * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
  * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
- * See FSE_buildCTable_wksp() for breakdown of workspace usage.
  */
-#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2)))
 #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
 size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 

lib/common/fse_decompress.c

@@ -365,7 +365,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co
 }
 
 #if DYNAMIC_BMI2
-BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
+TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
 {
     return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
 }