minor refactor to blocksplit

fix issue 44108
credit to oss-fuzz In rare circumstances, the block-splitter might cut a block at the exact beginning of a repcode. In which case, since litlength=0, if the repcode expected 1+ literals in front, its signification changes. This scenario is controlled in ZSTD_seqStore_resolveOffCodes(), and the repcode is transformed into a raw offset when its new meaning is incorrect. In more complex scenarios, the previous block might be emitted as uncompressed after all, thus modifying the expected repcode history. In the case discovered by oss-fuzz, the first block is emitted as uncompressed, so the repcode history remains at default values: 1,4,8. But since the starting repcode is repcode3, and the literal length is == 0, its meaning is : = repcode1 - 1. Since repcode1==1, it results in an offset value of 0, which is invalid. So that's what the `assert()` was verifying : the result of the repcode translation should be a valid offset. But actually, it doesn't matter, because this result will then be compared to reality, and since it's an invalid offset, it will necessarily be discarded if incorrect, then the repcode will be replaced by a raw offset. So the `assert()` is not useful. Furthermore, it's incorrect, because it assumes this situation cannot happen, but it does, as described in above scenario.
2022-01-27 11:06:16 -08:00 · 2022-01-27 05:49:59 -08:00 · 2022-01-26 18:02:05 -08:00 · 2022-01-26 15:22:06 -08:00 · 2022-01-26 14:47:24 -08:00 · 2022-01-26 14:47:24 -08:00
307 changed files with 21763 additions and 8367 deletions
@@ -7,19 +7,21 @@ jobs:
    #       preinstalled to reduce installation time.
    docker:
      - image: fbopensource/zstd-circleci-primary:0.0.1
+    # TODO: Re-enable aarch64 build:
+    #       make aarch64build     && make clean
    steps:
      - checkout
      - run:
          name: Test
          command: |
            ./tests/test-license.py
-            cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
-            make c99build         ; make clean
-            make c11build         ; make clean
-            make aarch64build     ; make clean
+            cc -v
+            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
+            make c99build; make clean
+            make c11build; make clean
            make -j regressiontest; make clean
-            make shortest         ; make clean
-            make cxxtest          ; make clean
+            make shortest; make clean
+            make cxxtest; make clean
  # the second half of the jobs are in this test
  short-tests-1:
    docker:
@@ -84,31 +86,10 @@ workflows:
  commit:
    jobs:
      # Run the tests in parallel
-      - short-tests-0:
-          filters:
-            tags:
-              only: /.*/
-      - short-tests-1:
-          filters:
-            tags:
-              only: /.*/
-      # Create a branch called regression and set it to dev to force a
-      # regression test run
-      - regression-test:
-          filters:
-            branches:
-              only:
-                - regression
-      # Only run on release tags.
-      - publish-github-release:
-          requires:
-            - short-tests-0
-            - short-tests-1
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /^v\d+\.\d+\.\d+$/
+      - short-tests-0
+      - short-tests-1
+      - regression-test
+
  nightly:
    triggers:
      - schedule:
@@ -120,7 +101,7 @@ workflows:
                - dev
                - master
    jobs:
-      # Run daily long regression tests
+      # Run daily regression tests
      - regression-test


@@ -0,0 +1,240 @@
+name: dev-long-tests
+# Tests longer than 10mn
+
+concurrency:
+  group: long-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+
+jobs:
+  make-all:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make all
+      run: make all
+
+  # lasts ~24mn
+  make-test:
+    runs-on: ubuntu-latest
+    env:
+      DEVNULLRIGHTS: 1
+      READFROMBLOCKDEVICE: 1
+    steps:
+    - uses: actions/checkout@v2
+    - name: make test
+      run: make test
+
+  # lasts ~26mn
+  make-test-osx:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: OS-X test
+      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
+
+  no-intrinsics-fuzztest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: no intrinsics fuzztest
+      run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
+
+  tsan-zstreamtest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: thread sanitizer zstreamtest
+      run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
+
+  # lasts ~15mn
+  tsan-fuzztest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: thread sanitizer fuzztest
+      run: CC=clang make tsan-fuzztest
+
+  # lasts ~23mn
+  gcc-8-asan-ubsan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-8 + ASan + UBSan + Test Zstd
+      run: |
+        sudo apt-get -qqq update
+        make gcc8install
+        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
+
+  gcc-asan-ubsan-testzstd-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + Test Zstd, 32bit mode
+      run: |
+        make libc6install
+        make -j uasan-test-zstd32 V=1
+
+    # Note : external libraries must be turned off when using MSAN tests,
+    # because they are not msan-instrumented,
+    # so any data coming from these libraries is always considered "uninitialized"
+
+  gcc-8-asan-ubsan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-8 + ASan + UBSan + Fuzz Test
+      run: |
+        make gcc8install
+        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
+  gcc-asan-ubsan-fuzz32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + Fuzz Test 32bit
+      run: |
+        make libc6install
+        CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
+  asan-ubsan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ASan + UBSan + Regression Test
+      run: make -j uasanregressiontest
+
+  msan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: MSan + Regression Test
+      run: make -j msanregressiontest
+
+  clang-msan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + MSan + Fuzz Test
+      run: |
+        sudo apt-get -qqq update
+        sudo apt-get install clang
+        CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
+
+  # lasts ~24mn
+  clang-msan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + MSan + Test Zstd
+      run: |
+        sudo apt-get update
+        sudo apt-get install clang
+        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
+
+  armfuzz:
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v2
+      - name: Qemu ARM emulation + Fuzz Test
+        run: |
+          sudo apt-get -qqq update
+          make arminstall
+          make armfuzz
+
+  valgrind-fuzz-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: valgrind + fuzz test stack mode    # ~ 7mn
+      shell: 'script -q -e -c "bash {0}"'
+      run: |
+        make valgrindinstall
+        make -C tests valgrindTest
+        make clean
+        make -C tests test-fuzzer-stackmode
+
+  mingw-long-test:
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: [
+          { compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
+          { compiler: gcc, platform: x64, action: test, script: ""},
+        ]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Mingw long test
+      run: |
+        $env:PATH_ORIGINAL = $env:PATH
+        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
+        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
+        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
+        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
+        IF ("${{matrix.platform}}" -eq "x64")
+        {
+            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
+        }
+        ELSEIF ("${{matrix.platform}}" -eq "x86")
+        {
+            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
+        }
+        IF ("${{matrix.action}}" -eq "build")
+        {
+          make -v
+          sh -c "${{matrix.compiler}} -v"
+          ECHO "Building zlib to static link"
+          $env:CC = "${{matrix.compiler}}"
+          sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
+          sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
+          ECHO "Building zstd"
+          $env:CPPFLAGS = "-I../../zlib"
+          $env:LDFLAGS = "../../zlib/libz.a"
+          sh -c "${{matrix.script}}"
+        }
+        ELSEIF ("${{matrix.action}}" -eq "test")
+        {
+            ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
+            $env:CC = "gcc"
+            $env:CXX = "g++"
+            MKDIR build\cmake\build
+            CD build\cmake\build
+            $env:FUZZERTEST = "-T2mn"
+            $env:ZSTREAM_TESTTIME = "-T2mn"
+            cmake -G "Visual Studio 14 2015 Win64" ..
+            cd ..\..\..
+            make clean
+        }
+
+  # lasts ~20mn
+  oss-fuzz:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address, undefined, memory]
+    steps:
+    - name: Build Fuzzers (${{ matrix.sanitizer }})
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'zstd'
+        dry-run: false
+        sanitizer: ${{ matrix.sanitizer }}
+    - name: Run Fuzzers (${{ matrix.sanitizer }})
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'zstd'
+        fuzz-seconds: 600
+        dry-run: false
+        sanitizer: ${{ matrix.sanitizer }}
+    - name: Upload Crash
+      uses: actions/upload-artifact@v1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: ${{ matrix.sanitizer }}-artifacts
+        path: ./out/artifacts
@@ -0,0 +1,448 @@
+name: dev-short-tests
+# Faster tests: mostly build tests, along with some other
+# misc tests
+
+concurrency:
+  group: fast-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ dev, release, actionsTest ]
+
+jobs:
+  linux-kernel:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: linux kernel, library + build + test
+      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
+
+  benchmarking:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make benchmarking
+      run: make benchmarking
+
+  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make check on 32-bit
+      run: |
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
+        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
+
+  check-x32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
+      env:
+        CHECK_CONSTRAINED_MEM: true
+      run: |
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
+        CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
+
+  gcc-7-libzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: gcc-7 + libzstdmt compilation
+      run: |
+        sudo apt-get -qqq update
+        make gcc7install
+        CC=gcc-7 CFLAGS=-Werror make -j all
+        make clean
+        LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
+
+    # candidate test (to check) : underlink test
+    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
+
+  cmake-build-and-test-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: cmake build and test check
+      run: |
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+        cp -r ./ "../zstd source"
+        cd "../zstd source"
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+
+  cpp-gnu90-c99-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: C++, gnu90 and c99 compatibility
+      run: |
+        make cxxtest
+        make clean
+        make gnu90build
+        make clean
+        make c99build
+        make clean
+        make travis-install   # just ensures `make install` works
+
+  mingw-cross-compilation:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: mingw cross-compilation
+      run: |
+        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
+        sudo apt-get -qqq update
+        sudo apt-get install gcc-mingw-w64
+        CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
+
+  armbuild:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: ARM Build Test
+      run: |
+        sudo apt-get -qqq update
+        make arminstall
+        make armbuild
+
+  bourne-shell:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bourne shell compatibility (shellcheck)
+      run: |
+        wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
+        tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
+        shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
+
+  zlib-wrapper:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: zlib wrapper test
+      run: |
+        sudo apt-get -qqq update
+        make valgrindinstall
+        make -C zlibWrapper test
+        make -C zlibWrapper valgrindTest
+
+  lz4-threadpool-libs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: LZ4, thread pool, and libs build testslib wrapper test
+      run: |
+        make lz4install
+        make -C tests test-lz4
+        make check < /dev/null | tee   # mess with lz4 console detection
+        make clean
+        make -C tests test-pool
+        make clean
+        bash tests/libzstd_builds.sh
+
+  gcc-make-tests-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Make all, 32bit mode
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CFLAGS="-Werror -m32" make -j all32
+
+  gcc-8-make:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: gcc-8 build
+        run: |
+          sudo apt-get -qqq update
+          make gcc8install
+          CC=gcc-8 CFLAGS="-Werror" make -j all
+
+  implicit-fall-through:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: -Wimplicit-fallthrough build
+        run: |
+          make clean
+          CC=gcc MOREFLAGS="-Werror -Wimplicit-fallthrough=2 -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
+          make clean
+          CC=clang MOREFLAGS="-Werror -Wimplicit-fallthrough -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
+
+  cmake-visual-2019:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        include:
+          - generator: "Visual Studio 16 2019"
+            flags: "-A x64"
+          - generator: "Visual Studio 16 2019"
+            flags: "-A Win32"
+          - generator: "MinGW Makefiles"
+    steps:
+    - uses: actions/checkout@v2
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@v1.0.2
+    - name: Build
+      working-directory: ${{env.GITHUB_WORKSPACE}}
+      run: |
+        cd build\cmake
+        mkdir build
+        cd build
+        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
+        cmake.exe --build .
+
+  visual-2019:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        platform: [x64, Win32]
+        configuration: [Debug, Release]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@v1.0.2
+    - name: Build
+      working-directory: ${{env.GITHUB_WORKSPACE}}
+      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
+      run: >
+        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142
+        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
+
+  visual-2015:
+    # only GH actions windows-2016 contains VS 2015
+    runs-on: windows-2016
+    strategy:
+      matrix:
+        platform: [x64, Win32]
+        configuration: [Debug, Release]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@v1.0.2
+    - name: Build
+      working-directory: ${{env.GITHUB_WORKSPACE}}
+      run: >
+        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
+        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
+
+  minimal-decompressor-macros:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: minimal decompressor macros
+      run: |
+        make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
+        make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
+        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+        make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+
+  dynamic-bmi2:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: dynamic bmi2 tests
+      run: |
+        make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
+        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1"
+        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1 -mbmi2"
+        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0"
+        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0 -mbmi2"
+
+  test-variants:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make all variants & validate
+      run: |
+        make -j -C programs allVariants MOREFLAGS=-O0
+        ./tests/test-variants.sh
+
+
+  qemu-consistency:
+    name: QEMU ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
+      matrix:
+        include: [
+          { name: ARM,      xcc_pkg: gcc-arm-linux-gnueabi,     xcc: arm-linux-gnueabi-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-arm-static     },
+          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-aarch64-static },
+          { name: PPC,      xcc_pkg: gcc-powerpc-linux-gnu,     xcc: powerpc-linux-gnu-gcc,     xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc-static     },
+          { name: PPC64LE,  xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc64le-static },
+          { name: S390X,    xcc_pkg: gcc-s390x-linux-gnu,       xcc: s390x-linux-gnu-gcc,       xemu_pkg: qemu-system-s390x,  xemu: qemu-s390x-static   },
+          { name: MIPS,     xcc_pkg: gcc-mips-linux-gnu,        xcc: mips-linux-gnu-gcc,        xemu_pkg: qemu-system-mips,   xemu: qemu-mips-static    },
+          { name: M68K,     xcc_pkg: gcc-m68k-linux-gnu,        xcc: m68k-linux-gnu-gcc,        xemu_pkg: qemu-system-m68k,   xemu: qemu-m68k-static    },
+        ]
+    env:                        # Set environment variables
+      XCC: ${{ matrix.xcc }}
+      XEMU: ${{ matrix.xemu }}
+    steps:
+    - uses: actions/checkout@v2 # https://github.com/actions/checkout
+    - name: apt update & install
+      run: |
+        sudo apt-get update
+        sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
+        sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
+    - name: Environment info
+      run: |
+        echo && which $XCC
+        echo && $XCC --version
+        echo && $XCC -v  # Show built-in specs
+        echo && which $XEMU
+        echo && $XEMU --version
+    - name: ARM
+      if: ${{ matrix.name == 'ARM' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: ARM64
+      if: ${{ matrix.name == 'ARM64' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: PPC
+      if: ${{ matrix.name == 'PPC' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: PPC64LE
+      if: ${{ matrix.name == 'PPC64LE' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: S390X
+      if: ${{ matrix.name == 'S390X' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: MIPS
+      if: ${{ matrix.name == 'MIPS' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+    - name: M68K
+      if: ${{ matrix.name == 'M68K' }}
+      run: |
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
+
+  mingw-short-test:
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: [
+          { compiler: gcc, platform: x64, script: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"},
+          { compiler: gcc, platform: x86, script: "CFLAGS=-Werror make -j allzstd"},
+          { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
+        ]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Mingw short test
+      run: |
+        ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
+        $env:PATH_ORIGINAL = $env:PATH
+        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
+        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
+        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
+        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
+        IF ("${{matrix.platform}}" -eq "x64")
+        {
+            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
+        }
+        ELSEIF ("${{matrix.platform}}" -eq "x86")
+        {
+            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
+        }
+        make -v
+        sh -c "${{matrix.compiler}} -v"
+        $env:CC = "${{matrix.compiler}}"
+        sh -c "${{matrix.script}}"
+        ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
+        make clean
+        make check
+
+
+  visual-runtime-tests:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        platform: [x64, Win32]
+        configuration: [Release]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@v1.0.2
+    - name: Build and run tests
+      working-directory: ${{env.GITHUB_WORKSPACE}}
+      env:
+        ZSTD_BIN: ./zstd.exe
+        DATAGEN_BIN: ./datagen.exe
+      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
+      run: |
+        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142 /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
+        COPY build\VS2010\bin\${{matrix.platform}}_${{matrix.configuration}}\*.exe tests\
+        CD tests
+        sh -e playTests.sh
+        DIR
+        .\fuzzer.exe -T2m
+
+  intel-cet-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build Zstd
+      run: |
+        make -j zstd V=1
+        readelf -n zstd
+    - name: Get Intel SDE
+      run: |
+        curl -LO https://downloadmirror.intel.com/684899/sde-external-9.0.0-2021-11-07-lin.tar.xz
+        tar xJvf sde-external-9.0.0-2021-11-07-lin.tar.xz
+    - name: Configure Permissions
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+    - name: Run Under SDE
+      run: |
+        sde-external-9.0.0-2021-11-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
+
+
+
+# This test currently fails on Github Actions specifically.
+# Possible reason : TTY emulation.
+# Note that the same test works fine locally and on travisCI.
+# This will have to be fixed before transferring the test to GA.
+#  versions-compatibility:
+#    runs-on: ubuntu-latest
+#    steps:
+#    - uses: actions/checkout@v2
+#    - name: Versions Compatibility Test
+#      run: |
+#        make -C tests versionsTest
+
+
+# For reference : icc tests
+# icc tests are currently failing on Github Actions, likely to issues during installation stage
+# To be fixed later
+#
+#  icc:
+#    name: icc-check
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: install icc
+#      run: |
+#        export DEBIAN_FRONTEND=noninteractive
+#        sudo apt-get -qqq update
+#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
+#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+#        sudo apt-get update
+#        sudo apt-get install -y intel-basekit intel-hpckit
+#    - uses: actions/checkout@v2
+#    - name: make check
+#      run: |
+#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
@@ -1,214 +0,0 @@
-name: generic-dev
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-
-# Dev PR jobs that still have to be migrated from travis
-#
-# versionTag (only on release tags)
-# valgrindTest (keeps failing for some reason. need investigation)
-# staticAnalyze (need trusty so need self-hosted)
-# pcc-fuzz: (need trusty so need self-hosted)
-# min-decomp-macros (flakey)
-#
-# setting up self-hosted is pretty straightforward, but
-# I need admins permissions to the repo for that it looks like
-# So I'm tabling that for now
-#
-# The release branch exclusive jobs will be in a separate
-# workflow file (the osx tests and meson build that is)
-
-  benchmarking:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make benchmarking
-      run: make benchmarking
-
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make test
-      run: make test
-
-  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make check on 32-bit
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
-
-  gcc-7-libzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-7 + libzstdmt compilation
-      run: |
-        make gcc7install
-        CC=gcc-7 CFLAGS=-Werror make -j all
-        make clean
-        LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
-        make -C tests zbufftest-dll
-
-    # candidate test (to check) : underlink test
-    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
-    # zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
-
-  gcc-8-asan-ubsan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Test Zstd
-      run: |
-        make gcc8install
-        CC=gcc-8 CFLAGS="-Werror" make -j all
-        make clean
-        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
-
-  gcc-asan-ubsan-testzstd-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Test Zstd, 32bit mode
-      run: |
-        make libc6install
-        CFLAGS="-Werror -m32" make -j all32
-        make clean
-        make -j uasan-test-zstd32 V=1
-
-  clang-msan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang + MSan + Test Zstd
-      run: |
-        sudo apt-get update
-        sudo apt-get install clang
-        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
-
-    # Note : external libraries must be turned off when using MSAN tests,
-    # because they are not msan-instrumented,
-    # so any data coming from these libraries is always considered "uninitialized"
-
-  cmake-build-and-test-check:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: cmake build and test check
-      run: |
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-        cp -r ./ "../zstd source"
-        cd "../zstd source"
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-
-  gcc-8-asan-ubsan-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Fuzz Test
-      run: |
-        make gcc8install
-        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
-
-  gcc-asan-ubsan-fuzz32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Fuzz Test 32bit
-      run: |
-        make libc6install
-        CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
-
-  asan-ubsan-msan-regression:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + MSan + Regression Test
-      run: |
-        make -j uasanregressiontest
-        make clean
-        make -j msanregressiontest
-
-  cpp-gnu90-c99-compatibility:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: C++, gnu90 and c99 compatibility
-      run: |
-        make cxxtest
-        make clean
-        make gnu90build
-        make clean
-        make c99build
-        make clean
-        make travis-install   # just ensures `make install` works
-
-  mingw-cross-compilation:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: mingw cross-compilation
-      run: |
-        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
-        sudo apt-get install gcc-mingw-w64
-        CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
-
-# TODO: Broken test - fix and uncomment
-#  armbuild:
-#    runs-on: ubuntu-16.04 # doesn't work on latest
-#    steps:
-#    - uses: actions/checkout@v2
-#    - name: ARM Build Test
-#      run: |
-#        make arminstall
-#        make armbuild
-
-# TODO: Broken test - fix and uncomment
-#  armfuzz:
-#    runs-on: ubuntu-16.04 # doesn't work on latest
-#    steps:
-#    - uses: actions/checkout@v2
-#    - name: Qemu ARM emulation + Fuzz Test
-#      run: |
-#        make arminstall
-#        make armfuzz
-
-  bourne-shell:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Bourne shell compatibility (shellcheck)
-      run: |
-        wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
-        tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
-        shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
-
-# For reference : icc tests
-# icc tests are currently failing on Github Actions, likely to issues during installation stage
-# To be fixed later
-#
-#  icc:
-#    name: icc-check
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: install icc
-#      run: |
-#        export DEBIAN_FRONTEND=noninteractive
-#        sudo apt-get -qqq update
-#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-#        sudo apt-get update
-#        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v2
-#    - name: make check
-#      run: |
-#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
@@ -1,67 +0,0 @@
-name: generic-release
-
-on:
-  pull_request:
-    # This will eventually only be for pushes to release
-    # but for dogfooding purposes, I'm running it even
-    # on dev pushes
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  # missing jobs
-  #
-  # ppc64le + fuzz test
-  # Qemu PPC64 + Fuzz test
-  # Qemu aarch64 + Fuzz Test (on Xenial)
-  # versions comp
-  # meson test
-
-  osx:
-    runs-on: macos-10.15
-    steps:
-    - uses: actions/checkout@v2
-    - name: OS-X
-      run: |
-        make test
-        # make -c lib all (need to fix. not working right now)
-
-  zbuff:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: zbuff test
-      run: |
-        make -C tests test-zbuff
-
-  tsan:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: thread sanitizer
-      run: |
-        CC=clang make tsan-test-zstream
-        CC=clang make tsan-fuzztest
-
-  zlib-wrapper:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: zlib wrapper test
-      run: |
-        make valgrindinstall
-        make -C zlibWrapper test
-        make -C zlibWrapper valgrindTest
-
-  lz4-threadpool-partial-libs:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: LZ4, thread pool, and partial libs testslib wrapper test
-      run: |
-        make lz4install
-        make -C tests test-lz4
-        make check < /dev/null | tee   # mess with lz4 console detection
-        make clean
-        make -C tests test-pool
-        make clean
-        bash tests/libzstd_partial_builds.sh
@@ -1,13 +0,0 @@
-name: linux-kernel
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Create linux kernel library + build + test
-      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
@@ -1,30 +0,0 @@
-name: CIFuzz
-on: [pull_request]
-jobs:
-  Fuzzing:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        sanitizer: [address, undefined]
-    steps:
-    - name: Build Fuzzers (${{ matrix.sanitizer }})
-      id: build
-      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Run Fuzzers (${{ matrix.sanitizer }})
-      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        fuzz-seconds: 600
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Upload Crash
-      uses: actions/upload-artifact@v1
-      if: failure() && steps.build.outcome == 'success'
-      with:
-        name: ${{ matrix.sanitizer }}-artifacts
-        path: ./out/artifacts
@@ -3,7 +3,7 @@ name: publish-release-artifacts
 on:
  release:
    types:
-      - created
+      - published

 jobs:
  publish-release-artifacts:
@@ -29,6 +29,7 @@ dictionary
 NUL

 # Build artefacts
+contrib/linux-kernel/linux/
 projects/
 bin/
 .buckd/
@@ -50,3 +51,5 @@ googletest/
 *.code-workspace
 compile_commands.json
 .clangd
+perf.data
+perf.data.old
@@ -1,5 +1,6 @@
-# Medium Tests: Run on all commits/PRs to dev branch
-
+# Travis CI is used to test platforms that github-actions currently doesn't support
+# without either self-hosting or some finnicky work-around. Also, some tests
+# are troublesome to migrate since GH Actions runs tests not in a tty.
 language: c

 git:
@@ -18,47 +19,48 @@ addons:

 env:
  global:
-    - FUZZERTEST=-T2mn
-      ZSTREAM_TESTTIME=-T2mn
+    - FUZZERTEST=-T1mn
+      ZSTREAM_TESTTIME=-T1mn
      DECODECORPUS_TESTTIME=-T1mn

-
 matrix:
  fast_finish: true
  include:
+    - name: S390X (big endian) + Fuzz test
+      dist: trusty
+      arch: s390x
+      script:
+        - FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
+
+    - name: S390X (big endian) + Fuzz test + no intrinsics
+      dist: trusty
+      arch: s390x
+      script:
+        - MOREFLAGS="-DZSTD_NO_INTRINSICS" FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
+
    - name: arm64    # ~2.5 mn
      os: linux
      arch: arm64
      script:
        - make check

-    - name: Minimal Decompressor Macros    # ~5mn
+    - name: arm64fuzz
+      os: linux
+      arch: arm64
      script:
-        - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-        - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+        - make -C tests fuzztest

-    - name: static analyzer scanbuild    # ~26mn
+    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
+    - name: static analyzer scanbuild    # ~8mn
      dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
      script:
        - make staticAnalyze

-    - name: Valgrind + Fuzz Test Stack Mode    # ~ 7mn
+    # GH actions can't run this command on OS-X, non-tty issues
+    - name: OS-X make all lib
+      os: osx
      script:
-        - make valgrindinstall
-        - make -C tests clean valgrindTest
-        - make clean
-        - make -C tests test-fuzzer-stackmode
-
-    - name: Qemu ARM emulation + Fuzz Test    # ~13.5mn
-      script:
-        - make arminstall
-        - make armfuzz
+        - make -C lib all

    # Introduced to check compat with old toolchains, to prevent e.g. #1872
    - name: ARM Build Test (on Trusty)
@@ -67,65 +69,31 @@ matrix:
        - make arminstall
        - make armbuild

-    - name: Qemu PPC + Fuzz Test    # ~13mn
-      dist: trusty   # it seems ppc cross-compilation fails on "current"
-      script:
-        - make ppcinstall
-        - make ppcfuzz
-
-    # check release number (release only)
+    # check release number (release/new tag only)
    - name: Tag-Specific Test
      if: tag =~ ^v[0-9]\.[0-9]
      script:
        - make -C tests checkTag
        - tests/checkTag "$TRAVIS_BRANCH"

-    # tests for release branch and cron job only
-    - name: OS-X    # ~13mn
-      if: branch = release
-      os: osx
+    - name: PPC64LE + Fuzz test  # ~13mn
+      arch: ppc64le
+      env:
+        - FUZZER_FLAGS=--no-big-tests
+        - MOREFLAGS="-static"
      script:
-        - make test
-        - make -C lib all
+        - cat /proc/cpuinfo
+        - make -C tests fuzztest

-    - name: Versions Compatibility Test   # 11.5mn
-      if: branch = release
+    # This test currently fails on GA specifically, for no obvious reason
+    # (it works fine on travisCI, and on local test platforms).
+    - name: Versions Compatibility Test   # ~6mn
      script:
        - make -C tests versionsTest

-    - name: thread sanitizer   # ~29mn
-      if: branch = release
-      script:
-        - make clang38install
-        - CC=clang-3.8 make tsan-test-zstream
-        - CC=clang-3.8 make tsan-fuzztest
-
-    - name: PPC64LE + Fuzz test  # ~13mn
-      if: branch = release
-      arch: ppc64le
-      script:
-        - cat /proc/cpuinfo
-        - make test
-
-    - name: Qemu PPC64 + Fuzz test  # ~13mn, presumed Big-Endian (?)
-      dist: trusty  # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
-      if: branch = release
-      script:
-        - make ppcinstall
-        - make ppc64fuzz
-
-    # note : we already have aarch64 tests on hardware
-    - name: Qemu aarch64 + Fuzz Test (on Xenial)    # ~14mn
-      if: branch = release
-      dist: xenial
-      script:
-        - make arminstall
-        - make aarch64fuzz
-
    # meson dedicated test
    - name: Xenial (Meson + clang)    # ~15mn
-      if: branch = release
-      dist: xenial
+      dist: bionic
      language: cpp
      compiler: clang
      install:
@@ -1,3 +1,82 @@
+v1.5.2 (Jan, 2022)
+perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969)
+build: Build Zstd with `noexecstack` on All Architectures (@felixhandte, #2964)
+doc: Clarify Licensing (@terrelln, #2981)
+
+v1.5.1 (Dec, 2021)
+perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42
+perf: faster huffman decoder, using x64 assembly, by @terrelln
+perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte
+perf: improved binary size and faster compilation times, by @terrelln
+perf: new row64 mode, used notably in level 12, by @senhuang42
+perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42
+perf: minor compression ratio improvements for small data at high levels, by @cyan4973
+perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln
+perf: faster compression speed on incompressible data, by @bindhvo
+perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo
+build: allows hiding static symbols in the dynamic library, using build macro, by @skitt
+build: support for m68k (Motorola 68000's), by @cyan4973
+build: improved AIX support, by @Helflym
+build: improved meson unofficial build, by @eli-schwartz
+cli : custom memory limit when training dictionary (#2925), by @embg
+cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB
+
+v1.5.0  (May 11, 2021)
+api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
+  `ZSTD_defaultCLevel()`
+  `ZSTD_getDictID_fromCDict()`
+api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42)
+  `ZSTD_compress_advanced()`
+  `ZSTD_compress_usingCDict_advanced()`
+  `ZSTD_compressBegin_advanced()`
+  `ZSTD_compressBegin_usingCDict_advanced()`
+  `ZSTD_initCStream_srcSize()`
+  `ZSTD_initCStream_usingDict()`
+  `ZSTD_initCStream_usingCDict()`
+  `ZSTD_initCStream_advanced()`
+  `ZSTD_initCStream_usingCDict_advanced()`
+  `ZSTD_resetCStream()`
+api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973)
+perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln)
+perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42)
+perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973)
+perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln)
+bug: Improve setting permissions of created files (#2525, @felixhandte)
+bug: Fix large dictionary non-determinism (#2607, @terrelln)
+bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln)
+bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte)
+bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42)
+bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink)
+bug: Fix UBSAN error in decompression (#2625, @terrelln)
+bug: Fix superblock compression divide by zero bug (#2592, @senhuang42)
+bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone)
+doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln)
+doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize)
+doc: Add style guide docs for open source contributors (#2626, @Cyan4973)
+tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42)
+tests: Better test coverage of index reduction (#2603, @terrelln)
+tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42)
+tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42)
+build: Dynamic library built multithreaded by default (#2584, @senhuang42)
+build: Move  `zstd_errors.h`  and  `zdict.h`  to  `lib/`  root (#2597, @terrelln)
+build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973)
+build: Single file library build script moved to `build/` directory (#2618, @felixhandte)
+build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42)
+build: Fixed Meson build (#2548, @SupervisedThinking @kloczek)
+build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson)
+build: Detect presence of `md5` on Darwin (#2609, @felixhandte)
+build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann)
+cli: `--progress` flag added to always display progress bar (#2595, @senhuang42)
+cli: Allow reading from block devices with `--force` (#2613, @felixhandte)
+cli: Fix CLI filesize display bug (#2550, @Cyan4973)
+cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973)
+contrib: Various fixes for linux kernel patch (#2539, @terrelln)
+contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42)
+contrib: Seekable format - New seek table-only API  (#2113 #2518, @mdittmer @Cyan4973)
+contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng)
+contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat)
+misc: Automatically published release tarballs available on Github (#2535, @felixhandte)
+
 v1.4.9  (Mar 1, 2021)
 bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
 bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
@@ -47,7 +47,7 @@ Our contribution process works in three main stages:
    * Topic and development:
        * Make a new branch on your fork about the topic you're developing for
        ```
-        # branch names should be consise but sufficiently informative
+        # branch names should be concise but sufficiently informative
        git checkout -b <branch-name>
        git push origin <branch-name>
        ```
@@ -68,8 +68,8 @@ Our contribution process works in three main stages:
            ```
 2. Code Review and CI tests
    * Ensure CI tests pass:
-        * Before sharing anything to the community, make sure that all CI tests pass on your local fork.
-        See our section on setting up your CI environment for more information on how to do this.
+        * Before sharing anything to the community, create a pull request in your own fork against the dev branch
+        and make sure that all GitHub Actions CI tests pass. See the Continuous Integration section below for more information.
        * Ensure that static analysis passes on your development machine. See the Static Analysis section
        below to see how to do this.
    * Create a pull request:
@@ -104,7 +104,7 @@ Our contribution process works in three main stages:
        issue at hand, then please indicate this by requesting that an issue be closed by commenting.
        * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
        that the change must make it to an official zstd release for it to be meaningful. We recommend
-        that contributers track the activity on their pull request and corresponding issue(s) page(s) until
+        that contributors track the activity on their pull request and corresponding issue(s) page(s) until
        their change makes it to the next release of zstd. Users will often discover bugs in your code or
        suggest ways to refine and improve your initial changes even after the pull request is merged.

@@ -140,6 +140,42 @@ It can be useful to look at additional static analyzers once in a while (and we

 This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.

+## Continuous Integration
+CI tests run every time a pull request (PR) is created or updated. The exact tests
+that get run will depend on the destination branch you specify. Some tests take
+longer to run than others. Currently, our CI is set up to run a short
+series of tests when creating a PR to the dev branch and a longer series of tests
+when creating a PR to the release branch. You can look in the configuration files
+of the respective CI platform for more information on what gets run when.
+
+Most people will just want to create a PR with the destination set to their local dev
+branch of zstd. You can then find the status of the tests on the PR's page. You can also
+re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
+
+Almost all of zstd's CI runs on GitHub Actions (configured at `.github/workflows`), which will automatically run on PRs to your
+own fork. A small number of tests run on other services (e.g. Travis CI, Circle CI, Appveyor).
+These require work to set up on your local fork, and (at least for Travis CI) cost money.
+Therefore, if the PR on your local fork passes GitHub Actions, feel free to submit a PR
+against the main repo.
+
+### Third-party CI
+A small number of tests cannot run on GitHub Actions, or have yet to be migrated.
+For these, we use a variety of third-party services (listed below). It is not necessary to set
+these up on your fork in order to contribute to zstd; however, we do link to instructions for those
+who want earlier signal.
+
+| Service   | Purpose                                                                                                    | Setup Links                                                                                                                                            | Config Path            |
+|-----------|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
+| Travis CI | Used for testing on non-x86 architectures such as PowerPC                                                  | https://docs.travis-ci.com/user/tutorial/#to-get-started-with-travis-ci-using-github <br> https://github.com/marketplace/travis-ci                     | `.travis.yml`          |
+| AppVeyor  | Used for some Windows testing (e.g. cygwin, mingw)                                                         | https://www.appveyor.com/blog/2018/10/02/github-apps-integration/ <br> https://github.com/marketplace/appveyor                                         | `appveyor.yml`         |
+| Cirrus CI | Used for testing on FreeBSD                                                                                | https://github.com/marketplace/cirrus-ci/                                                                                                              | `.cirrus.yml`          |
+| Circle CI | Historically was used to provide faster signal,<br/> but we may be able to migrate these to Github Actions | https://circleci.com/docs/2.0/getting-started/#setting-up-circleci <br> https://youtu.be/Js3hMUsSZ2c <br> https://circleci.com/docs/2.0/enable-checks/ | `.circleci/config.yml` |
+
+Note: the instructions linked above mostly cover how to set up a repository with CI from scratch. 
+The general idea should be the same for setting up CI on your fork of zstd, but you may have to 
+follow slightly different steps. In particular, please ignore any instructions related to setting up
+config files (since zstd already has configs for each of these services).
+
 ## Performance
 Performance is extremely important for zstd and we only merge pull requests whose performance
 landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
@@ -270,15 +306,15 @@ for level 1 compression on Zstd. Typically this means, you have identified a sec
 code that you think can be made to run faster.

 The first thing you will want to do is make sure that the piece of code is actually taking up
-a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
+a notable amount of time to run. It is usually not worth optimizing something which accounts for less than
 0.0001% of the total running time. Luckily, there are tools to help with this.
 Profilers will let you see how much time your code spends inside a particular function.
-If your target code snippit is only part of a function, it might be worth trying to
-isolate that snippit by moving it to its own function (this is usually not necessary but
+If your target code snippet is only part of a function, it might be worth trying to
+isolate that snippet by moving it to its own function (this is usually not necessary but
 might be).

-Most profilers (including the profilers dicusssed below) will generate a call graph of
-functions for you. Your goal will be to find your function of interest in this call grapch
+Most profilers (including the profilers discussed below) will generate a call graph of
+functions for you. Your goal will be to find your function of interest in this call graph
 and then inspect the time spent inside of it. You might also want to to look at the
 annotated assembly which most profilers will provide you with.

@@ -301,16 +337,16 @@ $ zstd -b1 -i5 <my-data> # this will run for 5 seconds
 5. Once you run your benchmarking script, switch back over to instruments and attach your
 process to the time profiler. You can do this by:
    * Clicking on the `All Processes` drop down in the top left of the toolbar.
-    * Selecting your process from the dropdown. In my case, it is just going to be labled
+    * Selecting your process from the dropdown. In my case, it is just going to be labeled
    `zstd`
    * Hitting the bright red record circle button on the top left of the toolbar
-6. You profiler will now start collecting metrics from your bencharking script. Once
+6. You profiler will now start collecting metrics from your benchmarking script. Once
 you think you have collected enough samples (usually this is the case after 3 seconds of
 recording), stop your profiler.
 7. Make sure that in toolbar of the bottom window, `profile` is selected.
 8. You should be able to see your call graph.
    * If you don't see the call graph or an incomplete call graph, make sure you have compiled
-    zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
+    zstd and your benchmarking script using debug flags. On mac and linux, this just means
    you will have to supply the `-g` flag alone with your build script. You might also
    have to provide the `-fno-omit-frame-pointer` flag
 9. Dig down the graph to find your function call and then inspect it by double clicking
@@ -329,7 +365,7 @@ Some general notes on perf:
 counter statistics. Perf uses a high resolution timer and this is likely one
 of the first things your team will run when assessing your PR.
 * Perf has a long list of hardware counters that can be viewed with `perf --list`.
-When measuring optimizations, something worth trying is to make sure the handware
+When measuring optimizations, something worth trying is to make sure the hardware
 counters you expect to be impacted by your change are in fact being so. For example,
 if you expect the L1 cache misses to decrease with your change, you can look at the
 counter `L1-dcache-load-misses`
@@ -339,57 +375,6 @@ counter `L1-dcache-load-misses`

 TODO

-
-## Setting up continuous integration (CI) on your fork
-Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
-are well tested before they make it to an official release. Specifically, we use the platforms
-travis-ci, circle-ci, and appveyor.
-
-Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
-The easiest way to run these CI tests on your own before submitting a PR to our dev branch
-is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
-instructions for doing this.
-
-### travis-ci
-Follow these steps to link travis-ci with your github fork of zstd
-
-1. Make sure you are logged into your github account
-2. Go to https://travis-ci.org/
-3. Click 'Sign in with Github' on the top right
-4. Click 'Authorize travis-ci'
-5. Click 'Activate all repositories using Github Apps'
-6. Select 'Only select repositories' and select your fork of zstd from the drop down
-7. Click 'Approve and Install'
-8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
-9. Click 'Authorize travis-pro'
-10. You should have travis set up on your fork now.
-
-### circle-ci
-TODO
-
-### appveyor
-Follow these steps to link circle-ci with your girhub fork of zstd
-
-1. Make sure you are logged into your github account
-2. Go to https://www.appveyor.com/
-3. Click 'Sign in' on the top right
-4. Select 'Github' on the left panel
-5. Click 'Authorize appveyor'
-6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
-7. You should have appveyor set up on your fork now.
-
-### General notes on CI
-CI tests run every time a pull request (PR) is created or updated. The exact tests
-that get run will depend on the destination branch you specify. Some tests take
-longer to run than others. Currently, our CI is set up to run a short
-series of tests when creating a PR to the dev branch and a longer series of tests
-when creating a PR to the release branch. You can look in the configuration files
-of the respective CI platform for more information on what gets run when.
-
-Most people will just want to create a PR with the destination set to their local dev
-branch of zstd. You can then find the status of the tests on the PR's page. You can also
-re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
-
 ## Issues
 We use GitHub issues to track public bugs. Please ensure your description is
 clear and has sufficient instructions to be able to reproduce the issue.
@@ -399,7 +384,105 @@ disclosure of security bugs. In those cases, please go through the process
 outlined on that page and do not file a public issue.

 ## Coding Style
+It's a pretty long topic, which is difficult to summarize in a single paragraph.
+As a rule of thumbs, try to imitate the coding style of
+similar lines of codes around your contribution.
+The following is a non-exhaustive list of rules employed in zstd code base:
+
+### C90
+This code base is following strict C90 standard,
+with 2 extensions : 64-bit `long long` types, and variadic macros.
+This rule is applied strictly to code within `lib/` and `programs/`.
+Sub-project in `contrib/` are allowed to use other conventions.
+
+### C++ direct compatibility : symbol mangling
+All public symbol declarations must be wrapped in `extern “C” { … }`,
+so that this project can be compiled as C++98 code,
+and linked into C++ applications.
+
+### Minimal Frugal
+This design requirement is fundamental to preserve the portability of the code base.
+#### Dependencies
+- Reduce dependencies to the minimum possible level.
+  Any dependency should be considered “bad” by default,
+  and only tolerated because it provides a service in a better way than can be achieved locally.
+  The only external dependencies this repository tolerates are
+  standard C libraries, and in rare cases, system level headers.
+- Within `lib/`, this policy is even more drastic.
+  The only external dependencies allowed are `<assert.h>`, `<stdlib.h>`, `<string.h>`,
+  and even then, not directly.
+  In particular, no function shall ever allocate on heap directly,
+  and must use instead `ZSTD_malloc()` and equivalent.
+  Other accepted non-symbol headers are `<stddef.h>` and `<limits.h>`.
+- Within the project, there is a strict hierarchy of dependencies that must be respected.
+  `programs/` is allowed to depend on `lib/`, but only its public API.
+  Within `lib/`, `lib/common` doesn't depend on any other directory.
+  `lib/compress` and `lib/decompress` shall not depend on each other.
+  `lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`.
+#### Resources
+- Functions in `lib/` must use very little stack space,
+  several dozens of bytes max.
+  Everything larger must use the heap allocator,
+  or require a scratch buffer to be emplaced manually.
+
+### Naming
+* All public symbols are prefixed with `ZSTD_`
+  + private symbols, with a scope limited to their own unit, are free of this restriction.
+    However, since `libzstd` source code can be amalgamated,
+    each symbol name must attempt to be (and remain) unique.
+    Avoid too generic names that could become ground for future collisions.
+    This generally implies usage of some form of prefix.
+* For symbols (functions and variables), naming convention is `PREFIX_camelCase`.
+  + In some advanced cases, one can also find :
+    - `PREFIX_prefix2_camelCase`
+    - `PREFIX_camelCase_extendedQualifier`
+* Multi-words names generally consist of an action followed by object:
+  - for example : `ZSTD_createCCtx()`
+* Prefer positive actions
+  - `goBackward` rather than `notGoForward`
+* Type names (`struct`, etc.) follow similar convention,
+  except that they are allowed and even invited to start by an Uppercase letter.
+  Example : `ZSTD_CCtx`, `ZSTD_CDict`
+* Macro names are all Capital letters.
+  The same composition rules (`PREFIX_NAME_QUALIFIER`) apply.
+* File names are all lowercase letters.
+  The convention is `snake_case`.
+  File names **must** be unique across the entire code base,
+  even when they stand in clearly separated directories.
+
+### Qualifiers
+* This code base is `const` friendly, if not `const` fanatical.
+  Any variable that can be `const` (aka. read-only) **must** be `const`.
+  Any pointer which content will not be modified must be `const`.
+  This property is then controlled at compiler level.
+  `const` variables are an important signal to readers that this variable isn’t modified.
+  Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function.
+* If a function must be inlined, mention it explicitly,
+  using project's own portable macros, such as `FORCE_INLINE_ATTR`,
+  defined in `lib/common/compiler.h`.
+
+### Debugging
+* **Assertions** are welcome, and should be used very liberally,
+  to control any condition the code expects for its correct execution.
+  These assertion checks will be run in debug builds, and disabled in production.
+* For traces, this project provides its own debug macros,
+  in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`.
+
+### Code documentation
+* Avoid code documentation that merely repeats what the code is already stating.
+  Whenever applicable, prefer employing the code as the primary way to convey explanations.
+  Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`.
+  Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`.
+* At declaration level, the documentation explains how to use the function or variable
+  and when applicable why it's needed, of the scenarios where it can be useful.
+* At implementation level, the documentation explains the general outline of the algorithm employed,
+  and when applicable why this specific choice was preferred.
+
+### General layout
 * 4 spaces for indentation rather than tabs
+* Code documentation shall directly precede function declaration or implementation
+* Function implementations and its code documentation should be preceded and followed by an empty line
+

 ## License
 By contributing to Zstandard, you agree that your contributions will be licensed
@@ -29,6 +29,7 @@ VOID = /dev/null
 # fail on other tested distros (ubuntu, debian) even
 # without manually specifying the TARGET_SYSTEM.
 TARGET_SYSTEM ?= $(OS)
+CP ?= cp

 ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
  EXT =.exe
@@ -57,8 +58,8 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32

-.PHONY: lib lib-release
-lib lib-release :
+.PHONY: lib lib-release lib-mt lib-nomt
+lib lib-release lib-mt lib-nomt:
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@

 .PHONY: zstd zstd-release
@@ -69,7 +70,7 @@ zstd zstd-release:
 .PHONY: zstdmt
 zstdmt:
 	$(Q)$(MAKE) -C $(PRGDIR) $@
-	$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+	$(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)

 .PHONY: zlibwrapper
 zlibwrapper: lib
@@ -122,8 +123,8 @@ contrib: lib
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/seekable_format/tests test
 	$(MAKE) -C contrib/largeNbDicts all
-	cd contrib/single_file_libs/ ; ./build_decoder_test.sh
-	cd contrib/single_file_libs/ ; ./build_library_test.sh
+	cd build/single_file_libs/ ; ./build_decoder_test.sh
+	cd build/single_file_libs/ ; ./build_library_test.sh

 .PHONY: cleanTabs
 cleanTabs:
@@ -148,10 +149,11 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))

 HOST_OS = POSIX
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
+
+MKDIR ?= mkdir -p

 HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 EGREP_OPTIONS ?=
@@ -179,7 +181,7 @@ list:
 	    done \
 	} | column -t -s $$'\t'

-.PHONY: install armtest usan asan uasan
+.PHONY: install armtest usan asan uasan msan asan32
 install:
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 	$(Q)$(MAKE) -C $(PRGDIR) $@
@@ -193,22 +195,19 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir

-.PHONY: gcc5build
+.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
 gcc5build: clean
 	gcc-5 -v
 	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"

-.PHONY: gcc6build
 gcc6build: clean
 	gcc-6 -v
 	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"

-.PHONY: gcc7build
 gcc7build: clean
 	gcc-7 -v
 	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"

-.PHONY: clangbuild
 clangbuild: clean
 	clang -v
 	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
@@ -221,7 +220,7 @@ armbuild: clean
 	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd

 aarch64build: clean
-	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd

 ppcbuild: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
@@ -229,6 +228,7 @@ ppcbuild: clean
 ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd

+.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest

@@ -242,7 +242,7 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest

-.PHONY: cxxtest
+.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
 cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
 cxxtest: clean
 	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
@@ -271,6 +271,7 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests

+.PHONY: arm-ppc-compilation
 arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
@@ -286,9 +287,24 @@ uasanregressiontest:
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"

+update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
+update_regressionResults:
+	$(MAKE) -C programs zstd
+	$(MAKE) -C tests/regression test
+	$(RM) -rf $(REGRESS_RESULTS_DIR)
+	$(MKDIR) $(REGRESS_RESULTS_DIR)
+	./tests/regression/test                         \
+        --cache  tests/regression/cache             \
+        --output $(REGRESS_RESULTS_DIR)/results.csv \
+        --zstd   programs/zstd
+	echo "Showing results differences"
+	! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv
+	echo "Updating results.csv"
+	$(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv
+
+
 # run UBsan with -fsanitize-recover=pointer-overflow
 # this only works with recent compilers such as gcc 8+
-
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"

@@ -316,13 +332,16 @@ uasan-%: clean
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

+.PHONY: apt-install
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)

+.PHONY: apt-add-repo
 apt-add-repo:
 	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 	sudo apt-get update -y -qq

+.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
 ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install

@@ -357,20 +376,22 @@ lz4install:
 endif


+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
+
 ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
 CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif

-
 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
-	mkdir $(BUILDIR)/cmake/build
+	$(MKDIR) $(BUILDIR)/cmake/build
 	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
 	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
 	$(MAKE) -C $(BUILDIR)/cmake/build install;
@@ -379,23 +400,23 @@ cmakebuild:

 c89build: clean
 	$(CC) -v
-	CFLAGS="-std=c89 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost  # will fail, due to missing support for `long long`

 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost

 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost

 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost

 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost

 bmix64build: clean
 	$(CC) -v
@@ -414,5 +435,5 @@ bmi32build: clean
 staticAnalyze: SCANBUILD ?= scan-build
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
 endif
@@ -0,0 +1,36 @@
+// swift-tools-version:5.0
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "zstd",
+    platforms: [
+        .macOS(.v10_10), .iOS(.v9), .tvOS(.v9)
+    ],
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "libzstd",
+            targets: [ "libzstd" ])
+    ],
+    dependencies: [
+        // Dependencies declare other packages that this package depends on.
+        // .package(url: /* package url */, from: "1.0.0"),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .target(
+            name: "libzstd",
+            path: "lib",
+            sources: [ "common", "compress", "decompress", "dictBuilder" ],
+            publicHeadersPath: ".",
+            cSettings: [
+                .headerSearchPath(".")
+            ])
+    ],
+    swiftLanguageVersions: [.v5],
+    cLanguageStandard: .gnu11,
+    cxxLanguageStandard: .gnucxx14
+)
@@ -4,7 +4,8 @@ __Zstandard__, or `zstd` as short version, is a fast lossless compression algori
 targeting real-time compression scenarios at zlib-level and better compression ratios.
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).

-The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
+Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
+This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
@@ -17,8 +18,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 [![Build status][CirrusDevBadge]][CirrusLink]
 [![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]

-[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
-[travisLink]: https://travis-ci.org/facebook/zstd
+[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
+[travisLink]: https://travis-ci.com/facebook/zstd
 [AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
@@ -31,8 +32,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 ## Benchmarks

 For reference, several fast compression algorithms were tested and compared
-on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
-with a Core i9-9900K CPU @ 5.0GHz,
+on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
+with a Core i7-9700K CPU @ 4.9GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
 compiled with [gcc] 9.3.0,
 on the [Silesia compression corpus].
@@ -43,25 +44,24 @@ on the [Silesia compression corpus].

 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.4.5 -1**       | 2.884 |   500 MB/s |  1660 MB/s |
-| zlib 1.2.11 -1          | 2.743 |    90 MB/s |   400 MB/s |
-| brotli 1.0.7 -0         | 2.703 |   400 MB/s |   450 MB/s |
-| **zstd 1.4.5 --fast=1** | 2.434 |   570 MB/s |  2200 MB/s |
-| **zstd 1.4.5 --fast=3** | 2.312 |   640 MB/s |  2300 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   560 MB/s |   710 MB/s |
-| **zstd 1.4.5 --fast=5** | 2.178 |   700 MB/s |  2420 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   690 MB/s |   820 MB/s |
-| lz4 1.9.2               | 2.101 |   740 MB/s |  4530 MB/s |
-| **zstd 1.4.5 --fast=7** | 2.096 |   750 MB/s |  2480 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   860 MB/s |
-| snappy 1.1.8            | 2.073 |   560 MB/s |  1790 MB/s |
+| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
+| [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
+| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
+| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
+| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
+| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
+| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
+| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
+| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |

 [zlib]: http://www.zlib.net/
-[LZ4]: http://www.lz4.org/
+[lz4]: http://www.lz4.org/

 The negative compression levels, specified with `--fast=#`,
-offer faster compression and decompression speed in exchange for some loss in
-compression ratio compared to level 1, as seen in the table above.
+offer faster compression and decompression speed
+at the cost of compression ratio (compared to level 1).

 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
@@ -40,5 +40,4 @@ They consist of the following tests:
 - Versions test (ensuring `zstd` can decode files from all previous versions)
 - `pzstd` with asan and tsan, as well as in 32-bits mode
 - Testing `zstd` with legacy mode off
- Testing `zbuff` (old streaming API)
 - Entire test suite and make install on macOS
@@ -23,34 +23,15 @@
      SCRIPT:   "make allzstd MOREFLAGS=-static"
      ARTIFACT: "true"
      BUILD:    "true"
-    - COMPILER: "clang"
-      HOST:     "mingw"
-      PLATFORM: "x64"
-      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
-      BUILD:    "true"

-    - COMPILER: "gcc"
-      HOST:     "mingw"
-      PLATFORM: "x64"
-      SCRIPT:   ""
-      TEST:     "cmake"
-
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
+    - COMPILER: "clang-cl"
+      HOST:     "cmake-visual"
      PLATFORM: "x64"
      CONFIGURATION: "Release"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Release"
+      CMAKE_GENERATOR: "Visual Studio 15 2017"
+      CMAKE_GENERATOR_PLATFORM: "x64"
+      CMAKE_GENERATOR_TOOLSET: "LLVM"
+      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"

  install:
  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -104,55 +85,14 @@
          appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
      )
    )
-  - if [%HOST%]==[visual] (
+  - if [%HOST%]==[cmake-visual] (
      ECHO *** &&
-      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
-      ECHO *** &&
-      msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
-      ECHO *** &&
-      ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
-      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
-      ECHO *** &&
-      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
-      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
-      ECHO *** &&
-      ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
-      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
-      ECHO *** &&
-      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
-      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
+      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
+      PUSHD build\cmake &&
+      cmake -DBUILD_TESTING=ON . &&
+      cmake --build . --config %CONFIGURATION% -j4 &&
+      POPD &&
+      ECHO ***
    )

  test_script:
@@ -168,18 +108,6 @@
      cd ..\..\.. &&
      make clean
    )
-  - SET "FUZZERTEST=-T30s"
-  - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
-      CD tests &&
-      SET ZSTD_BIN=./zstd.exe&&
-      SET DATAGEN_BIN=./datagen.exe&&
-      sh -e playTests.sh --test-large-data &&
-      fullbench.exe -i1 &&
-      fullbench.exe -i1 -P0 &&
-      fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
-    )


 # The following tests are for regular pushes
@@ -190,38 +118,35 @@
  version: 1.0.{build}
  environment:
    matrix:
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
+
    - COMPILER: "gcc"
      HOST:     "cygwin"
      PLATFORM: "x64"
-    - COMPILER: "gcc"
-      HOST:     "mingw"
-      PLATFORM: "x64"
-      SCRIPT:   "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
-    - COMPILER: "gcc"
-      HOST:     "mingw"
-      PLATFORM: "x86"
-      SCRIPT:   "CFLAGS=-Werror make -j allzstd"
-    - COMPILER: "clang"
-      HOST:     "mingw"
-      PLATFORM: "x64"
-      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"

-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
+    - COMPILER: "clang-cl"
+      HOST:     "cmake-visual"
      PLATFORM: "x64"
      CONFIGURATION: "Release"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Release"
+      CMAKE_GENERATOR: "Visual Studio 15 2017"
+      CMAKE_GENERATOR_PLATFORM: "x64"
+      CMAKE_GENERATOR_TOOLSET: "LLVM"
+      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"

  install:
  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -229,17 +154,10 @@
  - if [%HOST%]==[cygwin] (
      ECHO Installing Cygwin Packages &&
      C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
-        gcc-g++,^
        gcc,^
        cmake,^
        make
    )
-  - if [%HOST%]==[mingw] (
-      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
-      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
-      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
-      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
-    )
  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
    )
@@ -252,41 +170,36 @@
      C:\cygwin64\bin\bash --login -c "
        set -e;
        cd build/cmake;
-        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
-        make -j4;
+        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .;
+        make VERBOSE=1 -j;
        ctest -V -L Medium;
      "
    )
-  - if [%HOST%]==[mingw] (
-      ( if [%PLATFORM%]==[x64] (
-        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
-      ) else if [%PLATFORM%]==[x86] (
-        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
-      ) ) &&
-      make -v &&
-      sh -c "%COMPILER% -v" &&
-      set "CC=%COMPILER%" &&
-      sh -c "%SCRIPT%"
+  - if [%HOST%]==[cmake-visual] (
+      ECHO *** &&
+      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
+      PUSHD build\cmake &&
+      cmake -DBUILD_TESTING=ON . &&
+      cmake --build . --config %CONFIGURATION% -j4 &&
+      POPD &&
+      ECHO ***
    )
  - if [%HOST%]==[visual] (
      ECHO *** &&
-      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
      ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
-      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe
    )


  test_script:
  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET "FUZZERTEST=-T10s"
  - if [%HOST%]==[mingw] (
      set "CC=%COMPILER%" &&
      make clean &&
      make check
-    )
+    )
@@ -463,7 +463,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -483,7 +483,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -511,7 +511,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -384,6 +384,10 @@
 				RelativePath="..\..\..\programs\fileio.c"
 				>
 			</File>
+			<File
+                RelativePath="..\..\..\programs\fileio_asyncio.c"
+                >
+            </File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>
@@ -559,7 +563,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -575,7 +579,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -495,7 +495,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -523,7 +523,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -190,7 +190,7 @@
    <ClInclude Include="..\..\..\lib\zstd.h" />
    <ClInclude Include="..\..\..\lib\common\fse.h" />
    <ClInclude Include="..\..\..\lib\common\huf.h" />
-    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
    <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
    <ClInclude Include="..\..\..\lib\common\pool.h" />
    <ClInclude Include="..\..\..\lib\common\threading.h" />
@@ -196,7 +196,7 @@
    <ClInclude Include="..\..\..\lib\common\huf.h" />
    <ClInclude Include="..\..\..\lib\common\xxhash.h" />
    <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
    <ClInclude Include="..\..\..\lib\zstd.h" />
    <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
@@ -211,7 +211,7 @@
    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
    <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
-    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
+    <ClInclude Include="..\..\..\lib\zdict.h" />
    <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
    <ClInclude Include="..\..\..\programs\datagen.h" />
@@ -44,9 +44,6 @@
    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -64,12 +61,11 @@
    <ClInclude Include="..\..\..\lib\common\threading.h" />
    <ClInclude Include="..\..\..\lib\common\bitstream.h" />
    <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
    <ClInclude Include="..\..\..\lib\common\mem.h" />
    <ClInclude Include="..\..\..\lib\common\fse.h" />
    <ClInclude Include="..\..\..\lib\common\huf.h" />
    <ClInclude Include="..\..\..\lib\common\xxhash.h" />
-    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -44,9 +44,6 @@
    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
-    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
    <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -64,12 +61,11 @@
    <ClInclude Include="..\..\..\lib\common\threading.h" />
    <ClInclude Include="..\..\..\lib\common\bitstream.h" />
    <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
    <ClInclude Include="..\..\..\lib\common\mem.h" />
    <ClInclude Include="..\..\..\lib\common\fse.h" />
    <ClInclude Include="..\..\..\lib\common\huf.h" />
    <ClInclude Include="..\..\..\lib\common\xxhash.h" />
-    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
    <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -62,6 +62,7 @@
    <ClCompile Include="..\..\..\programs\datagen.c" />
    <ClCompile Include="..\..\..\programs\dibio.c" />
    <ClCompile Include="..\..\..\programs\fileio.c" />
+    <ClCompile Include="..\..\..\programs\fileio_asyncio.c" />
    <ClCompile Include="..\..\..\programs\zstdcli.c" />
    <ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
  </ItemGroup>
@@ -70,14 +71,14 @@
    <ClInclude Include="..\..\..\lib\common\threading.h" />
    <ClInclude Include="..\..\..\lib\common\xxhash.h" />
    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
+    <ClInclude Include="..\..\..\lib\zdict.h" />
    <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
    <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
    <ClInclude Include="..\..\..\lib\common\fse.h" />
    <ClInclude Include="..\..\..\lib\common\huf.h" />
    <ClInclude Include="..\..\..\lib\zstd.h" />
    <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
    <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
@@ -19,10 +19,10 @@ GOTO build
 :display_help

 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, ...)
+echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
-echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141)
+echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)

 EXIT /B 1

@@ -43,6 +43,16 @@ IF %msbuild_version% == VS2017 (
 	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
 )

+:: VS2019
+SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
+SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
+SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
+IF %msbuild_version% == VS2019 (
+	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
+	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
+	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
+)
+
 SET project="%~p0\..\VS2010\zstd.sln"

 SET msbuild_params=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%msbuild_platform% /p:Configuration=%msbuild_configuration%
@@ -92,8 +92,14 @@ else ()
    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
 endif ()

+if (ANDROID)
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
+else()
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
+endif()
+
 # Multi-threading support
-option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
+option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})

 if (ZSTD_MULTITHREAD_SUPPORT)
    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
@@ -26,7 +26,12 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
        EnableCompilerFlag("-std=c++11" false true)
        #Set c99 by default
        EnableCompilerFlag("-std=c99" true false)
-        EnableCompilerFlag("-Wall" true true)
+        if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
+            # clang-cl normally maps -Wall to -Weverything.
+            EnableCompilerFlag("/clang:-Wall" true true)
+        else ()
+            EnableCompilerFlag("-Wall" true true)
+        endif ()
        EnableCompilerFlag("-Wextra" true true)
        EnableCompilerFlag("-Wundef" true true)
        EnableCompilerFlag("-Wshadow" true true)
@@ -21,10 +21,16 @@ add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/O
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")

+if (ZSTD_BUILD_SHARED)
+    set(ZSTD_LIB libzstd_shared)
+else()
+    set(ZSTD_LIB libzstd_static)
+endif()
+
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 if (CMAKE_USE_PTHREADS_INIT)
-    target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
+  target_link_libraries(pzstd ${ZSTD_LIB} ${CMAKE_THREAD_LIBS_INIT})
 else()
    message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
 endif()
@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################

-project(libzstd C)
+project(libzstd C ASM)

 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
@@ -22,30 +22,31 @@ include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)

 file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
 file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
-file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
+if (MSVC)
+    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
+    add_compile_options(-DZSTD_DISABLE_ASM)
+else ()
+    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
+endif ()
 file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
-file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)

 set(Sources
        ${CommonSources}
        ${CompressSources}
        ${DecompressSources}
-        ${DictBuilderSources}
-        ${DeprecatedSources})
+        ${DictBuilderSources})

 file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
 file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
 file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
 file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
-file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)

 set(Headers
        ${LIBRARY_DIR}/zstd.h
        ${CommonHeaders}
        ${CompressHeaders}
        ${DecompressHeaders}
-        ${DictBuilderHeaders}
-        ${DeprecatedHeaders})
+        ${DictBuilderHeaders})

 if (ZSTD_LEGACY_SUPPORT)
    set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@@ -110,7 +111,7 @@ if (MSVC)
 endif ()

 # With MSVC static library needs to be renamed to avoid conflict with import library
-if (MSVC)
+if (MSVC OR (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT MINGW))
    set(STATIC_LIBRARY_BASE_NAME zstd_static)
 else ()
    set(STATIC_LIBRARY_BASE_NAME zstd)
@@ -162,8 +163,8 @@ endif ()
 # install target
 install(FILES
    "${LIBRARY_DIR}/zstd.h"
-    "${LIBRARY_DIR}/dictBuilder/zdict.h"
-    "${LIBRARY_DIR}/common/zstd_errors.h"
+    "${LIBRARY_DIR}/zdict.h"
+    "${LIBRARY_DIR}/zstd_errors.h"
    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")

 install(TARGETS ${library_targets}
@@ -172,6 +173,7 @@ install(TARGETS ${library_targets}
    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+    BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
    )

 # uninstall target
@@ -32,12 +32,14 @@ if (MSVC)
    set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()

-add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
+add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
 target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
    target_link_libraries(zstd rt)
 endif ()
-install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
+install(TARGETS zstd
+  RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+  BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}")

 if (UNIX)
    add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
@@ -73,7 +75,7 @@ if (UNIX)
        ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
        DESTINATION "${MAN_INSTALL_DIR}")

-    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
+    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
    target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
    set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
 endif ()
@@ -3,5 +3,4 @@ datagen
 fullbench
 fuzzer
 paramgrill
-zbufftest

@@ -57,13 +57,15 @@ target_link_libraries(datagen libzstd_static)
 # fullbench
 #
 add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
+set_property(TARGET fullbench APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND fullbench)
+add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})

 #
 # fuzzer
 #
 add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
+set_property(TARGET fuzzer APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(fuzzer libzstd_static)
 AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
    "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
@@ -76,6 +78,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
 # zstreamtest
 #
 add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
+set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
 target_link_libraries(zstreamtest libzstd_static)
 AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
    "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
@@ -18,7 +18,7 @@ pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
  join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
 pzstd = executable('pzstd',
  pzstd_sources,
-  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
+  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-Wno-deprecated-declarations' ],
  include_directories: pzstd_includes,
  dependencies: [ libzstd_dep, thread_dep ],
  install: true)
@@ -14,15 +14,13 @@ libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
  join_paths(zstd_rootdir, 'lib/common'),
  join_paths(zstd_rootdir, 'lib/compress'),
  join_paths(zstd_rootdir, 'lib/decompress'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder'),
-  join_paths(zstd_rootdir, 'lib/deprecated'))]
+  join_paths(zstd_rootdir, 'lib/dictBuilder'))]

 libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
  join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
  join_paths(zstd_rootdir, 'lib/common/threading.c'),
  join_paths(zstd_rootdir, 'lib/common/pool.c'),
  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
-  join_paths(zstd_rootdir, 'lib/common/zstd_trace.c'),
  join_paths(zstd_rootdir, 'lib/common/error_private.c'),
  join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
  join_paths(zstd_rootdir, 'lib/compress/hist.c'),
@@ -45,10 +43,16 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
  join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
  join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
  join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
-  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
-  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
-  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')]
+
+# really we need anything that defines __GNUC__ as that is what ZSTD_ASM_SUPPORTED is gated on
+# but these are the two compilers that are supported in tree and actually handle this correctly
+# Otherwise, explicitly disable assmebly.
+if [compiler_gcc, compiler_clang].contains(cc_id)
+  libzstd_sources += join_paths(zstd_rootdir, 'lib/decompress/huf_decompress_amd64.S')
+else
+  add_project_arguments('-DZSTD_DISABLE_ASM', language: 'c')
+endif

 # Explicit define legacy support
 add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
@@ -113,6 +117,7 @@ libzstd = library('zstd',
  libzstd_sources,
  include_directories: libzstd_includes,
  c_args: libzstd_c_args,
+  gnu_symbol_visibility: 'hidden',
  dependencies: libzstd_deps,
  install: true,
  version: zstd_libversion)
@@ -128,5 +133,5 @@ pkgconfig.generate(libzstd,
  url: 'http://www.zstd.net/')

 install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
-  join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))
+  join_paths(zstd_rootdir, 'lib/zdict.h'),
+  join_paths(zstd_rootdir, 'lib/zstd_errors.h'))
@@ -14,10 +14,14 @@ project('zstd',
  default_options : [
    'c_std=gnu99',
    'cpp_std=c++11',
-    'buildtype=release'
+    'buildtype=release',
+    'warning_level=3',
+    # -Wdocumentation does not actually pass, nor do the test binaries,
+    # so this isn't safe
+    #'werror=true'
  ],
  version: 'DUMMY',
-  meson_version: '>=0.47.0')
+  meson_version: '>=0.48.0')

 cc = meson.get_compiler('c')
 cxx = meson.get_compiler('cpp')
@@ -106,10 +110,8 @@ use_lz4 = lz4_dep.found()
 add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])

 if [compiler_gcc, compiler_clang].contains(cc_id)
-  common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
+  common_warning_flags = [ '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
  if cc_id == compiler_clang
-    # Should use Meson's own --werror build option
-    #common_warning_flags += '-Werror'
    common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
  endif
  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
@@ -10,7 +10,7 @@

 # Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting

-option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
+option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
  description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
 option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
  description: 'Enable run-time debug. See lib/common/debug.h')
@@ -14,18 +14,31 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
  join_paths(zstd_rootdir, 'programs/util.c'),
  join_paths(zstd_rootdir, 'programs/timefn.c'),
  join_paths(zstd_rootdir, 'programs/fileio.c'),
+  join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'),
  join_paths(zstd_rootdir, 'programs/benchfn.c'),
  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
  join_paths(zstd_rootdir, 'programs/datagen.c'),
  join_paths(zstd_rootdir, 'programs/dibio.c'),
-  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
-
-zstd_c_args = libzstd_debug_cflags
-if use_multi_thread
-  zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
-endif
+  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c'),
+  # needed due to use of private symbol + -fvisibility=hidden
+  join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c')]

 zstd_deps = [ libzstd_dep ]
+zstd_c_args = libzstd_debug_cflags
+
+zstd_frugal_deps = [ libzstd_dep ]
+zstd_frugal_c_args = [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ]
+
+if use_multi_thread
+  zstd_deps += [ thread_dep ]
+  zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
+  zstd_frugal_deps += [ thread_dep ]
+  zstd_frugal_c_args += [ '-DZSTD_MULTITHREAD' ]
+endif
+
 if use_zlib
  zstd_deps += [ zlib_dep ]
  zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
@@ -67,14 +80,18 @@ zstd = executable('zstd',
 zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
  join_paths(zstd_rootdir, 'programs/timefn.c'),
  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/fileio.c')]
+  join_paths(zstd_rootdir, 'programs/fileio.c'),
+  join_paths(zstd_rootdir, 'programs/fileio_asyncio.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c')]

 # Minimal target, with only zstd compression and decompression.
 # No bench. No legacy.
 executable('zstd-frugal',
  zstd_frugal_sources,
-  dependencies: libzstd_dep,
-  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
+  dependencies: zstd_frugal_deps,
+  c_args: zstd_frugal_c_args,
  install: true)

 install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
@@ -29,64 +29,62 @@ ZSTDRTTEST = ['--test-large-data']

 test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]

-datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'tests/datagencli.c')]
+testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c')]
+
+testcommon = static_library('testcommon',
+  testcommon_sources,
+  # needed due to use of private symbol + -fvisibility=hidden
+  objects: libzstd.extract_all_objects(recursive: false))
+
+testcommon_dep = declare_dependency(link_with: testcommon,
+  dependencies: libzstd_deps,
+  include_directories: libzstd_includes)
+
+datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
 datagen = executable('datagen',
  datagen_sources,
  c_args: [ '-DNDEBUG' ],
  include_directories: test_includes,
-  dependencies: libzstd_dep,
+  dependencies: testcommon_dep,
  install: false)

-fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/benchfn.c'),
-  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
-  join_paths(zstd_rootdir, 'tests/fullbench.c')]
+fullbench_sources = [join_paths(zstd_rootdir, 'tests/fullbench.c')]
 fullbench = executable('fullbench',
  fullbench_sources,
  include_directories: test_includes,
-  dependencies: libzstd_dep,
+  dependencies: testcommon_dep,
  install: false)

-fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'tests/fuzzer.c')]
+fuzzer_sources = [join_paths(zstd_rootdir, 'tests/fuzzer.c')]
 fuzzer = executable('fuzzer',
  fuzzer_sources,
  include_directories: test_includes,
-  dependencies: libzstd_dep,
+  dependencies: [ testcommon_dep, thread_dep ],
  install: false)

-zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'tests/seqgen.c'),
+zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'),
  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
 zstreamtest = executable('zstreamtest',
  zstreamtest_sources,
  include_directories: test_includes,
-  dependencies: libzstd_dep,
+  dependencies: testcommon_dep,
  install: false)

-paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
-  join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'tests/paramgrill.c')]
+paramgrill_sources = [join_paths(zstd_rootdir, 'tests/paramgrill.c')]
 paramgrill = executable('paramgrill',
  paramgrill_sources,
  include_directories: test_includes,
-  dependencies: [ libzstd_dep, libm_dep ],
+  dependencies: [ testcommon_dep, libm_dep ],
  install: false)

 roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
 roundTripCrash = executable('roundTripCrash',
  roundTripCrash_sources,
-  dependencies: [ libzstd_dep ],
+  dependencies: [ testcommon_dep ],
  install: false)

 longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
@@ -111,18 +109,14 @@ if 0 < legacy_level and legacy_level <= 4
    install: false)
 endif

-decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
+decodecorpus_sources = [join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
 decodecorpus = executable('decodecorpus',
  decodecorpus_sources,
  include_directories: test_includes,
-  dependencies: [ libzstd_dep, libm_dep ],
+  dependencies: [ testcommon_dep, libm_dep ],
  install: false)

-poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'tests/poolTests.c'),
+poolTests_sources = [join_paths(zstd_rootdir, 'tests/poolTests.c'),
  join_paths(zstd_rootdir, 'lib/common/pool.c'),
  join_paths(zstd_rootdir, 'lib/common/threading.c'),
  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
@@ -130,7 +124,7 @@ poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
 poolTests = executable('poolTests',
  poolTests_sources,
  include_directories: test_includes,
-  dependencies: [ libzstd_dep, thread_dep ],
+  dependencies: [ testcommon_dep, thread_dep ],
  install: false)

 checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
@@ -186,10 +180,6 @@ test('test-zstream-1',
  zstreamtest,
  args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
  timeout: 240)
-test('test-zstream-2',
-  zstreamtest,
-  args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
-  timeout: 120)
 test('test-zstream-3',
  zstreamtest,
  args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
@@ -11,8 +11,8 @@ This is the most common use case. The decompression library is small, adding, fo

 Create `zstddeclib.c` from the Zstd source using:
 ```
-cd zstd/contrib/single_file_libs
-./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
+cd zstd/build/single_file_libs
+python3 combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
 ```
 Then add the resulting file to your project (see the [example files](examples)).

@@ -25,8 +25,8 @@ The same tool can amalgamate the entire Zstd library for ease of adding both com

 Create `zstd.c` from the Zstd source using:
 ```
-cd zstd/contrib/single_file_libs
-./combine.sh -r ../../lib -o zstd.c zstd-in.c
+cd zstd/build/single_file_libs
+python3 combine.py -r ../../lib -x legacy/zstd_legacy.h -k zstd.h -o zstd.c zstd-in.c
 ```
 It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`).

@@ -69,7 +69,7 @@ fi
 echo "Single file library creation script: PASSED"

 # Copy the header to here (for the tests)
-cp "$ZSTD_SRC_ROOT/zstd.h" zstd.h
+cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h

 # Compile the generated output
 cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c
@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+
+# Tool to bundle multiple C/C++ source files, inlining any includes.
+# 
+# Note: there are two types of exclusion options: the '-x' flag, which besides
+# excluding a file also adds an #error directive in place of the #include, and
+# the '-k' flag, which keeps the #include and doesn't inline the file. The
+# intended use cases are: '-x' for files that would normally be #if'd out, so
+# features that 100% won't be used in the amalgamated file, for which every
+# occurrence adds the error, and '-k' for headers that we wish to manually
+# include, such as a project's public API, for which occurrences after the first
+# are removed.
+# 
+# Todo: the error handling could be better, which currently throws and halts
+# (which is functional just not very friendly).
+# 
+# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
+
+import argparse, re, sys
+
+from pathlib import Path
+from typing import Any, List, Optional, Pattern, Set, TextIO
+
+# Set of file roots when searching (equivalent to -I paths for the compiler).
+roots: Set[Path] = set()
+
+# Set of (canonical) file Path objects to exclude from inlining (and not only
+# exclude but to add a compiler error directive when they're encountered).
+excludes: Set[Path] = set()
+
+# Set of (canonical) file Path objects to keep as include directives.
+keeps: Set[Path] = set()
+
+# Whether to keep the #pragma once directives (unlikely, since this will result
+# in a warning, but the option is there).
+keep_pragma: bool = False
+
+# Destination file object (or stdout if no output file was supplied).
+destn: TextIO = sys.stdout
+
+# Set of file Path objects previously inlined (and to ignore if reencountering).
+found: Set[Path] = set()
+
+# Compiled regex Patern to handle "#pragma once" in various formats:
+# 
+#   #pragma once
+#     #pragma once
+#   #  pragma once
+#   #pragma   once
+#   #pragma once // comment
+# 
+# Ignoring commented versions, same as include_regex.
+# 
+pragma_regex: Pattern = re.compile(r'^\s*#\s*pragma\s*once\s*')
+
+# Compiled regex Patern to handle the following type of file includes:
+# 
+#   #include "file"
+#     #include "file"
+#   #  include "file"
+#   #include   "file"
+#   #include "file" // comment
+#   #include "file" // comment with quote "
+# 
+# And all combinations of, as well as ignoring the following:
+# 
+#   #include <file>
+#   //#include "file"
+#   /*#include "file"*/
+# 
+# We don't try to catch errors since the compiler will do this (and the code is
+# expected to be valid before processing) and we don't care what follows the
+# file (whether it's a valid comment or not, since anything after the quoted
+# string is ignored)
+# 
+include_regex: Pattern = re.compile(r'^\s*#\s*include\s*"(.+?)"')
+
+# Simple tests to prove include_regex's cases.
+# 
+def test_match_include() -> bool:
+    if (include_regex.match('#include "file"')   and
+        include_regex.match('  #include "file"') and
+        include_regex.match('#  include "file"') and
+        include_regex.match('#include   "file"') and
+        include_regex.match('#include "file" // comment')):
+            if (not include_regex.match('#include <file>')   and
+                not include_regex.match('//#include "file"') and
+                not include_regex.match('/*#include "file"*/')):
+                    found = include_regex.match('#include "file" // "')
+                    if (found and found.group(1) == 'file'):
+                        print('#include match valid')
+                        return True
+    return False
+
+# Simple tests to prove pragma_regex's cases.
+# 
+def test_match_pragma() -> bool:
+    if (pragma_regex.match('#pragma once')   and
+        pragma_regex.match('  #pragma once') and
+        pragma_regex.match('#  pragma once') and
+        pragma_regex.match('#pragma   once') and
+        pragma_regex.match('#pragma once // comment')):
+            if (not pragma_regex.match('//#pragma once') and
+                not pragma_regex.match('/*#pragma once*/')):
+                    print('#pragma once match valid')
+                    return True
+    return False
+
+# Finds 'file'. First the list of 'root' paths are searched, followed by the
+# the currently processing file's 'parent' path, returning a valid Path in
+# canonical form. If no match is found None is returned.
+# 
+def resolve_include(file: str, parent: Optional[Path] = None) -> Optional[Path]:
+    for root in roots:
+        found = root.joinpath(file).resolve()
+        if (found.is_file()):
+            return found
+    if (parent):
+        found = parent.joinpath(file).resolve();
+    else:
+        found = Path(file)
+    if (found.is_file()):
+        return found
+    return None
+
+# Helper to resolve lists of files. 'file_list' is passed in from the arguments
+# and each entry resolved to its canonical path (like any include entry, either
+# from the list of root paths or the owning file's 'parent', which in this case
+# is case is the input file). The results are stored in 'resolved'.
+# 
+def resolve_excluded_files(file_list: Optional[List[str]], resolved: Set[Path], parent: Optional[Path] = None) -> None:
+    if (file_list):
+        for filename in file_list:
+            found = resolve_include(filename, parent)
+            if (found):
+                resolved.add(found)
+            else:
+                error_line(f'Warning: excluded file not found: {filename}')
+
+# Writes 'line' to the open 'destn' (or stdout).
+# 
+def write_line(line: str) -> None:
+    print(line, file=destn)
+
+# Logs 'line' to stderr. This is also used for general notifications that we
+# don't want to go to stdout (so the source can be piped).
+# 
+def error_line(line: Any) -> None:
+    print(line, file=sys.stderr)
+
+# Inline the contents of 'file' (with any of its includes also inlined, etc.).
+# 
+# Note: text encoding errors are ignored and replaced with ? when reading the
+# input files. This isn't ideal, but it's more than likely in the comments than
+# code and a) the text editor has probably also failed to read the same content,
+# and b) the compiler probably did too.
+# 
+def add_file(file: Path, file_name: str = None) -> None:
+    if (file.is_file()):
+        if (not file_name):
+            file_name = file.name
+        error_line(f'Processing: {file_name}')
+        with file.open('r', errors='replace') as opened:
+            for line in opened:
+                line = line.rstrip('\n')
+                match_include = include_regex.match(line);
+                if (match_include):
+                    # We have a quoted include directive so grab the file
+                    inc_name = match_include.group(1)
+                    resolved = resolve_include(inc_name, file.parent)
+                    if (resolved):
+                        if (resolved in excludes):
+                            # The file was excluded so error if the compiler uses it
+                            write_line(f'#error Using excluded file: {inc_name} (re-amalgamate source to fix)')
+                            error_line(f'Excluding: {inc_name}')
+                        else:
+                            if (resolved not in found):
+                                # The file was not previously encountered
+                                found.add(resolved)
+                                if (resolved in keeps):
+                                    # But the include was flagged to keep as included
+                                    write_line(f'/**** *NOT* inlining {inc_name} ****/')
+                                    write_line(line)
+                                    error_line(f'Not inlining: {inc_name}')
+                                else:
+                                    # The file was neither excluded nor seen before so inline it
+                                    write_line(f'/**** start inlining {inc_name} ****/')
+                                    add_file(resolved, inc_name)
+                                    write_line(f'/**** ended inlining {inc_name} ****/')
+                            else:
+                                write_line(f'/**** skipping file: {inc_name} ****/')
+                    else:
+                        # The include file didn't resolve to a file
+                        write_line(f'#error Unable to find: {inc_name}')
+                        error_line(f'Error: Unable to find: {inc_name}')
+                else:
+                    # Skip any 'pragma once' directives, otherwise write the source line
+                    if (keep_pragma or not pragma_regex.match(line)):
+                        write_line(line)
+    else:
+        error_line(f'Error: Invalid file: {file}')
+
+# Start here
+parser = argparse.ArgumentParser(description='Amalgamate Tool', epilog=f'example: {sys.argv[0]} -r ../my/path -r ../other/path -o out.c in.c')
+parser.add_argument('-r', '--root', action='append', type=Path, help='file root search path')
+parser.add_argument('-x', '--exclude',  action='append', help='file to completely exclude from inlining')
+parser.add_argument('-k', '--keep', action='append', help='file to exclude from inlining but keep the include directive')
+parser.add_argument('-p', '--pragma', action='store_true', default=False, help='keep any "#pragma once" directives (removed by default)')
+parser.add_argument('-o', '--output', type=argparse.FileType('w'), help='output file (otherwise stdout)')
+parser.add_argument('input', type=Path, help='input file')
+args = parser.parse_args()
+
+# Fail early on an invalid input (and store it so we don't recurse)
+args.input = args.input.resolve(strict=True)
+found.add(args.input)
+
+# Resolve all of the root paths upfront (we'll halt here on invalid roots)
+if (args.root):
+    for path in args.root:
+        roots.add(path.resolve(strict=True))
+
+# The remaining params: so resolve the excluded files and #pragma once directive
+resolve_excluded_files(args.exclude, excludes, args.input.parent)
+resolve_excluded_files(args.keep,    keeps,    args.input.parent)
+keep_pragma = args.pragma;
+
+# Then recursively process the input file
+try:
+    if (args.output):
+        destn = args.output
+    add_file(args.input)
+finally:
+    if (destn):
+        destn.close()
@@ -130,7 +130,7 @@ add_file() {
        local res_inc="$(resolve_include "$srcdir" "$inc")"
        if list_has_item "$XINCS" "$inc"; then
          # The file was excluded so error if the source attempts to use it
-          write_line "#error Using excluded file: $inc"
+          write_line "#error Using excluded file: $inc (re-amalgamate source to fix)"
          log_line "Excluding: $inc"
        else
          if ! list_has_item "$FOUND" "$res_inc"; then
@@ -200,6 +200,7 @@ if [ -n "$1" ]; then
      printf "" > "$DESTN"
    fi
    test_deps
+    log_line "Processing using the slower shell script; this might take a while"
    add_file "$1"
  else
    echo "Input file not found: \"$1\""
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Where to find the sources
+ZSTD_SRC_ROOT="../../lib"
+
+# Amalgamate the sources
+echo "Amalgamating files..."
+# Using the faster Python script if we have 3.8 or higher
+if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
+  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
+else
+  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
+fi
+# Did combining work?
+if [ $? -ne 0 ]; then
+  echo "Combine script: FAILED"
+  exit 1
+fi
+echo "Combine script: PASSED"
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Where to find the sources
+ZSTD_SRC_ROOT="../../lib"
+
+# Amalgamate the sources
+echo "Amalgamating files..."
+# Using the faster Python script if we have 3.8 or higher
+if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
+  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
+else
+  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
+fi
+# Did combining work?
+if [ $? -ne 0 ]; then
+  echo "Combine script: FAILED"
+  exit 1
+fi
+echo "Combine script: PASSED"
@@ -4,11 +4,11 @@
 *
 * Generate using:
 * \code
- *	combine.sh -r ../../lib -o zstd.c zstd-in.c
+ *	python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
 * \endcode
 */
 /*
- * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -25,9 +25,13 @@
 * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
 * defined in mem.h (breaking C99 compatibility).
 *
- * Note: the undefs for xxHash allow Zstd's implementation to coinside with with
+ * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
 * standalone xxHash usage (with global defines).
 *
+ * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need
+ * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the
+ * legacy support at the source level).
+ *
 * Note: multithreading is enabled for all platforms apart from Emscripten.
 */
 #define DEBUGLEVEL 0
@@ -43,6 +47,8 @@
 #define ZSTD_MULTITHREAD
 #endif
 #define ZSTD_TRACE 0
+/* TODO: Can't amalgamate ASM function */
+#define ZSTD_DISABLE_ASM 1

 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC
@@ -4,11 +4,11 @@
 *
 * Generate using:
 * \code
- *	combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
+ *	python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
 * \endcode
 */
 /*
- * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -25,8 +25,12 @@
 * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
 * defined in mem.h (breaking C99 compatibility).
 *
- * Note: the undefs for xxHash allow Zstd's implementation to coinside with with
+ * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
 * standalone xxHash usage (with global defines).
+ *
+ * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need
+ * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the
+ * legacy support at the source level).
 */
 #define DEBUGLEVEL 0
 #define MEM_MODULE
@@ -39,6 +43,8 @@
 #define ZSTD_LEGACY_SUPPORT 0
 #define ZSTD_STRIP_ERROR_STRINGS
 #define ZSTD_TRACE 0
+/* TODO: Can't amalgamate ASM function */
+#define ZSTD_DISABLE_ASM 1

 /* Include zstd_deps.h first with all the options we need enabled. */
 #define ZSTD_DEPS_NEED_MALLOC
@@ -390,7 +390,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -426,7 +426,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -454,7 +454,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -363,6 +363,10 @@
 				RelativePath="..\..\..\programs\fileio.c"
 				>
 			</File>
+			<File
+                RelativePath="..\..\..\programs\fileio_asyncio.c"
+                >
+            </File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>
@@ -454,7 +458,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -482,7 +486,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -371,18 +371,6 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
 				>
@@ -458,7 +446,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zstd_errors.h"
+				RelativePath="..\..\..\lib\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -486,11 +474,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\common\zbuff.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+				RelativePath="..\..\..\lib\zdict.h"
 				>
 			</File>
 			<File
@@ -27,6 +27,7 @@ SKIPPED_FILES = [
    "common/pool.h",
    "common/threading.c",
    "common/threading.h",
+    "common/zstd_trace.h",
    "compress/zstdmt_compress.h",
    "compress/zstdmt_compress.c",
 ]
@@ -430,7 +431,7 @@ class Freestanding(object):
            external_xxhash: bool, xxh64_state: Optional[str],
            xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
            defs: [(str, Optional[str])], replaces: [(str, str)],
-            undefs: [str], excludes: [str]
+            undefs: [str], excludes: [str], seds: [str],
    ):
        self._zstd_deps = zstd_deps
        self._mem = mem
@@ -444,6 +445,7 @@ class Freestanding(object):
        self._replaces = replaces
        self._undefs = undefs
        self._excludes = excludes
+        self._seds = seds

    def _dst_lib_file_paths(self):
        """
@@ -458,7 +460,8 @@ class Freestanding(object):
        print(*args, **kwargs)

    def _copy_file(self, lib_path):
-        if not (lib_path.endswith(".c") or lib_path.endswith(".h")):
+        suffixes = [".c", ".h", ".S"]
+        if not any((lib_path.endswith(suffix) for suffix in suffixes)):
            return
        if lib_path in SKIPPED_FILES:
            self._log(f"\tSkipping file: {lib_path}")
@@ -471,24 +474,25 @@ class Freestanding(object):
        dst_path = os.path.join(self._dst_lib, lib_path)
        self._log(f"\tCopying: {src_path} -> {dst_path}")
        shutil.copyfile(src_path, dst_path)
-    
+
    def _copy_source_lib(self):
        self._log("Copying source library into output library")

        assert os.path.exists(self._src_lib)
        os.makedirs(self._dst_lib, exist_ok=True)
        self._copy_file("zstd.h")
+        self._copy_file("zstd_errors.h")
        for subdir in INCLUDED_SUBDIRS:
            src_dir = os.path.join(self._src_lib, subdir)
            dst_dir = os.path.join(self._dst_lib, subdir)
-            
+
            assert os.path.exists(src_dir)
            os.makedirs(dst_dir, exist_ok=True)

            for filename in os.listdir(src_dir):
                lib_path = os.path.join(subdir, filename)
                self._copy_file(lib_path)
-    
+
    def _copy_zstd_deps(self):
        dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
        self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
@@ -508,7 +512,7 @@ class Freestanding(object):
        assert not (undef and value is not None)
        for filepath in self._dst_lib_file_paths():
            file = FileLines(filepath)
-    
+
    def _hardwire_defines(self):
        self._log("Hardwiring macros")
        partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs)
@@ -536,7 +540,7 @@ class Freestanding(object):
                        skipped.append(line)
                        if end_re.search(line) is not None:
                            assert begin_re.search(line) is None
-                            self._log(f"\t\tRemoving excluded section: {exclude}") 
+                            self._log(f"\t\tRemoving excluded section: {exclude}")
                            for s in skipped:
                                self._log(f"\t\t\t- {s}")
                            emit = True
@@ -559,12 +563,12 @@ class Freestanding(object):
                e = match.end('include')
                file.lines[i] = line[:s] + rewritten + line[e:]
            file.write()
-    
+
    def _rewrite_includes(self):
        self._log("Rewriting includes")
        for original, rewritten in self._rewritten_includes:
            self._rewrite_include(original, rewritten)
-    
+
    def _replace_xxh64_prefix(self):
        if self._xxh64_prefix is None:
            return
@@ -596,6 +600,48 @@ class Freestanding(object):
                file.lines[i] = line
            file.write()

+    def _parse_sed(self, sed):
+        assert sed[0] == 's'
+        delim = sed[1]
+        match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed)
+        assert match is not None
+        regex = re.compile(match.group(1))
+        format_str = match.group(2)
+        is_global = match.group(3) == 'g'
+        return regex, format_str, is_global
+
+    def _process_sed(self, sed):
+        self._log(f"Processing sed: {sed}")
+        regex, format_str, is_global = self._parse_sed(sed)
+
+        for filepath in self._dst_lib_file_paths():
+            file = FileLines(filepath)
+            for i, line in enumerate(file.lines):
+                modified = False
+                while True:
+                    match = regex.search(line)
+                    if match is None:
+                        break
+                    replacement = format_str.format(match.groups(''), match.groupdict(''))
+                    b = match.start()
+                    e = match.end()
+                    line = line[:b] + replacement + line[e:]
+                    modified = True
+                    if not is_global:
+                        break
+                if modified:
+                    self._log(f"\t- {file.lines[i][:-1]}")
+                    self._log(f"\t+ {line[:-1]}")
+                file.lines[i] = line
+            file.write()
+
+    def _process_seds(self):
+        self._log("Processing seds")
+        for sed in self._seds:
+            self._process_sed(sed)
+
+
+
    def go(self):
        self._copy_source_lib()
        self._copy_zstd_deps()
@@ -604,6 +650,7 @@ class Freestanding(object):
        self._remove_excludes()
        self._rewrite_includes()
        self._replace_xxh64_prefix()
+        self._process_seds()


 def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
@@ -641,6 +688,7 @@ def main(name, args):
    parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'")
    parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
    parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
+    parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
    parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
    parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed mutliple times)")
    parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
@@ -656,6 +704,11 @@ def main(name, args):
        if name in args.undefs:
            raise RuntimeError(f"{name} is both defined and undefined!")

+    # Always set tracing to 0
+    if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs):
+        args.defs.append(("ZSTD_NO_TRACE", None))
+        args.defs.append(("ZSTD_TRACE", "0"))
+
    args.replaces = parse_pair(args.replaces)
    for name, _ in args.replaces:
        if name in args.undefs or name in args.defs:
@@ -688,7 +741,8 @@ def main(name, args):
        args.defs,
        args.replaces,
        args.undefs,
-        args.excludes
+        args.excludes,
+        args.seds,
    ).go()

 if __name__ == "__main__":
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2021, Facebook, Inc.
+# Copyright (c) Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -22,6 +22,10 @@ libzstd:
 		--xxh64-prefix 'xxh64' \
 		--rewrite-include '<limits\.h>=<linux/limits.h>' \
 		--rewrite-include '<stddef\.h>=<linux/types.h>' \
+		--rewrite-include '"\.\./zstd.h"=<linux/zstd.h>' \
+		--rewrite-include '"(\.\./)?zstd_errors.h"=<linux/zstd_errors.h>' \
+		--sed 's,/\*\*\*,/* *,g' \
+		--sed 's,/\*\*,/*,g' \
 		-DZSTD_NO_INTRINSICS \
 		-DZSTD_NO_UNUSED_FUNCTIONS \
 		-DZSTD_LEGACY_SUPPORT=0 \
@@ -31,11 +35,12 @@ libzstd:
 		-DXXH_STATIC_LINKING_ONLY \
 		-DMEM_FORCE_MEMORY_ACCESS=0 \
 		-D__GNUC__ \
+		-D__linux__=1 \
 		-DSTATIC_BMI2=0 \
 		-DZSTD_ADDRESS_SANITIZER=0 \
 		-DZSTD_MEMORY_SANITIZER=0 \
+		-DZSTD_DATAFLOW_SANITIZER=0 \
 		-DZSTD_COMPRESS_HEAPMODE=1 \
-		-UZSTD_NO_INLINE \
 		-UNO_PREFETCH \
 		-U__cplusplus \
 		-UZSTD_DLL_EXPORT \
@@ -46,8 +51,13 @@ libzstd:
 		-U_WIN32 \
 		-RZSTDLIB_VISIBILITY= \
 		-RZSTDERRORLIB_VISIBILITY= \
+		-RZSTD_FALLTHROUGH=fallthrough \
 		-DZSTD_HAVE_WEAK_SYMBOLS=0 \
-		-DZSTD_TRACE=0
+		-DZSTD_TRACE=0 \
+		-DZSTD_NO_TRACE \
+		-DZSTD_LINUX_KERNEL
+	mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
+	mv linux/lib/zstd/zstd_errors.h linux/include/linux/
 	cp linux_zstd.h linux/include/linux/zstd.h
 	cp zstd_compress_module.c linux/lib/zstd
 	cp zstd_decompress_module.c linux/lib/zstd
@@ -62,24 +72,34 @@ import: libzstd
 	rm -f $(LINUX)/include/linux/zstd_errors.h
 	rm -rf $(LINUX)/lib/zstd
 	cp linux/include/linux/zstd.h $(LINUX)/include/linux
+	cp linux/include/linux/zstd_lib.h $(LINUX)/include/linux
+	cp linux/include/linux/zstd_errors.h $(LINUX)/include/linux
 	cp -r linux/lib/zstd $(LINUX)/lib

 import-upstream:
 	rm -rf $(LINUX)/lib/zstd
 	mkdir $(LINUX)/lib/zstd
-	cp ../../lib/zstd.h $(LINUX)/lib/zstd
+	cp ../../lib/zstd.h $(LINUX)/include/linux/zstd_lib.h
 	cp -r ../../lib/common $(LINUX)/lib/zstd
 	cp -r ../../lib/compress $(LINUX)/lib/zstd
 	cp -r ../../lib/decompress $(LINUX)/lib/zstd
+	mv $(LINUX)/lib/zstd/zstd_errors.h $(LINUX)/include/linux
 	rm $(LINUX)/lib/zstd/common/threading.*
 	rm $(LINUX)/lib/zstd/common/pool.*
 	rm $(LINUX)/lib/zstd/common/xxhash.*
 	rm $(LINUX)/lib/zstd/compress/zstdmt_*

+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat \
+	    -Wimplicit-fallthrough
+
 .PHONY: test
 test: libzstd
-	$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS)" -j
+	$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS) $(DEBUGFLAGS) -Werror" -j

 .PHONY: clean
 clean:
-	$(RM) -rf linux
+	$(RM) -rf linux test/test test/static_test
@@ -1,4 +1,13 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */

 /*
 * This file includes every .c file needed for decompression.
@@ -7,6 +16,12 @@
 * decompression.
 */

+/*
+ * Disable the ASM Huffman implementation because we need to
+ * include all the sources.
+ */
+#define ZSTD_DISABLE_ASM 1
+
 #include "common/debug.c"
 #include "common/entropy_common.c"
 #include "common/error_private.c"
@@ -1,8 +1,17 @@
-# SPDX-License-Identifier: GPL-2.0-only
+# SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
 obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
 obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o

-ccflags-y += -O3
+ccflags-y += -Wno-error=deprecated-declarations

 zstd_compress-y := \
 		zstd_compress_module.o \
@@ -32,6 +41,7 @@ zstd_decompress-y := \
 		common/fse_decompress.o \
 		common/zstd_common.o \
 		decompress/huf_decompress.o \
+		decompress/huf_decompress_amd64.o \
 		decompress/zstd_ddict.o \
 		decompress/zstd_decompress.o \
 		decompress/zstd_decompress_block.o \
@@ -1,18 +1,13 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
 * All rights reserved.
 *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of https://github.com/facebook/zstd.
- * An additional grant of patent rights can be found in the PATENTS file in the
- * same directory.
- *
- * This program is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License version 2 as published by the
- * Free Software Foundation. This program is dual-licensed; you may select
- * either version 2 of the GNU General Public License ("GPL") or BSD license
- * ("BSD").
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd) and
+ * the GPLv2 (found in the COPYING file in the root directory of
+ * https://github.com/facebook/zstd). You may select, at your option, one of the
+ * above-listed licenses.
 */

 #ifndef LINUX_ZSTD_H
@@ -27,6 +22,8 @@

 /* ======   Dependency   ====== */
 #include <linux/types.h>
+#include <linux/zstd_errors.h>
+#include <linux/zstd_lib.h>

 /* ======   Helper Functions   ====== */
 /**
@@ -45,13 +42,18 @@ size_t zstd_compress_bound(size_t src_size);
 */
 unsigned int zstd_is_error(size_t code);

+/**
+ * enum zstd_error_code - zstd error codes
+ */
+typedef ZSTD_ErrorCode zstd_error_code;
+
 /**
 * zstd_get_error_code() - translates an error function result to an error code
 * @code:  The function result for which zstd_is_error(code) is true.
 *
 * Return: A unique error code for this error.
 */
-int zstd_get_error_code(size_t code);
+zstd_error_code zstd_get_error_code(size_t code);

 /**
 * zstd_get_error_name() - translates an error function result to a string
@@ -61,76 +63,67 @@ int zstd_get_error_code(size_t code);
 */
 const char *zstd_get_error_name(size_t code);

+/**
+ * zstd_min_clevel() - minimum allowed compression level
+ *
+ * Return: The minimum allowed compression level.
+ */
+int zstd_min_clevel(void);
+
+/**
+ * zstd_max_clevel() - maximum allowed compression level
+ *
+ * Return: The maximum allowed compression level.
+ */
+int zstd_max_clevel(void);
+
 /* ======   Parameter Selection   ====== */

 /**
 * enum zstd_strategy - zstd compression search strategy
 *
- * From faster to stronger.
+ * From faster to stronger. See zstd_lib.h.
 */
-enum zstd_strategy {
-	zstd_fast = 1,
-	zstd_dfast = 2,
-	zstd_greedy = 3,
-	zstd_lazy = 4,
-	zstd_lazy2 = 5,
-	zstd_btlazy2 = 6,
-	zstd_btopt = 7,
-	zstd_btultra = 8,
-	zstd_btultra2 = 9
-};
+typedef ZSTD_strategy zstd_strategy;

 /**
 * struct zstd_compression_parameters - zstd compression parameters
- * @window_log:    Log of the largest match distance. Larger means more
- *                 compression, and more memory needed during decompression.
- * @chain_log:     Fully searched segment. Larger means more compression,
- *                 slower, and more memory (useless for fast).
- * @hash_log:      Dispatch table. Larger means more compression,
- *                 slower, and more memory.
- * @search_log:    Number of searches. Larger means more compression and slower.
- * @search_length: Match length searched. Larger means faster decompression,
- *                 sometimes less compression.
- * @target_length: Acceptable match size for optimal parser (only). Larger means
- *                 more compression, and slower.
- * @strategy:      The zstd compression strategy.
+ * @windowLog:    Log of the largest match distance. Larger means more
+ *                compression, and more memory needed during decompression.
+ * @chainLog:     Fully searched segment. Larger means more compression,
+ *                slower, and more memory (useless for fast).
+ * @hashLog:      Dispatch table. Larger means more compression,
+ *                slower, and more memory.
+ * @searchLog:    Number of searches. Larger means more compression and slower.
+ * @searchLength: Match length searched. Larger means faster decompression,
+ *                sometimes less compression.
+ * @targetLength: Acceptable match size for optimal parser (only). Larger means
+ *                more compression, and slower.
+ * @strategy:     The zstd compression strategy.
+ *
+ * See zstd_lib.h.
 */
-struct zstd_compression_parameters {
-	unsigned int window_log;
-	unsigned int chain_log;
-	unsigned int hash_log;
-	unsigned int search_log;
-	unsigned int search_length;
-	unsigned int target_length;
-	enum zstd_strategy strategy;
-};
+typedef ZSTD_compressionParameters zstd_compression_parameters;

 /**
 * struct zstd_frame_parameters - zstd frame parameters
- * @content_size_flag: Controls whether content size will be present in the
- *                     frame header (when known).
- * @checksum_flag:     Controls whether a 32-bit checksum is generated at the
- *                     end of the frame for error detection.
- * @no_dict_id_flag:   Controls whether dictID will be saved into the frame
- *                     header when using dictionary compression.
+ * @contentSizeFlag: Controls whether content size will be present in the
+ *                   frame header (when known).
+ * @checksumFlag:    Controls whether a 32-bit checksum is generated at the
+ *                   end of the frame for error detection.
+ * @noDictIDFlag:    Controls whether dictID will be saved into the frame
+ *                   header when using dictionary compression.
 *
- * The default value is all fields set to 0.
+ * The default value is all fields set to 0. See zstd_lib.h.
 */
-struct zstd_frame_parameters {
-	unsigned int content_size_flag;
-	unsigned int checksum_flag;
-	unsigned int no_dict_id_flag;
-};
+typedef ZSTD_frameParameters zstd_frame_parameters;

 /**
 * struct zstd_parameters - zstd parameters
- * @cparams: The compression parameters.
- * @fparams: The frame parameters.
+ * @cParams: The compression parameters.
+ * @fParams: The frame parameters.
 */
-struct zstd_parameters {
-	struct zstd_compression_parameters cparams;
-	struct zstd_frame_parameters fparams;
-};
+typedef ZSTD_parameters zstd_parameters;

 /**
 * zstd_get_params() - returns zstd_parameters for selected level
@@ -140,12 +133,12 @@ struct zstd_parameters {
 *
 * Return:              The selected zstd_parameters.
 */
-struct zstd_parameters zstd_get_params(int level,
+zstd_parameters zstd_get_params(int level,
 	unsigned long long estimated_src_size);

 /* ======   Single-pass Compression   ====== */

-typedef struct ZSTD_CCtx_s zstd_cctx;
+typedef ZSTD_CCtx zstd_cctx;

 /**
 * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
@@ -158,8 +151,7 @@ typedef struct ZSTD_CCtx_s zstd_cctx;
 * Return:      A lower bound on the size of the workspace that is passed to
 *              zstd_init_cctx().
 */
-size_t zstd_cctx_workspace_bound(
-	const struct zstd_compression_parameters *parameters);
+size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);

 /**
 * zstd_init_cctx() - initialize a zstd compression context
@@ -186,11 +178,11 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
 *                zstd_is_error().
 */
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
-	const void *src, size_t src_size, const struct zstd_parameters *parameters);
+	const void *src, size_t src_size, const zstd_parameters *parameters);

 /* ======   Single-pass Decompression   ====== */

-typedef struct ZSTD_DCtx_s zstd_dctx;
+typedef ZSTD_DCtx zstd_dctx;

 /**
 * zstd_dctx_workspace_bound() - max memory needed to initialize a zstd_dctx
@@ -236,12 +228,10 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 * @size: Size of the input buffer.
 * @pos:  Position where reading stopped. Will be updated.
 *        Necessarily 0 <= pos <= size.
+ *
+ * See zstd_lib.h.
 */
-struct zstd_in_buffer {
-	const void *src;
-	size_t size;
-	size_t pos;
-};
+typedef ZSTD_inBuffer zstd_in_buffer;

 /**
 * struct zstd_out_buffer - output buffer for streaming
@@ -249,16 +239,14 @@ struct zstd_in_buffer {
 * @size: Size of the output buffer.
 * @pos:  Position where writing stopped. Will be updated.
 *        Necessarily 0 <= pos <= size.
+ *
+ * See zstd_lib.h.
 */
-struct zstd_out_buffer {
-	void *dst;
-	size_t size;
-	size_t pos;
-};
+typedef ZSTD_outBuffer zstd_out_buffer;

 /* ======   Streaming Compression   ====== */

-typedef struct ZSTD_CCtx_s zstd_cstream;
+typedef ZSTD_CStream zstd_cstream;

 /**
 * zstd_cstream_workspace_bound() - memory needed to initialize a zstd_cstream
@@ -267,8 +255,7 @@ typedef struct ZSTD_CCtx_s zstd_cstream;
 * Return:   A lower bound on the size of the workspace that is passed to
 *           zstd_init_cstream().
 */
-size_t zstd_cstream_workspace_bound(
-	const struct zstd_compression_parameters *cparams);
+size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);

 /**
 * zstd_init_cstream() - initialize a zstd streaming compression context
@@ -285,7 +272,7 @@ size_t zstd_cstream_workspace_bound(
 *
 * Return:            The zstd streaming compression context or NULL on error.
 */
-zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
+zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
 	unsigned long long pledged_src_size, void *workspace, size_t workspace_size);

 /**
@@ -320,8 +307,8 @@ size_t zstd_reset_cstream(zstd_cstream *cstream,
 *           function call or an error, which can be checked using
 *           zstd_is_error().
 */
-size_t zstd_compress_stream(zstd_cstream *cstream,
-	struct zstd_out_buffer *output, struct zstd_in_buffer *input);
+size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
+	zstd_in_buffer *input);

 /**
 * zstd_flush_stream() - flush internal buffers into output
@@ -336,7 +323,7 @@ size_t zstd_compress_stream(zstd_cstream *cstream,
 * Return:   The number of bytes still present within internal buffers or an
 *           error, which can be checked using zstd_is_error().
 */
-size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
+size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output);

 /**
 * zstd_end_stream() - flush internal buffers into output and end the frame
@@ -350,11 +337,11 @@ size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
 * Return:   The number of bytes still present within internal buffers or an
 *           error, which can be checked using zstd_is_error().
 */
-size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
+size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output);

 /* ======   Streaming Decompression   ====== */

-typedef struct ZSTD_DCtx_s zstd_dstream;
+typedef ZSTD_DStream zstd_dstream;

 /**
 * zstd_dstream_workspace_bound() - memory needed to initialize a zstd_dstream
@@ -411,8 +398,8 @@ size_t zstd_reset_dstream(zstd_dstream *dstream);
 *           using zstd_is_error(). The size hint will never load more than the
 *           frame.
 */
-size_t zstd_decompress_stream(zstd_dstream *dstream,
-	struct zstd_out_buffer *output, struct zstd_in_buffer *input);
+size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
+	zstd_in_buffer *input);

 /* ======   Frame Inspection Functions ====== */

@@ -431,20 +418,21 @@ size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);

 /**
 * struct zstd_frame_params - zstd frame parameters stored in the frame header
- * @frame_content_size: The frame content size, or 0 if not present.
- * @window_size:        The window size, or 0 if the frame is a skippable frame.
- * @dict_id:            The dictionary id, or 0 if not present.
- * @checksum_flag:      Whether a checksum was used.
+ * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
+ *                    present.
+ * @windowSize:       The window size, or 0 if the frame is a skippable frame.
+ * @blockSizeMax:     The maximum block size.
+ * @frameType:        The frame type (zstd or skippable)
+ * @headerSize:       The size of the frame header.
+ * @dictID:           The dictionary id, or 0 if not present.
+ * @checksumFlag:     Whether a checksum was used.
+ *
+ * See zstd_lib.h.
 */
-struct zstd_frame_params {
-	unsigned long long frame_content_size;
-	unsigned int window_size;
-	unsigned int dict_id;
-	unsigned int checksum_flag;
-};
+typedef ZSTD_frameHeader zstd_frame_header;

 /**
- * zstd_get_frame_params() - extracts parameters from a zstd or skippable frame
+ * zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
 * @params:   On success the frame parameters are written here.
 * @src:      The source buffer. It must point to a zstd or skippable frame.
 * @src_size: The size of the source buffer.
@@ -453,7 +441,7 @@ struct zstd_frame_params {
 *            must be provided to make forward progress. Otherwise it returns
 *            an error, which can be checked using zstd_is_error().
 */
-size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src,
+size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
 	size_t src_size);

 #endif  /* LINUX_ZSTD_H */
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
+ * Copyright (c) Yann Collet, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -29,6 +30,8 @@
 *  Basic Types
 *****************************************************************/
 typedef uint8_t  BYTE;
+typedef uint8_t  U8;
+typedef int8_t   S8;
 typedef uint16_t U16;
 typedef int16_t  S16;
 typedef uint32_t U32;
@@ -1,17 +1,30 @@
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################

 LINUX := ../linux
 LINUX_ZSTDLIB := $(LINUX)/lib/zstd

-CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG
+CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations
 # Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse
 CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE

 LINUX_ZSTD_MODULE     := $(wildcard $(LINUX_ZSTDLIB)/*.c)
-LINUX_ZSTD_COMMON     := $(wildcard $(LINUX_ZSTDLIB)/common/*.c) 
-LINUX_ZSTD_COMPRESS   := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c) 
-LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c) 
+LINUX_ZSTD_COMMON     := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
+LINUX_ZSTD_COMPRESS   := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
+LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c $(LINUX_ZSTDLIB)/decompress/*.S)
 LINUX_ZSTD_FILES      := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
-LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_FILES:.c=.o)
+LINUX_ZSTD_OBJECTS0   := $(LINUX_ZSTD_FILES:.c=.o)
+LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_OBJECTS0:.S=.o)
+
+%.o: %.S
+	$(COMPILE.S) $(OUTPUT_OPTION) $<

 liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
 	$(AR) $(ARFLAGS) $@ $^
@@ -29,6 +42,7 @@ run-test: test static_test

 .PHONY:
 clean:
+	$(RM) -f $(LINUX_ZSTDLIB)/*.o
 	$(RM) -f $(LINUX_ZSTDLIB)/**/*.o
 	$(RM) -f *.o *.a
 	$(RM) -f test
@@ -4,13 +4,24 @@
 #include <assert.h>
 #include <linux/types.h>

-#define _LITTLE_ENDIAN 1
+#ifndef __LITTLE_ENDIAN
+# if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN__)
+#  define __LITTLE_ENDIAN 1
+# endif
+#endif
+
+#ifdef __LITTLE_ENDIAN
+# define _IS_LITTLE_ENDIAN 1
+#else
+# define _IS_LITTLE_ENDIAN 0
+#endif

 static unsigned _isLittleEndian(void)
 {
    const union { uint32_t u; uint8_t c[4]; } one = { 1 };
-    assert(_LITTLE_ENDIAN == one.c[0]);
-    return _LITTLE_ENDIAN;
+    assert(_IS_LITTLE_ENDIAN == one.c[0]);
+    (void)one;
+    return _IS_LITTLE_ENDIAN;
 }

 static uint16_t _swap16(uint16_t in)
@@ -165,7 +176,7 @@ extern void __bad_unaligned_access_size(void);
    (void)0;                                                                   \
  })

-#if _LITTLE_ENDIAN
+#if _IS_LITTLE_ENDIAN
 #  define get_unaligned __get_unaligned_le
 #  define put_unaligned __put_unaligned_le
 #else
@@ -14,4 +14,10 @@
 #define inline __inline __attribute__((unused))
 #endif

+#ifndef noinline
+#define noinline __attribute__((noinline))
+#endif
+
+#define fallthrough __attribute__((__fallthrough__))
+
 #endif
@@ -12,4 +12,8 @@

 #define WARN_ON(x)

+#define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a))
+#define ALIGN(x, a)         ALIGN_MASK((x), (a) - 1)
+#define ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
 #endif
@@ -124,11 +124,10 @@ XXH_API uint64_t xxh64(const void *input, size_t length, uint64_t seed);
 static inline unsigned long xxhash(const void *input, size_t length,
 				   uint64_t seed)
 {
-#if BITS_PER_LONG == 64
-       return xxh64(input, length, seed);
-#else
-       return xxh32(input, length, seed);
-#endif
+	if (sizeof(size_t) == 8)
+		return xxh64(input, length, seed);
+	else
+		return xxh32(input, length, seed);
 }

 /*-****************************
@@ -36,9 +36,9 @@ test_not_present "ZSTD_NO_INTRINSICS"
 test_not_present "ZSTD_NO_UNUSED_FUNCTIONS"
 test_not_present "ZSTD_LEGACY_SUPPORT"
 test_not_present "STATIC_BMI2"
-test_not_present "ZSTD_NO_INLINE"
 test_not_present "ZSTD_DLL_EXPORT"
 test_not_present "ZSTD_DLL_IMPORT"
 test_not_present "__ICCARM__"
 test_not_present "_MSC_VER"
 test_not_present "_WIN32"
+test_not_present "__linux__"
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -28,17 +28,19 @@ static const char kEmptyZstdFrame[] = {
    0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x00, 0x01, 0x00, 0x00, 0x99, 0xe9, 0xd8, 0x51
 };

-static void test_decompress_unzstd() {
+static void test_decompress_unzstd(void) {
    fprintf(stderr, "Testing decompress unzstd... ");
    {
        size_t const wkspSize = zstd_dctx_workspace_bound();
        void* wksp = malloc(wkspSize);
-        CONTROL(wksp != NULL);
        ZSTD_DCtx* dctx = zstd_init_dctx(wksp, wkspSize);
+        CONTROL(wksp != NULL);
        CONTROL(dctx != NULL);
-        size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame));
-        CONTROL(!zstd_is_error(dSize));
-        CONTROL(dSize == 0);
+        {
+          size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame));
+          CONTROL(!zstd_is_error(dSize));
+          CONTROL(dSize == 0);
+        }
        free(wksp);
    }
    fprintf(stderr, "Ok\n");
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 7-2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -30,15 +30,15 @@ typedef struct {
  size_t compSize;
 } test_data_t;

-test_data_t create_test_data(void) {
+static test_data_t create_test_data(void) {
  test_data_t data;
  data.dataSize = 128 * 1024;
-  data.data = malloc(data.dataSize);
+  data.data = (char*)malloc(data.dataSize);
  CONTROL(data.data != NULL);
-  data.data2 = malloc(data.dataSize);
+  data.data2 = (char*)malloc(data.dataSize);
  CONTROL(data.data2 != NULL);
  data.compSize = zstd_compress_bound(data.dataSize);
-  data.comp = malloc(data.compSize);
+  data.comp = (char*)malloc(data.compSize);
  CONTROL(data.comp != NULL);
  memset(data.data, 0, data.dataSize);
  return data;
@@ -54,26 +54,27 @@ static void free_test_data(test_data_t const *data) {
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

 static void test_btrfs(test_data_t const *data) {
-  fprintf(stderr, "testing btrfs use cases... ");
  size_t const size = MIN(data->dataSize, 128 * 1024);
+  fprintf(stderr, "testing btrfs use cases... ");
  for (int level = -1; level < 16; ++level) {
-    struct zstd_parameters params = zstd_get_params(level, size);
-    CONTROL(params.cparams.window_log <= 17);
+    zstd_parameters params = zstd_get_params(level, size);
    size_t const workspaceSize =
-        MAX(zstd_cstream_workspace_bound(&params.cparams),
+        MAX(zstd_cstream_workspace_bound(&params.cParams),
            zstd_dstream_workspace_bound(size));
    void *workspace = malloc(workspaceSize);
-    CONTROL(workspace != NULL);

    char const *ip = data->data;
    char const *iend = ip + size;
    char *op = data->comp;
    char *oend = op + data->compSize;
+
+    CONTROL(params.cParams.windowLog <= 17);
+    CONTROL(workspace != NULL);
    {
      zstd_cstream *cctx = zstd_init_cstream(&params, size, workspace, workspaceSize);
+      zstd_out_buffer out = {NULL, 0, 0};
+      zstd_in_buffer in = {NULL, 0, 0};
      CONTROL(cctx != NULL);
-      struct zstd_out_buffer out = {NULL, 0, 0};
-      struct zstd_in_buffer in = {NULL, 0, 0};
      for (;;) {
        if (in.pos == in.size) {
          in.src = ip;
@@ -107,10 +108,10 @@ static void test_btrfs(test_data_t const *data) {
    op = data->data2;
    oend = op + size;
    {
-      zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cparams.window_log, workspace, workspaceSize);
+      zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cParams.windowLog, workspace, workspaceSize);
+      zstd_out_buffer out = {NULL, 0, 0};
+      zstd_in_buffer in = {NULL, 0, 0};
      CONTROL(dctx != NULL);
-      struct zstd_out_buffer out = {NULL, 0, 0};
-      struct zstd_in_buffer in = {NULL, 0, 0};
      for (;;) {
        if (in.pos == in.size) {
          in.src = ip;
@@ -125,15 +126,16 @@ static void test_btrfs(test_data_t const *data) {
          out.pos = 0;
          op += out.size;
        }
-
-        size_t const ret = zstd_decompress_stream(dctx, &out, &in);
-        CONTROL(!zstd_is_error(ret));
-        if (ret == 0) {
-          break;
+        {
+          size_t const ret = zstd_decompress_stream(dctx, &out, &in);
+          CONTROL(!zstd_is_error(ret));
+          if (ret == 0) {
+            break;
+          }
        }
      }
    }
-    CONTROL(op - data->data2 == data->dataSize);
+    CONTROL((size_t)(op - data->data2) == data->dataSize);
    CONTROL(!memcmp(data->data, data->data2, data->dataSize));
    free(workspace);
  }
@@ -141,14 +143,14 @@ static void test_btrfs(test_data_t const *data) {
 }

 static void test_decompress_unzstd(test_data_t const *data) {
-    fprintf(stderr, "Testing decompress unzstd... ");
    size_t cSize;
+    fprintf(stderr, "Testing decompress unzstd... ");
    {
-        struct zstd_parameters params = zstd_get_params(19, 0);
-        size_t const wkspSize = zstd_cctx_workspace_bound(&params.cparams);
+        zstd_parameters params = zstd_get_params(19, 0);
+        size_t const wkspSize = zstd_cctx_workspace_bound(&params.cParams);
        void* wksp = malloc(wkspSize);
-        CONTROL(wksp != NULL);
        zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize);
+        CONTROL(wksp != NULL);
        CONTROL(cctx != NULL);
        cSize = zstd_compress_cctx(cctx, data->comp, data->compSize, data->data, data->dataSize, &params);
        CONTROL(!zstd_is_error(cSize));
@@ -157,25 +159,34 @@ static void test_decompress_unzstd(test_data_t const *data) {
    {
        size_t const wkspSize = zstd_dctx_workspace_bound();
        void* wksp = malloc(wkspSize);
-        CONTROL(wksp != NULL);
        zstd_dctx* dctx = zstd_init_dctx(wksp, wkspSize);
+        CONTROL(wksp != NULL);
        CONTROL(dctx != NULL);
-        size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize);
-        CONTROL(!zstd_is_error(dSize));
-        CONTROL(dSize == data->dataSize);
+        {
+          size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize);
+          CONTROL(!zstd_is_error(dSize));
+          CONTROL(dSize == data->dataSize);
+        }
        CONTROL(!memcmp(data->data, data->data2, data->dataSize));
        free(wksp);
    }
    fprintf(stderr, "Ok\n");
 }

+static void test_f2fs(void) {
+  fprintf(stderr, "testing f2fs uses... ");
+  CONTROL(zstd_min_clevel() < 0);
+  CONTROL(zstd_max_clevel() == 22);
+  fprintf(stderr, "Ok\n");
+}
+
 static char *g_stack = NULL;

 static void __attribute__((noinline)) use(void *x) {
  asm volatile("" : "+r"(x));
 }

-static void __attribute__((noinline)) set_stack() {
+static void __attribute__((noinline)) set_stack(void) {

  char stack[8192];
  g_stack = stack;
@@ -183,18 +194,21 @@ static void __attribute__((noinline)) set_stack() {
  use(g_stack);
 }

-static void __attribute__((noinline)) check_stack() {
+static void __attribute__((noinline)) check_stack(void) {
  size_t cleanStack = 0;
  while (cleanStack < 8192 && g_stack[cleanStack] == 0x33) {
    ++cleanStack;
  }
-  size_t const stackSize = 8192 - cleanStack;
-  fprintf(stderr, "Maximum stack size: %zu\n", stackSize);
-  CONTROL(stackSize <= 2048 + 512);
+  {
+    size_t const stackSize = 8192 - cleanStack;
+    fprintf(stderr, "Maximum stack size: %zu\n", stackSize);
+    CONTROL(stackSize <= 2048 + 512);
+  }
 }

 static void test_stack_usage(test_data_t const *data) {
  set_stack();
+  test_f2fs();
  test_btrfs(data);
  test_decompress_unzstd(data);
  check_stack();
@@ -202,6 +216,7 @@ static void test_stack_usage(test_data_t const *data) {

 int main(void) {
  test_data_t data = create_test_data();
+  test_f2fs();
  test_btrfs(&data);
  test_decompress_unzstd(&data);
  test_stack_usage(&data);
@@ -1,102 +1,87 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */

 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/zstd.h>

-#include "zstd.h"
 #include "common/zstd_deps.h"
 #include "common/zstd_internal.h"

-static void zstd_check_structs(void) {
-	/* Check that the structs have the same size. */
-	ZSTD_STATIC_ASSERT(sizeof(ZSTD_parameters) ==
-		sizeof(struct zstd_parameters));
-	ZSTD_STATIC_ASSERT(sizeof(ZSTD_compressionParameters) ==
-		sizeof(struct zstd_compression_parameters));
-	ZSTD_STATIC_ASSERT(sizeof(ZSTD_frameParameters) ==
-		sizeof(struct zstd_frame_parameters));
-	/* Zstd guarantees that the layout of the structs never change. Verify it. */
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, cParams) ==
-		offsetof(struct zstd_parameters, cparams));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, fParams) ==
-		offsetof(struct zstd_parameters, fparams));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, windowLog) ==
-		offsetof(struct zstd_compression_parameters, window_log));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, chainLog) ==
-		offsetof(struct zstd_compression_parameters, chain_log));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, hashLog) ==
-		offsetof(struct zstd_compression_parameters, hash_log));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, searchLog) ==
-		offsetof(struct zstd_compression_parameters, search_log));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, minMatch) ==
-		offsetof(struct zstd_compression_parameters, search_length));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, targetLength) ==
-		offsetof(struct zstd_compression_parameters, target_length));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, strategy) ==
-		offsetof(struct zstd_compression_parameters, strategy));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, contentSizeFlag) ==
-		offsetof(struct zstd_frame_parameters, content_size_flag));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, checksumFlag) ==
-		offsetof(struct zstd_frame_parameters, checksum_flag));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, noDictIDFlag) ==
-		offsetof(struct zstd_frame_parameters, no_dict_id_flag));
-	/* Check that the strategies are the same. This can change. */
-	ZSTD_STATIC_ASSERT((int)ZSTD_fast == (int)zstd_fast);
-	ZSTD_STATIC_ASSERT((int)ZSTD_dfast == (int)zstd_dfast);
-	ZSTD_STATIC_ASSERT((int)ZSTD_greedy == (int)zstd_greedy);
-	ZSTD_STATIC_ASSERT((int)ZSTD_lazy == (int)zstd_lazy);
-	ZSTD_STATIC_ASSERT((int)ZSTD_lazy2 == (int)zstd_lazy2);
-	ZSTD_STATIC_ASSERT((int)ZSTD_btlazy2 == (int)zstd_btlazy2);
-	ZSTD_STATIC_ASSERT((int)ZSTD_btopt == (int)zstd_btopt);
-	ZSTD_STATIC_ASSERT((int)ZSTD_btultra == (int)zstd_btultra);
-	ZSTD_STATIC_ASSERT((int)ZSTD_btultra2 == (int)zstd_btultra2);
-	/* Check input buffer */
-	ZSTD_STATIC_ASSERT(sizeof(ZSTD_inBuffer) == sizeof(struct zstd_in_buffer));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, src) ==
-		offsetof(struct zstd_in_buffer, src));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, size) ==
-		offsetof(struct zstd_in_buffer, size));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, pos) ==
-		offsetof(struct zstd_in_buffer, pos));
-	/* Check output buffer */
-	ZSTD_STATIC_ASSERT(sizeof(ZSTD_outBuffer) ==
-		sizeof(struct zstd_out_buffer));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, dst) ==
-		offsetof(struct zstd_out_buffer, dst));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, size) ==
-		offsetof(struct zstd_out_buffer, size));
-	ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, pos) ==
-		offsetof(struct zstd_out_buffer, pos));
+#define ZSTD_FORWARD_IF_ERR(ret)            \
+	do {                                \
+		size_t const __ret = (ret); \
+		if (ZSTD_isError(__ret))    \
+			return __ret;       \
+	} while (0)
+
+static size_t zstd_cctx_init(zstd_cctx *cctx, const zstd_parameters *parameters,
+	unsigned long long pledged_src_size)
+{
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_reset(
+		cctx, ZSTD_reset_session_and_parameters));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setPledgedSrcSize(
+		cctx, pledged_src_size));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_windowLog, parameters->cParams.windowLog));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_hashLog, parameters->cParams.hashLog));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_chainLog, parameters->cParams.chainLog));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_searchLog, parameters->cParams.searchLog));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_minMatch, parameters->cParams.minMatch));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_targetLength, parameters->cParams.targetLength));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_strategy, parameters->cParams.strategy));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_contentSizeFlag, parameters->fParams.contentSizeFlag));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_checksumFlag, parameters->fParams.checksumFlag));
+	ZSTD_FORWARD_IF_ERR(ZSTD_CCtx_setParameter(
+		cctx, ZSTD_c_dictIDFlag, !parameters->fParams.noDictIDFlag));
+	return 0;
 }

+int zstd_min_clevel(void)
+{
+	return ZSTD_minCLevel();
+}
+EXPORT_SYMBOL(zstd_min_clevel);
+
+int zstd_max_clevel(void)
+{
+	return ZSTD_maxCLevel();
+}
+EXPORT_SYMBOL(zstd_max_clevel);
+
 size_t zstd_compress_bound(size_t src_size)
 {
 	return ZSTD_compressBound(src_size);
 }
 EXPORT_SYMBOL(zstd_compress_bound);

-struct zstd_parameters zstd_get_params(int level,
+zstd_parameters zstd_get_params(int level,
 	unsigned long long estimated_src_size)
 {
-	const ZSTD_parameters params = ZSTD_getParams(level, estimated_src_size, 0);
-	struct zstd_parameters out;
-
-	/* no-op */
-	zstd_check_structs();
-	ZSTD_memcpy(&out, &params, sizeof(out));
-	return out;
+	return ZSTD_getParams(level, estimated_src_size, 0);
 }
 EXPORT_SYMBOL(zstd_get_params);

-size_t zstd_cctx_workspace_bound(
-	const struct zstd_compression_parameters *cparams)
+size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
 {
-	ZSTD_compressionParameters p;
-
-	ZSTD_memcpy(&p, cparams, sizeof(p));
-	return ZSTD_estimateCCtxSize_usingCParams(p);
+	return ZSTD_estimateCCtxSize_usingCParams(*cparams);
 }
 EXPORT_SYMBOL(zstd_cctx_workspace_bound);

@@ -109,31 +94,23 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 EXPORT_SYMBOL(zstd_init_cctx);

 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
-	const void *src, size_t src_size, const struct zstd_parameters *parameters)
+	const void *src, size_t src_size, const zstd_parameters *parameters)
 {
-	ZSTD_parameters p;
-
-	ZSTD_memcpy(&p, parameters, sizeof(p));
-	return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, p);
+	ZSTD_FORWARD_IF_ERR(zstd_cctx_init(cctx, parameters, src_size));
+	return ZSTD_compress2(cctx, dst, dst_capacity, src, src_size);
 }
 EXPORT_SYMBOL(zstd_compress_cctx);

-size_t zstd_cstream_workspace_bound(
-	const struct zstd_compression_parameters *cparams)
+size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
 {
-	ZSTD_compressionParameters p;
-
-	ZSTD_memcpy(&p, cparams, sizeof(p));
-	return ZSTD_estimateCStreamSize_usingCParams(p);
+	return ZSTD_estimateCStreamSize_usingCParams(*cparams);
 }
 EXPORT_SYMBOL(zstd_cstream_workspace_bound);

-zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
+zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
 	unsigned long long pledged_src_size, void *workspace, size_t workspace_size)
 {
-	ZSTD_parameters p;
 	zstd_cstream *cstream;
-	size_t ret;

 	if (workspace == NULL)
 		return NULL;
@@ -146,9 +123,7 @@ zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
 	if (pledged_src_size == 0)
 		pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;

-	ZSTD_memcpy(&p, parameters, sizeof(p));
-	ret = ZSTD_initCStream_advanced(cstream, NULL, 0, p, pledged_src_size);
-	if (ZSTD_isError(ret))
+	if (ZSTD_isError(zstd_cctx_init(cstream, parameters, pledged_src_size)))
 		return NULL;

 	return cstream;
@@ -162,43 +137,22 @@ size_t zstd_reset_cstream(zstd_cstream *cstream,
 }
 EXPORT_SYMBOL(zstd_reset_cstream);

-size_t zstd_compress_stream(zstd_cstream *cstream,
-	struct zstd_out_buffer *output, struct zstd_in_buffer *input)
+size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
+	zstd_in_buffer *input)
 {
-	ZSTD_outBuffer o;
-	ZSTD_inBuffer i;
-	size_t ret;
-
-	ZSTD_memcpy(&o, output, sizeof(o));
-	ZSTD_memcpy(&i, input, sizeof(i));
-	ret = ZSTD_compressStream(cstream, &o, &i);
-	ZSTD_memcpy(output, &o, sizeof(o));
-	ZSTD_memcpy(input, &i, sizeof(i));
-	return ret;
+	return ZSTD_compressStream(cstream, output, input);
 }
 EXPORT_SYMBOL(zstd_compress_stream);

-size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output)
+size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output)
 {
-	ZSTD_outBuffer o;
-	size_t ret;
-
-	ZSTD_memcpy(&o, output, sizeof(o));
-	ret = ZSTD_flushStream(cstream, &o);
-	ZSTD_memcpy(output, &o, sizeof(o));
-	return ret;
+	return ZSTD_flushStream(cstream, output);
 }
 EXPORT_SYMBOL(zstd_flush_stream);

-size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output)
+size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
 {
-	ZSTD_outBuffer o;
-	size_t ret;
-
-	ZSTD_memcpy(&o, output, sizeof(o));
-	ret = ZSTD_endStream(cstream, &o);
-	ZSTD_memcpy(output, &o, sizeof(o));
-	return ret;
+	return ZSTD_endStream(cstream, output);
 }
 EXPORT_SYMBOL(zstd_end_stream);

@@ -1,13 +1,20 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */

 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/zstd.h>

-#include "zstd.h"
 #include "common/zstd_deps.h"
-#include "common/zstd_errors.h"

 /* Common symbols. zstd_compress must depend on zstd_decompress. */

@@ -17,7 +24,7 @@ unsigned int zstd_is_error(size_t code)
 }
 EXPORT_SYMBOL(zstd_is_error);

-int zstd_get_error_code(size_t code)
+zstd_error_code zstd_get_error_code(size_t code)
 {
 	return ZSTD_getErrorCode(code);
 }
@@ -74,19 +81,10 @@ size_t zstd_reset_dstream(zstd_dstream *dstream)
 }
 EXPORT_SYMBOL(zstd_reset_dstream);

-size_t zstd_decompress_stream(zstd_dstream *dstream,
-	struct zstd_out_buffer *output, struct zstd_in_buffer *input)
+size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
+	zstd_in_buffer *input)
 {
-	ZSTD_outBuffer o;
-	ZSTD_inBuffer i;
-	size_t ret;
-
-	ZSTD_memcpy(&o, output, sizeof(o));
-	ZSTD_memcpy(&i, input, sizeof(i));
-	ret = ZSTD_decompressStream(dstream, &o, &i);
-	ZSTD_memcpy(output, &o, sizeof(o));
-	ZSTD_memcpy(input, &i, sizeof(i));
-	return ret;
+	return ZSTD_decompressStream(dstream, output, input);
 }
 EXPORT_SYMBOL(zstd_decompress_stream);

@@ -96,27 +94,12 @@ size_t zstd_find_frame_compressed_size(const void *src, size_t src_size)
 }
 EXPORT_SYMBOL(zstd_find_frame_compressed_size);

-size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src,
+size_t zstd_get_frame_header(zstd_frame_header *header, const void *src,
 	size_t src_size)
 {
-	ZSTD_frameHeader h;
-	const size_t ret = ZSTD_getFrameHeader(&h, src, src_size);
-
-	if (ret != 0)
-		return ret;
-
-	if (h.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
-		params->frame_content_size = h.frameContentSize;
-	else
-		params->frame_content_size = 0;
-
-	params->window_size = h.windowSize;
-	params->dict_id = h.dictID;
-	params->checksum_flag = h.checksumFlag;
-
-	return ret;
+	return ZSTD_getFrameHeader(header, src, src_size);
 }
-EXPORT_SYMBOL(zstd_get_frame_params);
+EXPORT_SYMBOL(zstd_get_frame_header);

 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Decompressor");
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -71,7 +72,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 #endif /* ZSTD_DEPS_MATH64 */
 #endif /* ZSTD_DEPS_NEED_MATH64 */

-/* 
+/*
 * This is only requested when DEBUGLEVEL >= 1, meaning
 * it is disabled in production.
 * Need:
@@ -88,7 +89,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 #endif /* ZSTD_DEPS_ASSERT */
 #endif /* ZSTD_DEPS_NEED_ASSERT */

-/* 
+/*
 * This is only requested when DEBUGLEVEL >= 2, meaning
 * it is disabled in production.
 * Need:
@@ -105,7 +106,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 #endif /* ZSTD_DEPS_IO */
 #endif /* ZSTD_DEPS_NEED_IO */

-/* 
+/*
 * Only requested when MSAN is enabled.
 * Need:
 * intptr_t
@@ -30,6 +30,9 @@ CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
 CPPFLAGS ?=
 LDFLAGS  ?=

+# PZstd uses legacy APIs
+CFLAGS   += -Wno-deprecated-declarations
+
 # Include flags
 PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 GTEST_INC  = -isystem googletest/googletest/include
@@ -54,19 +57,6 @@ LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
 CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
 CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@

-# Get a list of all zstd files so we rebuild the static library when we need to
-ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
-                    $(wildcard $(ZSTDDIR)/common/*.h)
-ZSTDCOMP_FILES   := $(wildcard $(ZSTDDIR)/compress/*.c) \
-                    $(wildcard $(ZSTDDIR)/compress/*.h)
-ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
-                    $(wildcard $(ZSTDDIR)/decompress/*.h)
-ZSTDPROG_FILES   := $(wildcard $(PROGDIR)/*.c) \
-                    $(wildcard $(PROGDIR)/*.h)
-ZSTD_FILES       := $(wildcard $(ZSTDDIR)/*.h) \
-                    $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
-                    $(ZSTDPROG_FILES)
-
 # List all the pzstd source files so we can determine their dependencies
 PZSTD_SRCS  := $(wildcard *.cpp)
 PZSTD_TESTS := $(wildcard test/*.cpp)
@@ -186,7 +176,8 @@ roundtrip: test/RoundTripTest$(EXT)

 # Use the static library that zstd builds for simplicity and
 # so we get the compiler options correct
-$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
+.PHONY: $(ZSTDDIR)/libzstd.a
+$(ZSTDDIR)/libzstd.a:
 	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a

 # Rules to build the tests
@@ -87,7 +87,7 @@ void usage() {
  std::fprintf(stderr, "  -V, --version          : display version number and exit\n");
  std::fprintf(stderr, "  -v, --verbose          : verbose mode; specify multiple times to increase log level (default:2)\n");
  std::fprintf(stderr, "  -q, --quiet            : suppress warnings; specify twice to suppress errors too\n");
-  std::fprintf(stderr, "  -c, --stdout           : force write to standard output, even if it is the console\n");
+  std::fprintf(stderr, "  -c, --stdout           : write to standard output (even if it is the console)\n");
 #ifdef UTIL_HAS_CREATEFILELIST
  std::fprintf(stderr, "  -r                     : operate recursively on directories\n");
 #endif
@@ -9,6 +9,9 @@
 #pragma once

 #define ZSTD_STATIC_LINKING_ONLY
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
+                                         * and uses deprecated functions
+                                         */
 #include "zstd.h"
 #undef ZSTD_STATIC_LINKING_ONLY

@@ -17,6 +17,9 @@
 #include "utils/ThreadPool.h"
 #include "utils/WorkQueue.h"
 #define ZSTD_STATIC_LINKING_ONLY
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
+                                         * and uses deprecated functions
+                                         */
 #include "zstd.h"
 #undef ZSTD_STATIC_LINKING_ONLY

@@ -45,7 +45,7 @@ class Buffer {
      : buffer_(buffer), range_(data) {}

  Buffer(Buffer&&) = default;
-  Buffer& operator=(Buffer&&) & = default;
+  Buffer& operator=(Buffer&&) = default;

  /**
   * Splits the data into two pieces: [begin, begin + n), [begin + n, end).
@@ -6,7 +6,7 @@
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 */
- 
+
 /**
 * A subset of `folly/Range.h`.
 * All code copied verbatim modulo formatting
@@ -83,8 +83,8 @@ class Range {
  Range(const Range&) = default;
  Range(Range&&) = default;

-  Range& operator=(const Range&) & = default;
-  Range& operator=(Range&&) & = default;
+  Range& operator=(const Range&) = default;
+  Range& operator=(Range&&) = default;

  constexpr size_type size() const {
    return e_ - b_;
@@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)

-CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common

 CFLAGS ?= -O3
 CFLAGS += -g
@@ -21,7 +21,6 @@
 #  define SLEEP(x) usleep(x * 1000)
 #endif

-#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"

 #include "pool.h"      // use zstd thread pool for demo
@@ -99,6 +99,9 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end

    while (startOffset < endOffset) {
        size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
+        if (!result) {
+            break;
+        }

        if (ZSTD_isError(result)) {
            fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
@@ -104,6 +104,9 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end

    while (startOffset < endOffset) {
        size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
+        if (!result) {
+            break;
+        }

        if (ZSTD_isError(result)) {
            fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
@@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)

-CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common

 CFLAGS ?= -O3
 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \
@@ -12,7 +12,6 @@
 #include <assert.h>

 #define XXH_STATIC_LINKING_ONLY
-#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"

 #define ZSTD_STATIC_LINKING_ONLY
@@ -83,7 +82,7 @@ static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)

 ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
 {
-    ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog));
+    ZSTD_frameLog* const fl = (ZSTD_frameLog*)malloc(sizeof(ZSTD_frameLog));
    if (fl == NULL) return NULL;

    if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
@@ -108,7 +107,7 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl)

 ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void)
 {
-    ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream));
+    ZSTD_seekable_CStream* const zcs = (ZSTD_seekable_CStream*)malloc(sizeof(ZSTD_seekable_CStream));
    if (zcs == NULL) return NULL;

    memset(zcs, 0, sizeof(*zcs));
@@ -177,7 +176,7 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
    if (fl->size == fl->capacity) {
        /* exponential size increase for constant amortized runtime */
        size_t const newCapacity = fl->capacity * 2;
-        framelogEntry_t* const newEntries = realloc(fl->entries,
+        framelogEntry_t* const newEntries = (framelogEntry_t*)realloc(fl->entries,
                sizeof(framelogEntry_t) * newCapacity);

        if (newEntries == NULL) return ERROR(memory_allocation);
@@ -23,13 +23,64 @@
 #  endif
 #endif

+/* ************************************************************
+*  Detect POSIX version
+*  PLATFORM_POSIX_VERSION = 0 for non-Unix e.g. Windows
+*  PLATFORM_POSIX_VERSION = 1 for Unix-like but non-POSIX
+*  PLATFORM_POSIX_VERSION > 1 is equal to found _POSIX_VERSION
+*  Value of PLATFORM_POSIX_VERSION can be forced on command line
+***************************************************************/
+#ifndef PLATFORM_POSIX_VERSION
+
+#  if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
+     || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* BSD distros */
+     /* exception rule : force posix version to 200112L,
+      * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */
+#    define PLATFORM_POSIX_VERSION 200112L
+
+/* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html).
+ * note : there is no simple way to know in advance if <unistd.h> is present or not on target system,
+ * Posix specification mandates its presence and its content, but target system must respect this spec.
+ * It's necessary to _not_ #include <unistd.h> whenever target OS is not unix-like
+ * otherwise it will block preprocessing stage.
+ * The following list of build macros tries to "guess" if target OS is likely unix-like, and therefore can #include <unistd.h>
+ */
+#  elif !defined(_WIN32) \
+     && ( defined(__unix__) || defined(__unix) \
+       || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__) )
+
+#    if defined(__linux__) || defined(__linux) || defined(__CYGWIN__)
+#      ifndef _POSIX_C_SOURCE
+#        define _POSIX_C_SOURCE 200809L  /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */
+#      endif
+#    endif
+#    include <unistd.h>  /* declares _POSIX_VERSION */
+#    if defined(_POSIX_VERSION)  /* POSIX compliant */
+#      define PLATFORM_POSIX_VERSION _POSIX_VERSION
+#    else
+#      define PLATFORM_POSIX_VERSION 1
+#    endif
+
+#    ifdef __UCLIBC__
+#     ifndef __USE_MISC
+#      define __USE_MISC /* enable st_mtim on uclibc */
+#     endif
+#    endif
+
+#  else  /* non-unix target platform (like Windows) */
+#    define PLATFORM_POSIX_VERSION 0
+#  endif
+
+#endif   /* PLATFORM_POSIX_VERSION */
+
+
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
 ***************************************************************/
 #if defined(_MSC_VER) && _MSC_VER >= 1400
 #   define LONG_SEEK _fseeki64
 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
-#  define LONG_SEEK fseeko
+#   define LONG_SEEK fseeko
 #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__)
 #   define LONG_SEEK fseeko64
 #elif defined(_WIN32) && !defined(__DJGPP__)
@@ -60,7 +111,6 @@
 #include <assert.h>

 #define XXH_STATIC_LINKING_ONLY
-#define XXH_NAMESPACE ZSTD_
 #include "xxhash.h"

 #define ZSTD_STATIC_LINKING_ONLY
@@ -176,7 +226,7 @@ struct ZSTD_seekable_s {

 ZSTD_seekable* ZSTD_seekable_create(void)
 {
-    ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable));
+    ZSTD_seekable* const zs = (ZSTD_seekable*)malloc(sizeof(ZSTD_seekable));
    if (zs == NULL) return NULL;

    /* also initializes stage to zsds_init */
@@ -202,7 +252,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)

 ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
 {
-    ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable));
+    ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
    if (st==NULL) return NULL;

    st->checksumFlag = zs->seekTable.checksumFlag;
@@ -433,6 +483,11 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr

 size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset)
 {
+    unsigned long long const eos = zs->seekTable.entries[zs->seekTable.tableLen].dOffset;
+    if (offset + len > eos) {
+        len = eos - offset;
+    }
+
    U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset);
    U32 noOutputProgressCount = 0;
    size_t srcBytesRead = 0;
@@ -449,7 +504,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
            zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
            XXH64_reset(&zs->xxhState, 0);
            ZSTD_DCtx_reset(zs->dstream, ZSTD_reset_session_only);
-            if (srcBytesRead > zs->buffWrapper.size) {
+            if (zs->buffWrapper.size && srcBytesRead > zs->buffWrapper.size) {
                return ERROR(seekableIO);
            }
        }
@@ -502,6 +557,8 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
                if (zs->decompressedOffset < offset + len) {
                    /* go back to the start and force a reset of the stream */
                    targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, zs->decompressedOffset);
+                    /* in this case it will fail later with corruption_detected, since last block does not have checksum */
+                    assert(targetFrame != zs->seekTable.tableLen);
                }
                break;
            }
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# Where to find the sources
-ZSTD_SRC_ROOT="../../lib"
-
-# Amalgamate the sources
-echo "Amalgamating files... this can take a while"
-./combine.sh -r "$ZSTD_SRC_ROOT" -o zstddeclib.c zstddeclib-in.c
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Combine script: FAILED"
-  exit 1
-fi
-echo "Combine script: PASSED"
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# Where to find the sources
-ZSTD_SRC_ROOT="../../lib"
-
-# Amalgamate the sources
-echo "Amalgamating files... this can take a while"
-./combine.sh -r "$ZSTD_SRC_ROOT" -o zstd.c zstd-in.c
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Combine script: FAILED"
-  exit 1
-fi
-echo "Combine script: PASSED"
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
+# Copyright (c) Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -2145,7 +2145,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable,

    // "All remaining symbols are sorted in their natural order. Starting from
    // symbol 0 and table position 0, each symbol gets attributed as many cells
-    // as its probability. Cell allocation is spreaded, not linear."
+    // as its probability. Cell allocation is spread, not linear."
    // Place the rest in the table
    const u16 step = (size >> 1) + (size >> 3) + 3;
    const u16 mask = size - 1;
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2021, Facebook, Inc.
+ * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
@@ -1124,7 +1124,7 @@ These symbols define a full state reset, reading `Accuracy_Log` bits.
 Then, all remaining symbols, sorted in natural order, are allocated cells.
 Starting from symbol `0` (if it exists), and table position `0`,
 each symbol gets allocated as many cells as its probability.
-Cell allocation is spreaded, not linear :
+Cell allocation is spread, not linear :
 each successor position follows this rule :

 ```
@@ -1669,7 +1669,7 @@ or at least provide a meaningful error code explaining for which reason it canno

 Version changes
 ---------------
- 0.3.7 : clarifications for Repeat_Offsets
+- 0.3.7 : clarifications for Repeat_Offsets, matching RFC8878
 - 0.3.6 : clarifications for Dictionary_ID
 - 0.3.5 : clarifications for Block_Maximum_Size
 - 0.3.4 : clarifications for FSE decoding table
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.4.9 Manual</title>
+<title>zstd 1.5.2 Manual</title>
 </head>
 <body>
-<h1>zstd 1.4.9 Manual</h1>
+<h1>zstd 1.5.2 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -12,15 +12,15 @@
 <li><a href="#Chapter2">Version</a></li>
 <li><a href="#Chapter3">Simple API</a></li>
 <li><a href="#Chapter4">Explicit context</a></li>
-<li><a href="#Chapter5">Advanced compression API</a></li>
-<li><a href="#Chapter6">Advanced decompression API</a></li>
+<li><a href="#Chapter5">Advanced compression API (Requires v1.4.0+)</a></li>
+<li><a href="#Chapter6">Advanced decompression API (Requires v1.4.0+)</a></li>
 <li><a href="#Chapter7">Streaming</a></li>
 <li><a href="#Chapter8">Streaming compression - HowTo</a></li>
 <li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
 <li><a href="#Chapter10">Simple dictionary API</a></li>
 <li><a href="#Chapter11">Bulk processing dictionary API</a></li>
 <li><a href="#Chapter12">Dictionary helper functions</a></li>
-<li><a href="#Chapter13">Advanced dictionary and prefix API</a></li>
+<li><a href="#Chapter13">Advanced dictionary and prefix API (Requires v1.4.0+)</a></li>
 <li><a href="#Chapter14">experimental API (static linking only)</a></li>
 <li><a href="#Chapter15">Frame size functions</a></li>
 <li><a href="#Chapter16">Memory management</a></li>
@@ -141,8 +141,9 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed */<b>
+int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
 int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
+int         ZSTD_defaultCLevel(void);           </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
 </pre></b><BR>
 <a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>

@@ -157,7 +158,7 @@ int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression lev
 
 </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
                         void* dst, size_t dstCapacity,
@@ -179,7 +180,7 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
  Use one context per thread for parallel execution. 
 </pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
-size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
                           void* dst, size_t dstCapacity,
@@ -190,7 +191,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 </p></pre><BR>

-<a name="Chapter5"></a><h2>Advanced compression API</h2><pre></pre>
+<a name="Chapter5"></a><h2>Advanced compression API (Requires v1.4.0+)</h2><pre></pre>

 <pre><b>typedef enum { ZSTD_fast=1,
               ZSTD_dfast=2,
@@ -270,7 +271,6 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                              * The higher the value of selected strategy, the more complex it is,
                              * resulting in stronger and slower compression.
                              * Special: value 0 means "use default strategy". */
-
    </b>/* LDM mode parameters */<b>
    ZSTD_c_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
                                     * This parameter is designed to improve compression ratio
@@ -327,7 +327,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
    ZSTD_c_jobSize=401,      </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b>
                              * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                              * 0 means default, which is dynamically determined based on compression parameters.
-                              * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
+                              * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
                              * The minimum size is automatically and transparently enforced. */
    ZSTD_c_overlapLog=402,   </b>/* Control the overlap size, as a fraction of window size.<b>
                              * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
@@ -357,6 +357,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     * ZSTD_c_stableOutBuffer
     * ZSTD_c_blockDelimiters
     * ZSTD_c_validateSequences
+     * ZSTD_c_useBlockSplitter
+     * ZSTD_c_useRowMatchFinder
     * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
     * note : never ever use experimentalParam? names directly;
     *        also, the enums values themselves are unstable and can still change.
@@ -372,7 +374,10 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_c_experimentalParam9=1006,
     ZSTD_c_experimentalParam10=1007,
     ZSTD_c_experimentalParam11=1008,
-     ZSTD_c_experimentalParam12=1009
+     ZSTD_c_experimentalParam12=1009,
+     ZSTD_c_experimentalParam13=1010,
+     ZSTD_c_experimentalParam14=1011,
+     ZSTD_c_experimentalParam15=1012
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -456,7 +461,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 </p></pre><BR>

-<a name="Chapter6"></a><h2>Advanced decompression API</h2><pre></pre>
+<a name="Chapter6"></a><h2>Advanced decompression API (Requires v1.4.0+)</h2><pre></pre>

 <pre><b>typedef enum {

@@ -587,7 +592,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 <pre><b>typedef ZSTD_CCtx ZSTD_CStream;  </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b>
 </b></pre><BR>
 <h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <h3>Streaming compression functions</h3><pre></pre><b><pre>typedef enum {
    ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
@@ -681,7 +686,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 <pre><b>typedef ZSTD_DCtx ZSTD_DStream;  </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b>
 </b></pre><BR>
 <h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <h3>Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
@@ -697,7 +702,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
                               int compressionLevel);
 </b><p>  Compression at an explicit compression level using a Dictionary.
  A dictionary can be any arbitrary data segment (also called a prefix),
-  or a buffer with specified information (see dictBuilder/zdict.h).
+  or a buffer with specified information (see zdict.h).
  Note : This function loads the dictionary, resulting in significant startup delay.
         It's intended for a dictionary used only once.
  Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. 
@@ -732,7 +737,8 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 </p></pre><BR>

 <pre><b>size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-</b><p>  Function frees memory allocated by ZSTD_createCDict(). 
+</b><p>  Function frees memory allocated by ZSTD_createCDict().
+  If a NULL pointer is passed, no operation is performed. 
 </p></pre><BR>

 <pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -751,7 +757,8 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 </p></pre><BR>

 <pre><b>size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
-</b><p>  Function frees memory allocated with ZSTD_createDDict() 
+</b><p>  Function frees memory allocated with ZSTD_createDDict()
+  If a NULL pointer is passed, no operation is performed. 
 </p></pre><BR>

 <pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
@@ -770,6 +777,12 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
  It can still be loaded, but as a content-only dictionary. 
 </p></pre><BR>

+<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
+</b><p>  Provides the dictID of the dictionary loaded into `cdict`.
+  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
+</p></pre><BR>
+
 <pre><b>unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
 </b><p>  Provides the dictID of the dictionary loaded into `ddict`.
  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
@@ -788,9 +801,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 </p></pre><BR>

-<a name="Chapter13"></a><h2>Advanced dictionary and prefix API</h2><pre>
+<a name="Chapter13"></a><h2>Advanced dictionary and prefix API (Requires v1.4.0+)</h2><pre>
 This API allows dictionaries to be used with ZSTD_compress2(),
- ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
+ ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
 only reset with the context is reset with ZSTD_reset_parameters or
 ZSTD_reset_session_and_parameters. Prefixes are single-use.
 <BR></pre>
@@ -1058,9 +1071,19 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  ZSTD_lcm_uncompressed = 2   </b>/**< Always emit uncompressed literals. */<b>
 } ZSTD_literalCompressionMode_e;
 </b></pre><BR>
+<pre><b>typedef enum {
+  </b>/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final<b>
+   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+   * or ZSTD_ps_disable allow for a force enable/disable the feature.
+   */
+  ZSTD_ps_auto = 0,         </b>/* Let the library automatically determine whether the feature shall be enabled */<b>
+  ZSTD_ps_enable = 1,       </b>/* Force-enable the feature */<b>
+  ZSTD_ps_disable = 2       </b>/* Do not use the feature */<b>
+} ZSTD_paramSwitch_e;
+</b></pre><BR>
 <a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>

-<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+<pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  `srcSize` must be the _exact_ size of this series
       (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1083,7 +1106,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            however it does mean that all frame data must be present and valid. 
 </p></pre><BR>

-<pre><b>unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+<pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
  `srcSize` must be the _exact_ size of this series
       (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1098,7 +1121,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 </b><p>  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
 @return : size of the Frame Header,
           or an error code (if srcSize is too small) 
@@ -1125,7 +1148,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
 </b><p> Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
 by merging them into into the literals of the next sequence.

@@ -1138,10 +1161,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
                      const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
                      const void* src, size_t srcSize);
-</b><p> Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
+</b><p> Compress an array of ZSTD_Sequence, associted with @src buffer, into dst.
+ @src contains the entire input (not just the literals).
+ If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
 If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
 The entire source is compressed into a single frame.

@@ -1170,7 +1195,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
                                const void* src, size_t srcSize, unsigned magicVariant);
 </b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.

@@ -1186,12 +1211,31 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 </p></pre><BR>

+<pre><b>size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
+                                const void* src, size_t srcSize);
+</b><p> Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+
+ The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
+ in the magicVariant.
+
+ Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+
+ @return : number of bytes written or a ZSTD error.
+ 
+</p></pre><BR>
+
+<pre><b>unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+</b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
+ 
+</p></pre><BR>
+
 <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>

-<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
-size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-size_t ZSTD_estimateDCtxSize(void);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
 </b><p>  These functions make it possible to estimate memory usage
  of a future {D,C}Ctx, before its creation.

@@ -1216,11 +1260,11 @@ size_t ZSTD_estimateDCtxSize(void);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
-size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-size_t ZSTD_estimateDStreamSize(size_t windowSize);
-size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 </b><p>  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
  It will also consider src size to be arbitrarily "large", which is worst case.
  If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
@@ -1235,17 +1279,17 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
         In this case, get total size by adding ZSTD_estimate?DictSize 
 </p></pre><BR>

-<pre><b>size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
-size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
-size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
 </b><p>  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
  ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
  Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
 
 </p></pre><BR>

-<pre><b>ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
-ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
+<pre><b>ZSTDLIB_STATIC_API ZSTD_CCtx*    ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
 </b><p>  Initialize an object using a pre-allocated fixed-size buffer.
  workspace: The memory area to emplace the object into.
             Provided pointer *must be 8-bytes aligned*.
@@ -1268,7 +1312,7 @@ ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);
 
 </p></pre><BR>

-<pre><b>ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticDCtx() */<b>
+<pre><b>ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticDCtx() */<b>
 </b></pre><BR>
 <pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
@@ -1284,9 +1328,24 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this con
 
 </p></pre><BR>

+<pre><b>typedef struct POOL_ctx_s ZSTD_threadPool;
+ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
+ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  </b>/* accept NULL pointer */<b>
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
+</b><p>  These prototypes make it possible to share a thread pool among multiple compression contexts.
+  This can limit resources for applications with multiple threads where each one uses
+  a threaded compression mode (via ZSTD_c_nbWorkers parameter).
+  ZSTD_createThreadPool creates a new thread pool with a given number of threads.
+  Note that the lifetime of such pool must exist while being used.
+  ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
+  to use an internal thread pool).
+  ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
+ 
+</p></pre><BR>
+
 <a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre>

-<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 </b><p>  Create a digested dictionary for compression
  Dictionary content is just referenced, not duplicated.
  As a consequence, `dictBuffer` **must** outlive CDict,
@@ -1294,28 +1353,22 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this con
  note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef 
 </p></pre><BR>

-<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
-</b><p>  Provides the dictID of the dictionary loaded into `cdict`.
-  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
-  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
-</p></pre><BR>
-
-<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 </b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
 `estimatedSrcSize` value is optional, select 0 if not known 
 </p></pre><BR>

-<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 </b><p>  same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
  All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 
 </p></pre><BR>

-<pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 </b><p>  Ensure param values remain within authorized range.
 @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
 </p></pre><BR>

-<pre><b>ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 </b><p>  optimize params for a given `srcSize` and `dictSize`.
 `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
 `dictSize` must be `0` when there is no dictionary.
@@ -1323,51 +1376,53 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this con
  This function never fails (wide contract) 
 </p></pre><BR>

-<pre><b>size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2")
+size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
                              void* dst, size_t dstCapacity,
                        const void* src, size_t srcSize,
                        const void* dict,size_t dictSize,
                              ZSTD_parameters params);
 </b><p>  Note : this function is now DEPRECATED.
         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
-  This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x 
+  This prototype will generate compilation warnings. 
 </p></pre><BR>

-<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                  void* dst, size_t dstCapacity,
                            const void* src, size_t srcSize,
                            const ZSTD_CDict* cdict,
                                  ZSTD_frameParameters fParams);
-</b><p>  Note : this function is now REDUNDANT.
+</b><p>  Note : this function is now DEPRECATED.
         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
-  This prototype will be marked as deprecated and generate compilation warning in some future version 
+  This prototype will generate compilation warnings. 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 </b><p>  Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
  It saves some memory, but also requires that `dict` outlives its usage within `cctx` 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
  how to load the dictionary (by copy ? by reference ?)
  and how to interpret it (automatic ? force raw mode ? full mode only ?) 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_CCtx_refPrefix(), but gives finer control over
  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
 </b><p>  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  and store it into int* value.
 @return : 0, or an error code (which can be tested with ZSTD_isError()).
 
 </p></pre><BR>

-<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
-size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);  </b>/* accept NULL pointer */<b>
 </b><p>  Quick howto :
  - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
  - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
@@ -1379,31 +1434,31 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
                                    These parameters will be applied to
                                    all subsequent frames.
  - ZSTD_compressStream2() : Do compression using the CCtx.
-  - ZSTD_freeCCtxParams() : Free the memory.
+  - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.

  This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
  for static allocation of CCtx for single-threaded compression.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
 </b><p>  Reset params to default values.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
 </b><p>  Initializes the compression parameters of cctxParams according to
  compression level. All other parameters are reset to their default values.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
 </b><p>  Initializes the compression and frame parameters of cctxParams according to
  params. All other parameters are reset to their default values.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 </b><p>  Similar to ZSTD_CCtx_setParameter.
  Set one compression parameter, selected by enum ZSTD_cParameter.
  Parameters must be applied to a ZSTD_CCtx using
@@ -1413,14 +1468,14 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
 </b><p> Similar to ZSTD_CCtx_getParameter.
 Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
 @result : 0, or an error code (which can be tested with ZSTD_isError()).
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
        ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
 </b><p>  Apply a set of ZSTD_CCtx_params to the compression context.
  This can be done even after compression is started,
@@ -1430,7 +1485,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_compressStream2_simpleArgs (
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
                ZSTD_CCtx* cctx,
                void* dst, size_t dstCapacity, size_t* dstPos,
          const void* src, size_t srcSize, size_t* srcPos,
@@ -1444,40 +1499,40 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);

 <a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre>

-<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
+<pre><b>ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier.
  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
  Note 3 : Skippable Frame Identifiers are considered valid. 
 </p></pre><BR>

-<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 </b><p>  Create a digested dictionary, ready to start decompression operation without startup delay.
  Dictionary content is referenced, and therefore stays in dictBuffer.
  It is important that dictBuffer outlives DDict,
  it must remain read accessible throughout the lifetime of DDict 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 </b><p>  Same as ZSTD_DCtx_loadDictionary(),
  but references `dict` content instead of copying it into `dctx`.
  This saves memory if `dict` remains around.,
  However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_DCtx_loadDictionary(),
  but gives direct control over
  how to load the dictionary (by copy ? by reference ?)
  and how to interpret it (automatic ? force raw mode ? full mode only ?). 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
 </b><p>  Same as ZSTD_DCtx_refPrefix(), but gives finer control over
  how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
 </b><p>  Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
  This protects a decoder context from reserving too much memory for itself (potential attack scenario).
  This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
@@ -1486,21 +1541,23 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
 </b><p>  Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
  and store it into int* value.
 @return : 0, or an error code (which can be tested with ZSTD_isError()).
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
-</b><p>  Instruct the decoder context about what kind of data to decode next.
+<pre><b>ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
+</b><p>  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
+  Instruct the decoder context about what kind of data to decode next.
  This instruction is mandatory to decode data without a fully-formed header,
  such ZSTD_f_zstd1_magicless for example.
 @return : 0, or an error code (which can be tested using ZSTD_isError()). 
 </p></pre><BR>

-<pre><b>size_t ZSTD_decompressStream_simpleArgs (
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
                ZSTD_DCtx* dctx,
                void* dst, size_t dstCapacity, size_t* dstPos,
          const void* src, size_t srcSize, size_t* srcPos);
@@ -1517,11 +1574,11 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 <BR></pre>

 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>size_t
-ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
             int compressionLevel,
             unsigned long long pledgedSrcSize);
-</b><p> This function is deprecated, and equivalent to:
+</b><p> This function is DEPRECATED, and equivalent to:
     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@@ -1530,15 +1587,15 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
 pledgedSrcSize must be correct. If it is not known at init time, use
 ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
 "0" also disables frame content size field. It may be enabled in the future.
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ This prototype will generate compilation warnings.
 
 </p></pre><BR>

-<pre><b>size_t
-ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
         const void* dict, size_t dictSize,
               int compressionLevel);
-</b><p> This function is deprecated, and is equivalent to:
+</b><p> This function is DEPRECATED, and is equivalent to:
     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1547,16 +1604,16 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
 dict == NULL or dictSize < 8, in which case no dict is used.
 Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
 it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ This prototype will generate compilation warnings.
 
 </p></pre><BR>

-<pre><b>size_t
-ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
        const void* dict, size_t dictSize,
              ZSTD_parameters params,
              unsigned long long pledgedSrcSize);
-</b><p> This function is deprecated, and is approximately equivalent to:
+</b><p> This function is DEPRECATED, and is approximately equivalent to:
     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
     // Pseudocode: Set each zstd parameter and leave the rest as-is.
     for ((param, value) : params) {
@@ -1568,22 +1625,23 @@ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
 dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
 pledgedSrcSize must be correct.
 If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ This prototype will generate compilation warnings.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
-</b><p> This function is deprecated, and equivalent to:
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+</b><p> This function is DEPRECATED, and equivalent to:
     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
     ZSTD_CCtx_refCDict(zcs, cdict);

 note : cdict will just be referenced, and must outlive compression session
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ This prototype will generate compilation warnings.
 
 </p></pre><BR>

-<pre><b>size_t
-ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                   const ZSTD_CDict* cdict,
                         ZSTD_frameParameters fParams,
                         unsigned long long pledgedSrcSize);
@@ -1599,14 +1657,18 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
 pledgedSrcSize must be correct. If srcSize is not known at init time, use
 value ZSTD_CONTENTSIZE_UNKNOWN.
- Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ This prototype will generate compilation warnings.
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
-</b><p> This function is deprecated, and is equivalent to:
+<pre><b>ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+</b><p> This function is DEPRECATED, and is equivalent to:
     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+ Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
+       ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
+       explicitly specified.

  start a new frame, using same parameters from previous frame.
  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
@@ -1616,7 +1678,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
 @return : 0, or an error code (which can be tested using ZSTD_isError())
-  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+  This prototype will generate compilation warnings.
 
 </p></pre><BR>

@@ -1629,7 +1691,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
    unsigned nbActiveWorkers;      </b>/* MT only : nb of workers actively compressing at probe time */<b>
 } ZSTD_frameProgression;
 </b></pre><BR>
-<pre><b>size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 </b><p>  Tell how many bytes are ready to be flushed immediately.
  Useful for multithreading scenarios (nbWorkers >= 1).
  Probe the oldest active job, defined as oldest job not yet entirely flushed,
@@ -1645,7 +1707,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 </p></pre><BR>

 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 </b><p>
     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
@@ -1655,7 +1717,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 </b><p>
     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
     ZSTD_DCtx_refDDict(zds, ddict);
@@ -1665,7 +1727,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 
 </p></pre><BR>

-<pre><b>size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 </b><p>
     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);

@@ -1687,8 +1749,7 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  ZSTD_CCtx object can be re-used multiple times within successive compression operations.

  Start by initializing a context.
-  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
-  or ZSTD_compressBegin_advanced(), for finer parameter control.
+  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()

  Then, consume your input using ZSTD_compressContinue().
@@ -1710,13 +1771,13 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
 <BR></pre>

-<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
-size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
+<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
+ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </pre></b><BR>
+<pre><b>size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
+</b></pre><BR>
 <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
  A ZSTD_DCtx object is required to track streaming operations.
  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
@@ -1799,12 +1860,12 @@ typedef struct {
    unsigned checksumFlag;
 } ZSTD_frameHeader;
 </pre></b><BR>
-<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
 </b>/*! ZSTD_getFrameHeader_advanced() :<b>
 *  same as ZSTD_getFrameHeader(),
 *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </b><p>  decode Frame Header, or requires larger `srcSize`.
 @return : 0, `zfhPtr` is correctly filled,
          >0, `srcSize` is too small, value is wanted `srcSize` amount,
@@ -1840,10 +1901,10 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
        Use ZSTD_insertBlock() for such a case.
 </p></pre><BR>

-<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
-size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
+<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
+ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
 </pre></b><BR>
 </html>
 </body>
--- a/Show More
+++ b/Show More