Merge pull request #1375 from facebook/dev

minor add for v1.3.7

.circleci/config.yml

@@ -1,110 +1,111 @@
 version: 2
 
+references:
+  # Install the dependencies required for tests.
+  # Add the step "- *install-dependencies" to the beginning of your job to run
+  # this command.
+  install-dependencies: &install-dependencies
+    run:
+      name: Install dependencies
+      # TODO: We can split these dependencies up by job to reduce installation
+      # time.
+      command: |
+        sudo dpkg --add-architecture i386
+        sudo apt-get -y -qq update
+        sudo apt-get -y install \
+            gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
+            libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+            libc6-dev-ppc64-powerpc-cross
+
 jobs:
   # the first half of the jobs are in this test
   short-tests-0:
     # TODO: Create a small custom docker image with all the dependencies we need
     #       preinstalled to reduce installation time.
     docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
-    # TODO: Re-enable aarch64 build:
-    #       make aarch64build     && make clean
+      - image: circleci/buildpack-deps:bionic
     steps:
       - checkout
+      - *install-dependencies
       - run:
           name: Test
           command: |
-            ./tests/test-license.py
-            cc -v
-            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
-            make c99build; make clean
-            make c11build; make clean
+            cc -v; CFLAGS="-O0 -Werror" make all && make clean
+            make c99build         ; make clean
+            make c11build         ; make clean
+            make aarch64build     ; make clean
             make -j regressiontest; make clean
-            make shortest; make clean
-            make cxxtest; make clean
+            make shortest         ; make clean
+            make cxxtest          ; make clean
   # the second half of the jobs are in this test
   short-tests-1:
     docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
+      - image: circleci/buildpack-deps:bionic
     steps:
       - checkout
+      - *install-dependencies
       - run:
           name: Test
           command: |
             make gnu90build; make clean
             make gnu99build; make clean
-            make ppc64build V=1; make clean
-            make ppcbuild   V=1; make clean
-            make armbuild   V=1; make clean
-            make -C tests test-legacy test-longmatch; make clean
+            make ppc64build; make clean
+            make ppcbuild  ; make clean
+            make armbuild  ; make clean
+            make -C tests test-legacy test-longmatch test-symbols; make clean
             make -C lib libzstd-nomt; make clean
-  # This step should only be run in a cron job
-  regression-test:
+  # This step is only run on release tags.
+  # It publishes the source tarball as artifacts and if the GITHUB_TOKEN
+  # environment variable is set it will publish the source tarball to the
+  # tagged release.
+  publish-github-release:
     docker:
-      - image: fbopensource/zstd-circleci-primary:0.0.1
+      - image: cibuilds/github:0.12.0
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
     steps:
       - checkout
-      # Restore the cached resources.
-      - restore_cache:
-          # We try our best to bust the cache when the data changes by hashing
-          # data.c. If that doesn't work, simply update the version number here
-          # and below. If we fail to bust the cache, the regression testing will
-          # still work, since it has its own stamp, but will need to redownload
-          # everything.
-          keys:
-            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
       - run:
-          name: Regression Test
+          name: Install dependencies
           command: |
-            make -C programs zstd
-            make -C tests/regression test
+            apk add -q gzip coreutils
+      - run:
+          name: Publish
+          command: |
+            export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
+            export ZSTD_VERSION=zstd-$VERSION
+            git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
+                        -o $ZSTD_VERSION.tar
+            gzip -9 $ZSTD_VERSION.tar
+            sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256sum
             mkdir -p $CIRCLE_ARTIFACTS
-            ./tests/regression/test                     \
-                --cache  tests/regression/cache         \
-                --output $CIRCLE_ARTIFACTS/results.csv  \
-                --zstd   programs/zstd
-            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
-            echo "      If this fails, go to the Artifacts pane in CircleCI, "
-            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
-            echo "      are still good, copy it into the repo and commit it."
-            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
-            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
-      # Only save the cache on success (default), since if the failure happened
-      # before we stamp the data cache, we will have a bad cache for this key.
-      - save_cache:
-          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
-          paths:
-            - tests/regression/cache
+            cp $ZSTD_VERSION.tar.gz{,.sha256sum} $CIRCLE_ARTIFACTS
       - store_artifacts:
           path: /tmp/circleci-artifacts
 
-
 workflows:
   version: 2
   commit:
     jobs:
       # Run the tests in parallel
-      - short-tests-0
-      - short-tests-1
-      - regression-test
-
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
+      - short-tests-0:
+          filters:
+            tags:
+              only: /.*/
+      - short-tests-1:
+          filters:
+            tags:
+              only: /.*/
+      # Only run on release tags.
+      - publish-github-release:
+          requires:
+            - short-tests-0
+            - short-tests-1
           filters:
             branches:
-              only:
-                - release
-                - dev
-                - master
-    jobs:
-      # Run daily regression tests
-      - regression-test
-
-
+              ignore: /.*/
+            tags:
+              only: /^v\d+\.\d+\.\d+$/
 
   # Longer tests
     #- make -C tests test-zstd-nolegacy && make clean

.circleci/images/primary/Dockerfile

@@ -1,9 +0,0 @@
-FROM circleci/buildpack-deps:bionic
-
-RUN sudo dpkg --add-architecture i386
-RUN sudo apt-get -y -qq update
-RUN sudo apt-get -y install \
-    gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
-    libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
-    libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
-    libcurl4-openssl-dev

.cirrus.yml

@@ -1,11 +0,0 @@
-task:
-  name: FreeBSD (shortest)
-  freebsd_instance:
-    matrix:
-      image_family: freebsd-12-2
-      # The stable 11.3 image causes "Agent is not responding" so use a snapshot
-      image_family: freebsd-11-3-snap
-  install_script: pkg install -y gmake coreutils
-  script: |
-    MOREFLAGS="-Werror" gmake -j all
-    gmake shortest

.gitattributes

@@ -19,3 +19,6 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
+
+# .travis.yml merging
+.travis.yml merge=ours

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,35 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Downloads data '...'
-2. Run '...' with flags '...'
-3. Scroll up on the log to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots and charts**
-If applicable, add screenshots and charts to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. Mac]
- - Version [e.g. 22]
- - Compiler [e.g gcc]
- - Flags [e.g O2]
- - Other relevant hardware specs [e.g. Dual-core]
- - Build system [e.g. Makefile]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/workflows/dev-long-tests.yml

@@ -1,240 +0,0 @@
-name: dev-long-tests
-# Tests longer than 10mn
-
-concurrency:
-  group: long-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  make-all:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make all
-      run: make all
-
-  # lasts ~24mn
-  make-test:
-    runs-on: ubuntu-latest
-    env:
-      DEVNULLRIGHTS: 1
-      READFROMBLOCKDEVICE: 1
-    steps:
-    - uses: actions/checkout@v2
-    - name: make test
-      run: make test
-
-  # lasts ~26mn
-  make-test-osx:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: OS-X test
-      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
-
-  no-intrinsics-fuzztest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: no intrinsics fuzztest
-      run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
-
-  tsan-zstreamtest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: thread sanitizer zstreamtest
-      run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
-
-  # lasts ~15mn
-  tsan-fuzztest:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: thread sanitizer fuzztest
-      run: CC=clang make tsan-fuzztest
-
-  # lasts ~23mn
-  gcc-8-asan-ubsan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Test Zstd
-      run: |
-        sudo apt-get -qqq update
-        make gcc8install
-        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
-
-  gcc-asan-ubsan-testzstd-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Test Zstd, 32bit mode
-      run: |
-        make libc6install
-        make -j uasan-test-zstd32 V=1
-
-    # Note : external libraries must be turned off when using MSAN tests,
-    # because they are not msan-instrumented,
-    # so any data coming from these libraries is always considered "uninitialized"
-
-  gcc-8-asan-ubsan-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-8 + ASan + UBSan + Fuzz Test
-      run: |
-        make gcc8install
-        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
-
-  gcc-asan-ubsan-fuzz32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Fuzz Test 32bit
-      run: |
-        make libc6install
-        CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
-
-  asan-ubsan-regression:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ASan + UBSan + Regression Test
-      run: make -j uasanregressiontest
-
-  msan-regression:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: MSan + Regression Test
-      run: make -j msanregressiontest
-
-  clang-msan-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang + MSan + Fuzz Test
-      run: |
-        sudo apt-get -qqq update
-        sudo apt-get install clang
-        CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
-
-  # lasts ~24mn
-  clang-msan-testzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang + MSan + Test Zstd
-      run: |
-        sudo apt-get update
-        sudo apt-get install clang
-        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
-
-  armfuzz:
-      runs-on: ubuntu-latest
-      steps:
-      - uses: actions/checkout@v2
-      - name: Qemu ARM emulation + Fuzz Test
-        run: |
-          sudo apt-get -qqq update
-          make arminstall
-          make armfuzz
-
-  valgrind-fuzz-test:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: valgrind + fuzz test stack mode    # ~ 7mn
-      shell: 'script -q -e -c "bash {0}"'
-      run: |
-        make valgrindinstall
-        make -C tests valgrindTest
-        make clean
-        make -C tests test-fuzzer-stackmode
-
-  mingw-long-test:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
-          { compiler: gcc, platform: x64, action: test, script: ""},
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Mingw long test
-      run: |
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        IF ("${{matrix.action}}" -eq "build")
-        {
-          make -v
-          sh -c "${{matrix.compiler}} -v"
-          ECHO "Building zlib to static link"
-          $env:CC = "${{matrix.compiler}}"
-          sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
-          sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
-          ECHO "Building zstd"
-          $env:CPPFLAGS = "-I../../zlib"
-          $env:LDFLAGS = "../../zlib/libz.a"
-          sh -c "${{matrix.script}}"
-        }
-        ELSEIF ("${{matrix.action}}" -eq "test")
-        {
-            ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-            $env:CC = "gcc"
-            $env:CXX = "g++"
-            MKDIR build\cmake\build
-            CD build\cmake\build
-            $env:FUZZERTEST = "-T2mn"
-            $env:ZSTREAM_TESTTIME = "-T2mn"
-            cmake -G "Visual Studio 14 2015 Win64" ..
-            cd ..\..\..
-            make clean
-        }
-
-  # lasts ~20mn
-  oss-fuzz:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        sanitizer: [address, undefined, memory]
-    steps:
-    - name: Build Fuzzers (${{ matrix.sanitizer }})
-      id: build
-      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Run Fuzzers (${{ matrix.sanitizer }})
-      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
-      with:
-        oss-fuzz-project-name: 'zstd'
-        fuzz-seconds: 600
-        dry-run: false
-        sanitizer: ${{ matrix.sanitizer }}
-    - name: Upload Crash
-      uses: actions/upload-artifact@v1
-      if: failure() && steps.build.outcome == 'success'
-      with:
-        name: ${{ matrix.sanitizer }}-artifacts
-        path: ./out/artifacts

.github/workflows/dev-short-tests.yml

@@ -1,427 +0,0 @@
-name: dev-short-tests
-# Faster tests: mostly build tests, along with some other
-# misc tests
-
-concurrency:
-  group: fast-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-jobs:
-  linux-kernel:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: linux kernel, library + build + test
-      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
-
-  benchmarking:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make benchmarking
-      run: make benchmarking
-
-  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make check on 32-bit
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
-
-  check-x32:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
-      env:
-        CHECK_CONSTRAINED_MEM: true
-      run: |
-        sudo apt update
-        APT_PACKAGES="gcc-multilib" make apt-install
-        CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
-
-  gcc-7-libzstd:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: gcc-7 + libzstdmt compilation
-      run: |
-        sudo apt-get -qqq update
-        make gcc7install
-        CC=gcc-7 CFLAGS=-Werror make -j all
-        make clean
-        LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
-
-    # candidate test (to check) : underlink test
-    # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
-
-  cmake-build-and-test-check:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: cmake build and test check
-      run: |
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-        cp -r ./ "../zstd source"
-        cd "../zstd source"
-        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
-
-  cpp-gnu90-c99-compatibility:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: C++, gnu90 and c99 compatibility
-      run: |
-        make cxxtest
-        make clean
-        make gnu90build
-        make clean
-        make c99build
-        make clean
-        make travis-install   # just ensures `make install` works
-
-  mingw-cross-compilation:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: mingw cross-compilation
-      run: |
-        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
-        sudo apt-get -qqq update
-        sudo apt-get install gcc-mingw-w64
-        CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
-
-  armbuild:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: ARM Build Test
-      run: |
-        sudo apt-get -qqq update
-        make arminstall
-        make armbuild
-
-  bourne-shell:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Bourne shell compatibility (shellcheck)
-      run: |
-        wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
-        tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
-        shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
-
-  zlib-wrapper:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: zlib wrapper test
-      run: |
-        sudo apt-get -qqq update
-        make valgrindinstall
-        make -C zlibWrapper test
-        make -C zlibWrapper valgrindTest
-
-  lz4-threadpool-libs:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: LZ4, thread pool, and libs build testslib wrapper test
-      run: |
-        make lz4install
-        make -C tests test-lz4
-        make check < /dev/null | tee   # mess with lz4 console detection
-        make clean
-        make -C tests test-pool
-        make clean
-        bash tests/libzstd_builds.sh
-
-  gcc-make-tests-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Make all, 32bit mode
-      run: |
-        sudo apt-get -qqq update
-        make libc6install
-        CFLAGS="-Werror -m32" make -j all32
-
-  gcc-8-make:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: gcc-8 build
-        run: |
-          sudo apt-get -qqq update
-          make gcc8install
-          CC=gcc-8 CFLAGS="-Werror" make -j all
-
-  implicit-fall-through:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: -Wimplicit-fallthrough build
-        run: |
-          make clean
-          CC=gcc MOREFLAGS="-Werror -Wimplicit-fallthrough=2 -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
-          make clean
-          CC=clang MOREFLAGS="-Werror -Wimplicit-fallthrough -O0" make -C lib -j libzstd.a ZSTD_LEGACY_SUPPORT=0
-
-  cmake-visual-2019:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        include:
-          - generator: "Visual Studio 16 2019"
-            flags: "-A x64"
-          - generator: "Visual Studio 16 2019"
-            flags: "-A Win32"
-          - generator: "MinGW Makefiles"
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      run: |
-        cd build\cmake
-        mkdir build
-        cd build
-        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
-        cmake.exe --build .
-
-  visual-2019:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Debug, Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-
-  visual-2015:
-    # only GH actions windows-2016 contains VS 2015
-    runs-on: windows-2016
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Debug, Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-
-  minimal-decompressor-macros:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: minimal decompressor macros
-      run: |
-        make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
-        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
-        make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
-        make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-        make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-
-  dynamic-bmi2:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: dynamic bmi2 tests
-      run: |
-        make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=1 -mbmi2"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0"
-        make clean && make -j check MOREFLAGS="-O0 -Werror -DDYNAMIC_BMI2=0 -mbmi2"
-
-  test-variants:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make all variants & validate
-      run: |
-        make -j -C programs allVariants MOREFLAGS=-O0
-        ./tests/test-variants.sh
-
-
-  qemu-consistency:
-    name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
-      matrix:
-        include: [
-          { name: ARM,      xcc_pkg: gcc-arm-linux-gnueabi,     xcc: arm-linux-gnueabi-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-arm-static     },
-          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-aarch64-static },
-          { name: PPC,      xcc_pkg: gcc-powerpc-linux-gnu,     xcc: powerpc-linux-gnu-gcc,     xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc-static     },
-          { name: PPC64LE,  xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc64le-static },
-          { name: S390X,    xcc_pkg: gcc-s390x-linux-gnu,       xcc: s390x-linux-gnu-gcc,       xemu_pkg: qemu-system-s390x,  xemu: qemu-s390x-static   },
-          { name: MIPS,     xcc_pkg: gcc-mips-linux-gnu,        xcc: mips-linux-gnu-gcc,        xemu_pkg: qemu-system-mips,   xemu: qemu-mips-static    },
-          { name: M68K,     xcc_pkg: gcc-m68k-linux-gnu,        xcc: m68k-linux-gnu-gcc,        xemu_pkg: qemu-system-m68k,   xemu: qemu-m68k-static    },
-        ]
-    env:                        # Set environment variables
-      XCC: ${{ matrix.xcc }}
-      XEMU: ${{ matrix.xemu }}
-    steps:
-    - uses: actions/checkout@v2 # https://github.com/actions/checkout
-    - name: apt update & install
-      run: |
-        sudo apt-get update
-        sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
-        sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
-    - name: Environment info
-      run: |
-        echo && which $XCC
-        echo && $XCC --version
-        echo && $XCC -v  # Show built-in specs
-        echo && which $XEMU
-        echo && $XEMU --version
-    - name: ARM
-      if: ${{ matrix.name == 'ARM' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: ARM64
-      if: ${{ matrix.name == 'ARM64' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: PPC
-      if: ${{ matrix.name == 'PPC' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: PPC64LE
-      if: ${{ matrix.name == 'PPC64LE' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: S390X
-      if: ${{ matrix.name == 'S390X' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: MIPS
-      if: ${{ matrix.name == 'MIPS' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: M68K
-      if: ${{ matrix.name == 'M68K' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-
-  mingw-short-test:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: gcc, platform: x64, script: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"},
-          { compiler: gcc, platform: x86, script: "CFLAGS=-Werror make -j allzstd"},
-          { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Mingw short test
-      run: |
-        ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        make -v
-        sh -c "${{matrix.compiler}} -v"
-        $env:CC = "${{matrix.compiler}}"
-        sh -c "${{matrix.script}}"
-        ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-        make clean
-        make check
-
-
-  visual-runtime-tests:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        platform: [x64, Win32]
-        configuration: [Release]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
-    - name: Build and run tests
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      env:
-        ZSTD_BIN: ./zstd.exe
-        DATAGEN_BIN: ./datagen.exe
-      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      run: |
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v142 /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
-        COPY build\VS2010\bin\${{matrix.platform}}_${{matrix.configuration}}\*.exe tests\
-        CD tests
-        sh -e playTests.sh
-        DIR
-        .\fuzzer.exe -T2m
-
-# This test currently fails on Github Actions specifically.
-# Possible reason : TTY emulation.
-# Note that the same test works fine locally and on travisCI.
-# This will have to be fixed before transferring the test to GA.
-#  versions-compatibility:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v2
-#    - name: Versions Compatibility Test
-#      run: |
-#        make -C tests versionsTest
-
-
-# For reference : icc tests
-# icc tests are currently failing on Github Actions, likely to issues during installation stage
-# To be fixed later
-#
-#  icc:
-#    name: icc-check
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: install icc
-#      run: |
-#        export DEBIAN_FRONTEND=noninteractive
-#        sudo apt-get -qqq update
-#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-#        sudo apt-get update
-#        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v2
-#    - name: make check
-#      run: |
-#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

.github/workflows/publish-release-artifacts.yml

@@ -1,68 +0,0 @@
-name: publish-release-artifacts
-
-on:
-  release:
-    types:
-      - created
-
-jobs:
-  publish-release-artifacts:
-    runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags/')
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Archive
-        env:
-          RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
-          RELEASE_SIGNING_KEY_PASSPHRASE: ${{ secrets.RELEASE_SIGNING_KEY_PASSPHRASE }}
-        run: |
-          # compute file name
-          export TAG="$(echo "$GITHUB_REF" | sed -n 's_^refs/tags/__p')"
-          if [ -z "$TAG" ]; then
-            echo "action must be run on a tag. GITHUB_REF is not a tag: $GITHUB_REF"
-            exit 1
-          fi
-          # Attempt to extract "1.2.3" from "v1.2.3" to maintain artifact name backwards compat.
-          # Otherwise, degrade to using full tag.
-          export VERSION="$(echo "$TAG" | sed 's_^v\([0-9]\+\.[0-9]\+\.[0-9]\+\)$_\1_')"
-          export ZSTD_VERSION="zstd-$VERSION"
-
-          # archive
-          git archive $TAG \
-              --prefix $ZSTD_VERSION/ \
-              --format tar \
-              -o $ZSTD_VERSION.tar
-
-          # Do the rest of the work in a sub-dir so we can glob everything we want to publish.
-          mkdir artifacts/
-          mv $ZSTD_VERSION.tar artifacts/
-          cd artifacts/
-
-          # compress
-          zstd -k -19 $ZSTD_VERSION.tar
-          gzip -k  -9 $ZSTD_VERSION.tar
-
-          # we only publish the compressed tarballs
-          rm $ZSTD_VERSION.tar
-
-          # hash
-          sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
-          sha256sum $ZSTD_VERSION.tar.gz  > $ZSTD_VERSION.tar.gz.sha256
-
-          # sign
-          if [ -n "$RELEASE_SIGNING_KEY" ]; then
-            export GPG_BATCH_OPTS="--batch --no-use-agent --pinentry-mode loopback --no-tty --yes"
-            echo "$RELEASE_SIGNING_KEY" | gpg $GPG_BATCH_OPTS --import
-            gpg $GPG_BATCH_OPTS --armor --sign --sign-with signing@zstd.net --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.zst.sig $ZSTD_VERSION.tar.zst
-            gpg $GPG_BATCH_OPTS --armor --sign --sign-with signing@zstd.net --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.gz.sig  $ZSTD_VERSION.tar.gz
-          fi
-
-      - name: Publish
-        uses: skx/github-action-publish-binaries@release-1.3
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          args: artifacts/*

.gitignore

@@ -14,7 +14,7 @@
 *.dylib
 
 # Executables
-/zstd
+zstd
 zstdmt
 *.exe
 *.out
@@ -22,20 +22,14 @@ zstdmt
 
 # Test artefacts
 tmp*
-*.zst
-*.zstd
-dictionary.
-dictionary
+dictionary*
 NUL
 
 # Build artefacts
-contrib/linux-kernel/linux/
 projects/
 bin/
 .buckd/
 buck-out/
-build-*
-*.gcda
 
 # Other files
 .directory
@@ -47,9 +41,3 @@ _zstdbench/
 .DS_Store
 googletest/
 *.d
-*.vscode
-*.code-workspace
-compile_commands.json
-.clangd
-perf.data
-perf.data.old

.travis.yml

@@ -1,7 +1,59 @@
-# Travis CI is used to test platforms that github-actions currently doesn't support
-# without either self-hosting or some finnicky work-around. Also, some tests
-# are troublesome to migrate since GH Actions runs tests not in a tty.
+# Medium Tests: Run on all commits/PRs to dev branch
+
 language: c
+dist: trusty
+sudo: required
+addons:
+  apt:
+    update: true
+
+matrix:
+  include:
+    # Ubuntu 14.04
+    - env: Cmd='make test'
+
+    - env: Cmd='make gcc6install && CC=gcc-6 CFLAGS=-Werror make -j all
+             && make clean && CC=gcc-6 make -j uasan-test-zstd </dev/null'   # test when stdin is not a tty
+    - env: Cmd='make gcc6install libc6install
+             && make clean && CC=gcc-6 make -j uasan-test-zstd32'
+    - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd'
+    - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
+
+    - env: Cmd='make staticAnalyze'
+
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
+    - env: Cmd='make gcc6install libc6install
+             && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
+
+    - env: Cmd='make -j uasanregressiontest
+             && make clean && make -j msanregressiontest'
+
+    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
+             && make clean && make -C tests test-fuzzer-stackmode'
+
+    - env: Cmd='make arminstall && make armfuzz'
+    - env: Cmd='make arminstall && make aarch64fuzz'
+    - env: Cmd='make ppcinstall && make ppcfuzz'
+    - env: Cmd='make ppcinstall && make ppc64fuzz'
+
+    - env: Cmd='make lz4install && make -C tests test-lz4
+             && make clean && make -C tests test-pool
+             && make clean && bash tests/libzstd_partial_builds.sh'
+
+    # tag-specific test
+    - if: tag =~ ^v[0-9]\.[0-9]
+      env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'
+
+    # additional tests for master branch and cron job
+    - env: Cmd="make test"
+      os: osx
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
+    - env: Cmd='make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install'
+    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
+
 
 git:
   depth: 1
@@ -9,120 +61,13 @@ git:
 branches:
   only:
   - dev
-  - release
   - master
   - travisTest
 
-addons:
-  apt:
-    update: true
-
-env:
-  global:
-    - FUZZERTEST=-T1mn
-      ZSTREAM_TESTTIME=-T1mn
-      DECODECORPUS_TESTTIME=-T1mn
-
-matrix:
-  fast_finish: true
-  include:
-    - name: S390X (big endian) + Fuzz test
-      dist: trusty
-      arch: s390x
-      script:
-        - FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
-    - name: S390X (big endian) + Fuzz test + no intrinsics
-      dist: trusty
-      arch: s390x
-      script:
-        - MOREFLAGS="-DZSTD_NO_INTRINSICS" FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
-
-    - name: arm64    # ~2.5 mn
-      os: linux
-      arch: arm64
-      script:
-        - make check
-
-    - name: arm64fuzz
-      os: linux
-      arch: arm64
-      script:
-        - make -C tests fuzztest
-
-    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
-    - name: static analyzer scanbuild    # ~8mn
-      dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
-      script:
-        - make staticAnalyze
-
-    # GH actions can't run this command on OS-X, non-tty issues
-    - name: OS-X make all lib
-      os: osx
-      script:
-        - make -C lib all
-
-    # Introduced to check compat with old toolchains, to prevent e.g. #1872
-    - name: ARM Build Test (on Trusty)
-      dist: trusty
-      script:
-        - make arminstall
-        - make armbuild
-
-    # check release number (release/new tag only)
-    - name: Tag-Specific Test
-      if: tag =~ ^v[0-9]\.[0-9]
-      script:
-        - make -C tests checkTag
-        - tests/checkTag "$TRAVIS_BRANCH"
-
-    - name: PPC64LE + Fuzz test  # ~13mn
-      arch: ppc64le
-      env:
-        - FUZZER_FLAGS=--no-big-tests
-        - MOREFLAGS="-static"
-      script:
-        - cat /proc/cpuinfo
-        - make -C tests fuzztest
-
-    # This test currently fails on GA specifically, for no obvious reason
-    # (it works fine on travisCI, and on local test platforms).
-    - name: Versions Compatibility Test   # ~6mn
-      script:
-        - make -C tests versionsTest
-
-    # meson dedicated test
-    - name: Xenial (Meson + clang)    # ~15mn
-      dist: bionic
-      language: cpp
-      compiler: clang
-      install:
-        - sudo apt-get install -qq liblz4-dev valgrind tree
-        - |
-          travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
-          unzip ~/ninja.zip -d ~/.local/bin
-        - |
-          travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
-          python3 ~/get-pip.py --user &&
-          pip3 install --user meson
-      script:
-        - |
-          meson setup \
-            --buildtype=debugoptimized \
-            -Db_lundef=false \
-            -Dauto_features=enabled \
-            -Dbin_programs=true \
-            -Dbin_tests=true \
-            -Dbin_contrib=true \
-            -Ddefault_library=both \
-             build/meson builddir
-        - pushd builddir
-        - ninja
-        - meson test --verbose --no-rebuild
-        - DESTDIR=./staging ninja install
-        - tree ./staging
-      after_failure:
-        - cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
-
-  allow_failures:
-    - env: ALLOW_FAILURES=true
+script:
+  - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
+  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
+  - export FUZZERTEST=-T2mn;
+    export ZSTREAM_TESTTIME=-T2mn;
+    export DECODECORPUS_TESTTIME=-T1mn;
+    sh -c "$Cmd" || travis_terminate 1;

CHANGELOG

@@ -1,716 +0,0 @@
-v1.5.1 (Dec, 2021)
-perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42
-perf: faster huffman decoder, using x64 assembly, by @terrelln
-perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte
-perf: improved binary size and faster compilation times, by @terrelln
-perf: new row64 mode, used notably in level 12, by @senhuang42
-perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42
-perf: minor compression ratio improvements for small data at high levels, by @cyan4973
-perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln
-perf: faster compression speed on incompressible data, by @bindhvo
-perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo
-build: allows hiding static symbols in the dynamic library, using build macro, by @skitt
-build: support for m68k (Motorola 68000's), by @cyan4973
-build: improved AIX support, by @Helflym
-build: improved meson unofficial build, by @eli-schwartz
-cli : custom memory limit when training dictionary (#2925), by @embg
-cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB
-
-v1.5.0  (May 11, 2021)
-api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
-  `ZSTD_defaultCLevel()`
-  `ZSTD_getDictID_fromCDict()`
-api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42)
-  `ZSTD_compress_advanced()`
-  `ZSTD_compress_usingCDict_advanced()`
-  `ZSTD_compressBegin_advanced()`
-  `ZSTD_compressBegin_usingCDict_advanced()`
-  `ZSTD_initCStream_srcSize()`
-  `ZSTD_initCStream_usingDict()`
-  `ZSTD_initCStream_usingCDict()`
-  `ZSTD_initCStream_advanced()`
-  `ZSTD_initCStream_usingCDict_advanced()`
-  `ZSTD_resetCStream()`
-api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973)
-perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln)
-perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42)
-perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973)
-perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln)
-bug: Improve setting permissions of created files (#2525, @felixhandte)
-bug: Fix large dictionary non-determinism (#2607, @terrelln)
-bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln)
-bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte)
-bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42)
-bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink)
-bug: Fix UBSAN error in decompression (#2625, @terrelln)
-bug: Fix superblock compression divide by zero bug (#2592, @senhuang42)
-bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone)
-doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln)
-doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize)
-doc: Add style guide docs for open source contributors (#2626, @Cyan4973)
-tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42)
-tests: Better test coverage of index reduction (#2603, @terrelln)
-tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42)
-tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42)
-build: Dynamic library built multithreaded by default (#2584, @senhuang42)
-build: Move  `zstd_errors.h`  and  `zdict.h`  to  `lib/`  root (#2597, @terrelln)
-build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973)
-build: Single file library build script moved to `build/` directory (#2618, @felixhandte)
-build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42)
-build: Fixed Meson build (#2548, @SupervisedThinking @kloczek)
-build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson)
-build: Detect presence of `md5` on Darwin (#2609, @felixhandte)
-build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann)
-cli: `--progress` flag added to always display progress bar (#2595, @senhuang42)
-cli: Allow reading from block devices with `--force` (#2613, @felixhandte)
-cli: Fix CLI filesize display bug (#2550, @Cyan4973)
-cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973)
-contrib: Various fixes for linux kernel patch (#2539, @terrelln)
-contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42)
-contrib: Seekable format - New seek table-only API  (#2113 #2518, @mdittmer @Cyan4973)
-contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng)
-contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat)
-misc: Automatically published release tarballs available on Github (#2535, @felixhandte)
-
-v1.4.9  (Mar 1, 2021)
-bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
-bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
-api: Add (De)Compression Tracing Functionality (#2482, @terrelln)
-api: Support References to Multiple DDicts (#2446, @senhuang42)
-api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
-perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
-perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
-perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
-cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
-cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
-cli: Improve Help Message (#2500, @senhuang42)
-tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
-tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat)
-tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte)
-build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973)
-build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt)
-build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime)
-build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka)
-build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln)
-build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg)
-build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973)
-contrib: Add `recover_directory` Program (#2473, @terrelln)
-doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42)
-doc: Fix Typos (#2459, @ThomasWaldmann)
-
-v1.4.8  (Dec 18, 2020)
-hotfix: wrong alignment of an internal buffer
-
-v1.4.7  (Dec 16, 2020)
-perf: stronger --long mode at high compression levels, by @senhuang42
-perf: stronger --patch-from at high compression levels, thanks to --long improvements
-perf: faster dictionary compression at medium compression levels, by @felixhandte
-perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln
-perf: improved fast compression speeds with Visual Studio, by @animalize
-cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42
-cli : accept decompressing files with *.zstd suffix
-cli : provide a condensed summary by default when processing multiple files
-cli : fix : stdin input no longer confused as user prompt
-cli : improve accuracy of several error messages
-api : new sequence ingestion API, by @senhuang42
-api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin
-api : new ZSTD_getDictID_fromCDict(), by @LuAPi
-api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln
-api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli)
-api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli)
-build: fix cmake script when using path with spaces, by @terrelln
-build: improved compile-time detection of aarch64/neon platforms, by @bsdimp
-build: Fix building on AIX 5.1, by @likema
-build: compile paramgrill with cmake on Windows, requested by @mirh
-doc : clarify repcode updates in format specification, by @felixhandte
-
-v1.4.6
-fix : Always return dstSize_tooSmall when that is the case
-fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary
-perf: Improve small block decompression speed by 20%+, by @terrelln
-perf: Reduce compression stack usage by 1 KB, by @terrelln
-perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256)
-perf: Improve histogram construction, by @cyan4973 (#2253)
-cli : Add --output-dir-mirror option, by @xxie24 (#2219)
-cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279)
-cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283)
-cli : Call stat less often, by @felixhandte (#2262)
-cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250)
-cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206)
-api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166).
-api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291)
-api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228).
-build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183)
-build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221)
-build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186)
-build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions
-build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics
-build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258)
-build: Fix -Wcomma warnings, by @cwoffenden
-build: Remove distutils requirement for meson build, by @neheb (#2197)
-build: Fix cli compilation with uclibc
-build: Fix cli compilation without st_mtime, by @ffontaine (#2246)
-build: Fix shadowing warnings in library
-build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227)
-misc: Improve single file library and include dictBuilder, by @cwoffenden
-misc: Allow compression dictionaries with missing symbols
-misc: Add freestanding translation script in contrib/freestanding_lib
-misc: Collect all of zstd's libc dependencies into zstd_deps.h
-doc : Add ZSTD_versionString() to manual, by @animalize
-doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270)
-
-v1.4.5  (May 22, 2020)
-fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
-perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
-perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
-perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
-perf: Small level 1 compression speed gains (depending on compiler)
-cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
-cli : New --filelist= : Provide a list of files to operate upon from a file
-cli : -b -d command can now benchmark decompression on multiple files
-cli : New --no-content-size command
-cli : New --show-default-cparams information command
-api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
-api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
-build: Generate a single-file libzstd library (#2065, by @cwoffenden)
-build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
-build: zstd now compiles cleanly under -pedantic (#2099)
-build: zstd now compiles with make-4.3
-build: Support mingw cross-compilation from Linux, by @Ericson2314
-build: Meson multi-thread build fix on windows
-build: Some misc icc fixes backed by new ci test on travis
-misc: bitflip analyzer tool, by @felixhandte
-misc: Extend largeNbDicts benchmark to compression
-misc: Edit-distance match finder in contrib/
-doc : Improved beginner CONTRIBUTING.md docs
-doc : New issue templates for zstd
-
-v1.4.4  (Nov 6, 2019)
-perf: Improved decompression speed, by > 10%, by @terrelln
-perf: Better compression speed when re-using a context, by @felixhandte
-perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
-perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
-perf: minor generic speed optimization, by @davidbolvansky
-api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
-api: fixed decoding of magic-less frames, by @terrelln
-api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
-cli: Named pipes support, by @bimbashrestha
-cli: short tar's extension support, by @stokito
-cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
-cli: commands --stream-size=# and --size-hint=#, by @nmagerko
-cli: command --exclude-compressed, by @shashank0791
-cli: faster `-t` test mode
-cli: improved some error messages, by @vangyzen
-cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
-cli: fix rare deadlock condition within dictionary builder, by @terrelln
-build: single-file decoder with emscripten compilation script, by @cwoffenden
-build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
-build: fixed deprecation warning for certain gcc version, reported by @jasonma163
-build: fix compilation on old gcc versions, by @cemeyer
-build: improved installation directories for cmake script, by Dmitri Shubin
-pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
-misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
-misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
-
-v1.4.3  (Aug 20, 2019)
-bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
-bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
-build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
-
-v1.4.2  (Jul 26, 2019)
-bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
-bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
-misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
-misc: Restructure source files by @ephiepark (#1679)
-
-v1.4.1  (Jul 20, 2019)
-bug: Fix data corruption in niche use cases by @terrelln (#1659)
-bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
-bug: Fix out of bounds read by @terrelln (#1590)
-perf: Improve decode speed by ~7% @mgrice (#1668)
-perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
-perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
-perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
-perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
-api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
-cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
-cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
-cli: Restrict read permissions on destination files by @chungy (#1644)
-cli: zstdgrep: handle -f flag by @felixhandte (#1618)
-cli: zstdcat: follow symlinks by @vejnar (#1604)
-doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
-doc: Fix typo by @yk-tanigawa (#1633)
-doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
-build: CMake: support building with LZ4 @leeyoung624 (#1626)
-build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
-build: CMake: respect existing uninstall target by @j301scott (#1619)
-build: Make: skip multithread tests when built without support by @michaelforney (#1620)
-build: Make: Fix examples/ test target by @sjnam (#1603)
-build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
-build: Meson: fix build by @lzutao (#1602)
-build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
-build: Visual Studio: fix linking by @absotively (#1639)
-build: Fix MinGW-W64 build by @myzhang1029 (#1600)
-misc: Expand decodecorpus coverage by @ephiepark (#1664)
-
-v1.4.0  (Apr 17, 2019)
-perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
-api: Move the advanced API, including all functions in the staging section, to the stable section
-api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
-api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
-api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
-api: Don't export ZSTDMT functions from the shared library by default
-api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
-api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
-api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
-api: Move ZSTD_getDictID_*() functions to the stable section
-api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
-api: Allow compression parameters to be set when a dictionary is used
-api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
-api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
-api: Setting ZSTD_d_maxWindowLog to 0 means use the default
-cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
-cli: Add --[no-]compress-literals flag to enable or disable literal compression
-doc: Update the examples to use the advanced API
-doc: Explain how to transition from old streaming functions to the advanced API in the header
-build: Improve the Windows release packages
-build: Improve CMake build by @hjmjohnson
-build: Build fixes for FreeBSD by @lwhsu
-build: Remove redundant warnings by @thatsafunnyname
-build: Fix tests on OpenBSD by @bket
-build: Extend fuzzer build system to work with the new clang engine
-build: CMake now creates the libzstd.so.1 symlink
-build: Improve Menson build by @lzutao
-misc: Fix symbolic link detection on FreeBSD
-misc: Use physical core count for -T0 on FreeBSD by @cemeyer
-misc: Fix zstd --list on truncated files by @kostmo
-misc: Improve logging in debug mode by @felixhandte
-misc: Add CirrusCI tests by @lwhsu
-misc: Optimize dictionary memory usage in corner cases
-misc: Improve the dictionary builder on small or homogeneous data
-misc: Fix spelling across the repo by @jsoref
-
-v1.3.8  (Dec 28, 2018)
-perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
-perf: slightly better compression ratio at high compression modes
-api : finalized advanced API, last stage before "stable" status
-api : new --rsyncable mode, by @terrelln
-api : support decompression of empty frames into NULL (used to be an error) (#1385)
-build: new set of macros to build a minimal size decoder, by @felixhandte
-build: fix compilation on MIPS32, reported by @clbr (#1441)
-build: fix compilation with multiple -arch flags, by @ryandesign
-build: highly upgraded meson build, by @lzutao
-build: improved buck support, by @obelisk
-build: fix cmake script : can create debug build, by @pitrou
-build: Makefile : grep works on both colored consoles and systems without color support
-build: fixed zstd-pgo, by @bmwiedemann
-cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
-cli : --no-progress flag, preserving final summary (#1371), by @terrelln
-cli : ensure destination file is not source file (#1422)
-cli : clearer error messages, especially when input file not present
-doc : clarified zstd_compression_format.md, by @ulikunitz
-misc: fixed zstdgrep, returns 1 on failure, by @lzutao
-misc: NEWS renamed as CHANGELOG, in accordance with fboss
-
-v1.3.7  (Oct 20, 2018)
-perf: slightly better decompression speed on clang (depending on hardware target)
-fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
-build: no longer build backtrace by default in release mode; restrict further automatic mode
-build: control backtrace support through build macro BACKTRACE
-misc: added man pages for zstdless and zstdgrep, by @samrussell
-
-v1.3.6  (Oct 6, 2018)
-perf: much faster dictionary builder, by @jenniferliu
-perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
-perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
-cli : fix : does no longer overwrite destination when source does not exist (#1082)
-cli : new command --adapt, for automatic compression level adaptation
-api : fix : block api can be streamed with > 4 GB, reported by @catid
-api : reduced ZSTD_DDict size by 2 KB
-api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
-build: support Haiku target, by @korli
-build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
-doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
-misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
-
-v1.3.5  (Jun 29, 2018)
-perf: much faster dictionary compression, by @felixhandte
-perf: small quality improvement for dictionary generation, by @terrelln
-perf: slightly improved high compression levels (notably level 19)
-mem : automatic memory release for long duration contexts
-cli : fix : overlapLog can be manually set
-cli : fix : decoding invalid lz4 frames
-api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
-api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
-build: select custom libzstd scope through control macros, by @GeorgeLu97
-build: OpenBSD patch, by @bket
-build: make and make all are compatible with -j
-doc : clarify zstd_compression_format.md, updated for IETF RFC process
-misc: pzstd compatible with reproducible compilation, by @lamby
-
-v1.3.4  (Mar 27, 2018)
-perf: faster speed (especially decoding speed) on recent cpus (haswell+)
-perf: much better performance associating --long with multi-threading, by @terrelln
-perf: better compression at levels 13-15
-cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
-cli : smoother status report in multi-threading mode
-cli : added command --fast=#, for faster compression modes
-cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
-api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
-api : compression levels can be negative, for even more speed
-api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
-api : ZSTDMT can accept new compression parameters during compression
-api : implemented all advanced dictionary decompression prototypes
-build: improved meson recipe, by Shawn Landden (@shawnl)
-build: VS2017 scripts, by @HaydnTrigg
-misc: all /contrib projects fixed
-misc: added /contrib/docker script by @gyscos
-
-v1.3.3  (Dec 21, 2017)
-perf: faster zstd_opt strategy (levels 16-19)
-fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
-cli : fix : content size written in header by default
-cli : fix : improved LZ4 format support, by @felixhandte
-cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
-api : fix : support large skippable frames, by @terrelln
-api : fix : streaming interface was adding a useless 3-bytes null block to small frames
-api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
-build: fix : compilation under rhel6 and centos6, reported by @pixelb
-build: added `check` target
-
-v1.3.2  (Oct 10, 2017)
-new : long range mode, using --long command, by Stella Lau (@stellamplau)
-new : ability to generate and decode magicless frames (#591)
-changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
-fix : multi-threading compression works with custom allocators
-fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
-fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
-fix : 32-bits build can now decode large offsets (levels 21+)
-cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
-cli : improved --list output
-cli : new : can split input file for dictionary training, using command -B#
-cli : new : clean operation artefact on Ctrl-C interruption
-cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
-cli : fix : write file size in header in multiple-files mode
-api : added macro ZSTD_COMPRESSBOUND() for static allocation
-api : experimental : new advanced decompression API
-api : fix : sizeof_CCtx() used to over-estimate
-build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
-build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
-example : added streaming_memory_usage
-license : changed /examples license to BSD + GPLv2
-license : fix a few header files to reflect new license (#825)
-
-v1.3.1  (Aug 21, 2017)
-New license : BSD + GPLv2
-perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
-perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
-cli : improved and fixed --list command, by @ib (#772)
-cli : command -vV to list supported formats, by @ib (#771)
-build : fixed binary variants, reported by @svenha (#788)
-build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
-API exp : breaking change : ZSTD_getframeHeader() provides more information
-API exp : breaking change : pinned down values of error codes
-doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
-new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
-new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
-updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
-
-v1.3.0  (Jul 6, 2017)
-cli : new : `--list` command, by Paul Cruz
-cli : changed : xz/lzma support enabled by default
-cli : changed : `-t *` continue processing list after a decompression error
-API : added : ZSTD_versionString()
-API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
-API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
-API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
-API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
-API exp : clarified memory estimation / measurement functions.
-API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
-tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
-new : contrib/seekable_format, demo and API, by Sean Purcell
-changed : contrib/linux-kernel, updated version and license, by Nick Terrell
-
-v1.2.0  (May 5, 2017)
-cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
-cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
-cli : new : zstdmt symlink hardwired to `zstd -T0`
-cli : new : command --threads=# (#671)
-cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
-cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
-cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
-cli : fix : does not output compressed data on console
-cli : fix : ignore symbolic links unless --force specified,
-API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
-API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
-API : improved: ZSTDMT_compressCCtx() reduced memory usage
-API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
-API : fix : src size stored in frame header is controlled at end of frame
-API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
-API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
-build: improved cmake script, by @Majlen
-build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
-tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
-new : contrib/linux-kernel version, by Nick Terrell
-
-v1.1.4  (Mar 18, 2017)
-cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
-cli : new : advanced benchmark command --priority=rt
-cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
-cli : fix : --rm remains silent when input is stdin
-cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
-speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
-memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
-arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
-API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
-API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
-build : new: meson build system in contrib/meson, by Dima Krasner
-build : improved cmake script, by @Majlen
-build : added -Wformat-security flag, as recommended by Padraig Brady
-doc : new : educational decoder, by Sean Purcell
-
-v1.1.3  (Feb 7, 2017)
-cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
-cli : new : experimental target `make zstdmt`, with multi-threading support
-cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
-cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
-cli : fix zstdless on Mac OS-X, by Andrew Janke
-cli : fix #232 "compress non-files"
-dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
-API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
-API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
-API : new : ZDICT_finalizeDictionary()
-API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
-API : fix : all symbols properly exposed in libzstd, by Nick Terrell
-build : support for Solaris target, by Przemyslaw Skibinski
-doc : clarified specification, by Sean Purcell
-
-v1.1.2  (Dec 15, 2016)
-API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
-API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
-API : zbuff : changed : prototypes now generate deprecation warnings
-lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
-lib : changed : only public ZSTD_ symbols are now exposed
-lib : changed : reduced usage  of stack memory
-lib : fixed : several corner case bugs, by Nick Terrell
-cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
-cli : new : preserve file attributes
-cli : new : added zstdless and zstdgrep tools
-cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
-cli : fixed : zstdcat
-zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
-install : better compatibility with FreeBSD, by Dimitry Andric
-source tree : changed : zbuff source files moved to lib/deprecated
-
-v1.1.1  (Nov 2, 2016)
-New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
-New : doc/zstd_manual.html, by Przemyslaw Skibinski
-Improved : slightly better compression ratio at --ultra levels (>= 20)
-Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
-Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
-Added : example/multiple_streaming_compression.c
-Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
-Updated man page
-Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
-
-v1.1.0  (Sep 28, 2016)
-New : contrib/pzstd, parallel version of zstd, by Nick Terrell
-added : NetBSD install target (#338)
-Improved : speed for batches of small files
-Improved : speed of zlib wrapper, by Przemyslaw Skibinski
-Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
-Fixed : CLI -d output to stdout by default when input is stdin (#322)
-Fixed : CLI correctly detects console on Mac OS-X
-Fixed : CLI supports recursive mode `-r` on Mac OS-X
-Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
-Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
-Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
-Fixed : zstd-pgo, reported by octoploid (#329)
-
-v1.0.0  (Sep 1, 2016)
-Change Licensing, all project is now BSD, Copyright Facebook
-Small decompression speed improvement
-API : Streaming API supports legacy format
-API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
-CLI supports legacy formats v0.4+
-Fixed : compression fails on certain huge files, reported by Jesse McGrew
-Enhanced documentation, by Przemyslaw Skibinski
-
-v0.8.1  (Aug 18, 2016)
-New streaming API
-Changed : --ultra now enables levels beyond 19
-Changed : -i# now selects benchmark time in second
-Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
-Fixed : speed regression on specific patterns (#272)
-Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
-Fixed : ICC compilation, by Przemyslaw Skibinski
-
-v0.8.0  (Aug 2, 2016)
-Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
-New : Build on FreeBSD and DragonFly, thanks to JrMarino
-Changed : modified API : ZSTD_compressEnd()
-Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
-Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
-Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
-Fixed : checksum correctly checked in single-pass mode
-Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
-Modified : minor compression level adaptations
-Updated : compression format specification to v0.2.0
-changed : zstd.h moved to /lib directory
-
-v0.7.5  (Aug 1, 2016)
-Transition version, supporting decoding of v0.8.x
-
-v0.7.4  (Jul 17, 2016)
-Added : homebrew for Mac, by Daniel Cade
-Added : more examples
-Fixed : segfault when using small dictionaries, reported by Felix Handte
-Modified : default compression level for CLI is now 3
-Updated : specification, to v0.1.1
-
-v0.7.3  (Jul 9, 2016)
-New : compression format specification
-New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
-New : `ZSTD_getDecompressedSize()`
-New : OpenBSD target, by Juan Francisco Cantero Hurtado
-New : `examples` directory
-fixed : dictBuilder using HC levels, reported by Bartosz Taudul
-fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
-fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
-modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
-modified : legacy functions no longer need magic number
-
-v0.7.2  (Jul 4, 2016)
-fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
-fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
-fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
-
-v0.7.1  (Jun 23, 2016)
-fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
-fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
-fixed : corruption issue, reported by cj
-modified : checksum enabled by default in command line mode
-
-v0.7.0  (Jun 17, 2016)
-New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
-New : Command `--rm`, to remove source file after successful de/compression
-New : Visual build scripts, by Christophe Chevalier
-New : Support for Sparse File-systems (do not use space for zero-filled sectors)
-New : Frame checksum support
-New : Support pass-through mode (when using `-df`)
-API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
-API : create dictionary files from custom content, by Giuseppe Ottaviano
-API : support for custom malloc/free functions
-New : controllable Dictionary ID
-New : Support for skippable frames
-
-v0.6.1  (May 13, 2016)
-New : zlib wrapper API, thanks to Przemyslaw Skibinski
-New : Ability to compile compressor / decompressor separately
-Changed : new lib directory structure
-Fixed : Legacy codec v0.5 compatible with dictionary decompression
-Fixed : Decoder corruption error (#173)
-Fixed : null-string roundtrip (#176)
-New : benchmark mode can select directory as input
-Experimental : midipix support, VMS support
-
-v0.6.0  (Apr 13, 2016)
-Stronger high compression modes, thanks to Przemyslaw Skibinski
-API : ZSTD_getFrameParams() provides size of decompressed content
-New : highest compression modes require `--ultra` command to fully unleash their capacity
-Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
-
-v0.5.1  (Feb 18, 2016)
-New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
-Changed : Dictionary builder integrated into libzstd and zstd cli
-Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
-Fix : high compression modes for big-endian platforms
-New : zstd cli : `-t` | `--test` command
-
-v0.5.0  (Feb 5, 2016)
-New : dictionary builder utility
-Changed : streaming & dictionary API
-Improved : better compression of small data
-
-v0.4.7  (Jan 22, 2016)
-Improved : small compression speed improvement in HC mode
-Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
-fix : bt search bug
-
-v0.4.6  (Jan 13, 2016)
-fix : fast compression mode on Windows
-New : cmake configuration file, thanks to Artyom Dymchenko
-Improved : high compression mode on repetitive data
-New : block-level API
-New : ZSTD_duplicateCCtx()
-
-v0.4.5  (Dec 18, 2015)
-new : -m/--multiple : compress/decompress multiple files
-
-v0.4.4  (Dec 14, 2015)
-Fixed : high compression modes for Windows 32 bits
-new : external dictionary API extended to buffered mode and accessible through command line
-new : windows DLL project, thanks to Christophe Chevalier
-
-v0.4.3  (Dec 7, 2015)
-new : external dictionary API
-new : zstd-frugal
-
-v0.4.2  (Dec 2, 2015)
-Generic minor improvements for small blocks
-Fixed : big-endian compatibility, by Peter Harris (#85)
-
-v0.4.1  (Dec 1, 2015)
-Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
-removed `zstd.c`
-
-v0.4.0  (Nov 29, 2015)
-Command line utility compatible with high compression levels
-Removed zstdhc => merged into zstd
-Added : ZBUFF API (see zstd_buffered.h)
-Rolling buffer support
-
-v0.3.6  (Nov 10, 2015)
-small blocks params
-
-v0.3.5  (Nov 9, 2015)
-minor generic compression improvements
-
-v0.3.4  (Nov 6, 2015)
-Faster fast cLevels
-
-v0.3.3  (Nov 5, 2015)
-Small compression ratio improvement
-
-v0.3.2  (Nov 2, 2015)
-Fixed Visual Studio
-
-v0.3.1  (Nov 2, 2015)
-Small compression ratio improvement
-
-v0.3  (Oct 30, 2015)
-HC mode : compression levels 2-26
-
-v0.2.2  (Oct 28, 2015)
-Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
-
-v0.2.1  (Oct 24, 2015)
-Fix : Read errors, advanced fuzzer tests, by Hanno Böck
-
-v0.2.0  (Oct 22, 2015)
-**Breaking format change**
-Faster decompression speed
-Can still decode v0.1 format
-
-v0.1.3  (Oct 15, 2015)
-fix uninitialization warning, reported by Evan Nemerson
-
-v0.1.2  (Sep 11, 2015)
-frame concatenation support
-
-v0.1.1  (Aug 27, 2015)
-fix compression bug
-detects write-flush errors
-
-v0.1.0  (Aug 25, 2015)
-first release

CONTRIBUTING.md

@@ -5,7 +5,7 @@ possible.
 ## Our Development Process
 New versions are being developed in the "dev" branch,
 or in their own feature branch.
-When they are deemed ready for a release, they are merged into "release".
+When they are deemed ready for a release, they are merged into "master".
 
 As a consequences, all contributions must stage first through "dev"
 or their own feature branch.
@@ -26,370 +26,6 @@ to do this once to work on any of Facebook's open source projects.
 
 Complete your CLA here: <https://code.facebook.com/cla>
 
-## Workflow
-Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
-will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
-related changes into a branch.
-
-Our contribution process works in three main stages:
-1. Local development
-    * Update:
-        * Checkout your fork of zstd if you have not already
-        ```
-        git checkout https://github.com/<username>/zstd
-        cd zstd
-        ```
-        * Update your local dev branch
-        ```
-        git pull https://github.com/facebook/zstd dev
-        git push origin dev
-        ```
-    * Topic and development:
-        * Make a new branch on your fork about the topic you're developing for
-        ```
-        # branch names should be concise but sufficiently informative
-        git checkout -b <branch-name>
-        git push origin <branch-name>
-        ```
-        * Make commits and push
-        ```
-        # make some changes =
-        git add -u && git commit -m <message>
-        git push origin <branch-name>
-        ```
-        * Note: run local tests to ensure that your changes didn't break existing functionality
-            * Quick check
-            ```
-            make shortest
-            ```
-            * Longer check
-            ```
-            make test
-            ```
-2. Code Review and CI tests
-    * Ensure CI tests pass:
-        * Before sharing anything to the community, make sure that all CI tests pass on your local fork.
-        See our section on setting up your CI environment for more information on how to do this.
-        * Ensure that static analysis passes on your development machine. See the Static Analysis section
-        below to see how to do this.
-    * Create a pull request:
-        * When you are ready to share you changes to the community, create a pull request from your branch
-        to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
-        page.
-        * From there, select the branch where you made changes as your source branch and facebook:dev
-        as the destination.
-        * Examine the diff presented between the two branches to make sure there is nothing unexpected.
-    * Write a good pull request description:
-        * While there is no strict template that our contributors follow, we would like them to
-        sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
-        at least indirectly, address the following points.
-            * Is this pull request important and why?
-            * Is it addressing an issue? If so, what issue? (provide links for convenience please)
-            * Is this a new feature? If so, why is it useful and/or necessary?
-            * Are there background references and documents that reviewers should be aware of to properly assess this change?
-        * Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
-        * Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
-    * Submit the pull request and iterate with feedback.
-3. Merge and Release
-    * Getting approval:
-        * You will have to iterate on your changes with feedback from other collaborators to reach a point
-        where your pull request can be safely merged.
-        * To avoid too many comments on style and convention, make sure that you have a
-        look at our style section below before creating a pull request.
-        * Eventually, someone from the zstd team will approve your pull request and not long after merge it into
-        the dev branch.
-    * Housekeeping:
-        * Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
-        the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
-        issue at hand, then please indicate this by requesting that an issue be closed by commenting.
-        * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
-        that the change must make it to an official zstd release for it to be meaningful. We recommend
-        that contributors track the activity on their pull request and corresponding issue(s) page(s) until
-        their change makes it to the next release of zstd. Users will often discover bugs in your code or
-        suggest ways to refine and improve your initial changes even after the pull request is merged.
-
-## Static Analysis
-Static analysis is a process for examining the correctness or validity of a program without actually
-executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
-static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
-
-Once installed, you can ensure that our static analysis tests pass on your local development machine
-by running:
-```
-make staticAnalyze
-```
-
-In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
-just `contrib/largeNbDicts` and nothing else, you can run:
-
-```
-scan-build make -C contrib/largeNbDicts largeNbDicts
-```
-
-### Pitfalls of static analysis
-`scan-build` is part of our regular CI suite. Other static analyzers are not.
-
-It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
-
-- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
-- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
-- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
-   + torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
-   + sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
-- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
-
-This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.
-
-## Performance
-Performance is extremely important for zstd and we only merge pull requests whose performance
-landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
-This high bar for performance means that every PR which has the potential to
-impact performance takes a very long time for us to properly review. That being said, we
-always welcome contributions to improve performance (or worsen performance for the trade-off of
-something else). Please keep the following in mind before submitting a performance related PR:
-
-1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
-very well documented via past Github issues and pull requests. It may be the case that your
-particular performance optimization has already been considered in the past. Please take some
-time to search through old issues and pull requests using keywords specific to your
-would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
-on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
-it will be helpful for you to have context from that topic's history before contributing.
-2. The distinction between noise and actual performance gains can unfortunately be very subtle
-especially when microbenchmarking extremely small wins or losses. The only remedy to getting
-something subtle merged is extensive benchmarking. You will be doing us a great favor if you
-take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
-benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
-your changes on every single processor and os out there (and neither will we) but do that best
-you can:) We've adding some things to think about when benchmarking below in the Benchmarking
-Performance section which might be helpful for you.
-3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
-legitimate thing to do as long as it does not harm the overall performance health of Zstd.
-This is a hard balance to strike but please keep in mind other aspects of Zstd when
-submitting changes that are clang-specific, windows-specific, etc.
-
-## Benchmarking Performance
-Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
-testing over theoretical speculation. This guide it not perfect but for most scenarios, it
-is a good place to start.
-
-### Stability
-Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
-benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
-will typically not be stable enough to obtain reliable benchmark results. If you can get your
-hands on a desktop, this is usually a better scenario.
-
-Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
-do a little more work to ensure that you are in fact measuring the changes you've made not and
-noise. Here are some things you can do to make your benchmarks more stable:
-
-1. The most simple thing you can do to drastically improve the stability of your benchmark is
-to run it multiple times and then aggregate the results of those runs. As a general rule of
-thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
-you will have to aggregate over to get reliable results. Here are some additional things to keep in
-mind when running multiple trials:
-    * How you aggregate your samples are important. You might be tempted to use the mean of your
-    results. While this is certainly going to be a more stable number than a raw single sample
-    benchmark number, you might have more luck by taking the median. The mean is not robust to
-    outliers whereas the median is. Better still, you could simply take the fastest speed your
-    benchmark achieved on each run since that is likely the fastest your process will be
-    capable of running your code. In our experience, this (aggregating by just taking the sample
-    with the fastest running time) has been the most stable approach.
-    * The more samples you have, the more stable your benchmarks should be. You can verify
-    your improved stability by looking at the size of your confidence intervals as you
-    increase your sample count. These should get smaller and smaller. Eventually hopefully
-    smaller than the performance win you are expecting.
-    * Most processors will take some time to get `hot` when running anything. The observations
-    you collect during that time period will very different from the true performance number. Having
-    a very large number of sample will help alleviate this problem slightly but you can also
-    address is directly by simply not including the first `n` iterations of your benchmark in
-    your aggregations. You can determine `n` by simply looking at the results from each iteration
-    and then hand picking a good threshold after which the variance in results seems to stabilize.
-2. You cannot really get reliable benchmarks if your host machine is simultaneously running
-another cpu/memory-intensive application in the background. If you are running benchmarks on your
-personal laptop for instance, you should close all applications (including your code editor and
-browser) before running your benchmarks. You might also have invisible background applications
-running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
-on Windows. You will get more stable benchmark results of you end those processes as well.
-    * If you have multiple cores, you can even run your benchmark on a reserved core to prevent
-    pollution from other OS and user processes. There are a number of ways to do this depending
-    on your OS:
-        * On linux boxes, you have use https://github.com/lpechacek/cpuset.
-        * On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
-        * On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
-3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
-Dynamically linking your library will introduce some added variation (not a large amount but
-definitely some). Statically linking libzstd will be more stable. Static libraries should
-be enabled by default when building zstd.
-4. Use a profiler with a good high resolution timer. See the section below on profiling for
-details on this.
-5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
-6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
-tmpfs avoids touching a real storage system, which can have a pretty big variability.
-
-Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
-
-### Zstd benchmark
-The fastest signal you can get regarding your performance changes is via the in-build zstd cli
-bench option. You can run Zstd as you typically would for your scenario using some set of options
-and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
-for that options you have just provided. If you want to look at the internals of how this
-benchmarking script works, you can check out programs/benchzstd.c
-
-For example: say you have made a change that you believe improves the speed of zstd level 1. The
-very first thing you should use to asses whether you actually achieved any sort of improvement
-is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
-specify a running time for your benchmark in seconds (default is 3 seconds).
-Usually, the longer the running time, the more stable your results will be.
-
-```
-$ git checkout <commit-before-your-change>
-$ make && cp zstd zstd-old
-$ git checkout <commit-after-your-change>
-$ make && cp zstd zstd-new
-$ zstd-old -i5 -b1 <your-test-data>
- 1<your-test-data>         :      8990 ->      3992 (2.252), 302.6 MB/s , 626.4 MB/s
-$ zstd-new -i5 -b1 <your-test-data>
- 1<your-test-data>         :      8990 ->      3992 (2.252), 302.8 MB/s , 628.4 MB/s
-```
-
-Unless your performance win is large enough to be visible despite the intrinsic noise
-on your computer, benchzstd alone will likely not be enough to validate the impact of your
-changes. For example, the results of the example above indicate that effectively nothing
-changed but there could be a small <3% improvement that the noise on the host machine
-obscured. So unless you see a large performance win (10-15% consistently) using just
-this method of evaluation will not be sufficient.
-
-### Profiling
-There are a number of great profilers out there. We're going to briefly mention how you can
-profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
-on windows.
-
-Say you have an idea for a change that you think will provide some good performance gains
-for level 1 compression on Zstd. Typically this means, you have identified a section of
-code that you think can be made to run faster.
-
-The first thing you will want to do is make sure that the piece of code is actually taking up
-a notable amount of time to run. It is usually not worth optimizing something which accounts for less than
-0.0001% of the total running time. Luckily, there are tools to help with this.
-Profilers will let you see how much time your code spends inside a particular function.
-If your target code snippet is only part of a function, it might be worth trying to
-isolate that snippet by moving it to its own function (this is usually not necessary but
-might be).
-
-Most profilers (including the profilers discussed below) will generate a call graph of
-functions for you. Your goal will be to find your function of interest in this call graph
-and then inspect the time spent inside of it. You might also want to to look at the
-annotated assembly which most profilers will provide you with.
-
-#### Instruments
-We will once again consider the scenario where you think you've identified a piece of code
-whose performance can be improved upon. Follow these steps to profile your code using
-Instruments.
-
-1. Open Instruments
-2. Select `Time Profiler` from the list of standard templates
-3. Close all other applications except for your instruments window and your terminal
-4. Run your benchmarking script from your terminal window
-    * You will want a benchmark that runs for at least a few seconds (5 seconds will
-    usually be long enough). This way the profiler will have something to work with
-    and you will have ample time to attach your profiler to this process:)
-    * I will just use benchzstd as my bencharmking script for this example:
-```
-$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
-```
-5. Once you run your benchmarking script, switch back over to instruments and attach your
-process to the time profiler. You can do this by:
-    * Clicking on the `All Processes` drop down in the top left of the toolbar.
-    * Selecting your process from the dropdown. In my case, it is just going to be labeled
-    `zstd`
-    * Hitting the bright red record circle button on the top left of the toolbar
-6. You profiler will now start collecting metrics from your benchmarking script. Once
-you think you have collected enough samples (usually this is the case after 3 seconds of
-recording), stop your profiler.
-7. Make sure that in toolbar of the bottom window, `profile` is selected.
-8. You should be able to see your call graph.
-    * If you don't see the call graph or an incomplete call graph, make sure you have compiled
-    zstd and your benchmarking script using debug flags. On mac and linux, this just means
-    you will have to supply the `-g` flag alone with your build script. You might also
-    have to provide the `-fno-omit-frame-pointer` flag
-9. Dig down the graph to find your function call and then inspect it by double clicking
-the list item. You will be able to see the annotated source code and the assembly side by
-side.
-
-#### Perf
-
-This wiki has a pretty detailed tutorial on getting started working with perf so we'll
-leave you to check that out of you're getting started:
-
-https://perf.wiki.kernel.org/index.php/Tutorial
-
-Some general notes on perf:
-* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
-counter statistics. Perf uses a high resolution timer and this is likely one
-of the first things your team will run when assessing your PR.
-* Perf has a long list of hardware counters that can be viewed with `perf --list`.
-When measuring optimizations, something worth trying is to make sure the hardware
-counters you expect to be impacted by your change are in fact being so. For example,
-if you expect the L1 cache misses to decrease with your change, you can look at the
-counter `L1-dcache-load-misses`
-* Perf hardware counters will not work on a virtual machine.
-
-#### Visual Studio
-
-TODO
-
-
-## Setting up continuous integration (CI) on your fork
-Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
-are well tested before they make it to an official release. Specifically, we use the platforms
-travis-ci, circle-ci, and appveyor.
-
-Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
-The easiest way to run these CI tests on your own before submitting a PR to our dev branch
-is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
-instructions for doing this.
-
-### travis-ci
-Follow these steps to link travis-ci with your github fork of zstd
-
-1. Make sure you are logged into your github account
-2. Go to https://travis-ci.org/
-3. Click 'Sign in with Github' on the top right
-4. Click 'Authorize travis-ci'
-5. Click 'Activate all repositories using Github Apps'
-6. Select 'Only select repositories' and select your fork of zstd from the drop down
-7. Click 'Approve and Install'
-8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
-9. Click 'Authorize travis-pro'
-10. You should have travis set up on your fork now.
-
-### circle-ci
-TODO
-
-### appveyor
-Follow these steps to link circle-ci with your github fork of zstd
-
-1. Make sure you are logged into your github account
-2. Go to https://www.appveyor.com/
-3. Click 'Sign in' on the top right
-4. Select 'Github' on the left panel
-5. Click 'Authorize appveyor'
-6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
-7. You should have appveyor set up on your fork now.
-
-### General notes on CI
-CI tests run every time a pull request (PR) is created or updated. The exact tests
-that get run will depend on the destination branch you specify. Some tests take
-longer to run than others. Currently, our CI is set up to run a short
-series of tests when creating a PR to the dev branch and a longer series of tests
-when creating a PR to the release branch. You can look in the configuration files
-of the respective CI platform for more information on what gets run when.
-
-Most people will just want to create a PR with the destination set to their local dev
-branch of zstd. You can then find the status of the tests on the PR's page. You can also
-re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
-
 ## Issues
 We use GitHub issues to track public bugs. Please ensure your description is
 clear and has sufficient instructions to be able to reproduce the issue.
@@ -398,106 +34,8 @@ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
 disclosure of security bugs. In those cases, please go through the process
 outlined on that page and do not file a public issue.
 
-## Coding Style
-It's a pretty long topic, which is difficult to summarize in a single paragraph.
-As a rule of thumbs, try to imitate the coding style of
-similar lines of codes around your contribution.
-The following is a non-exhaustive list of rules employed in zstd code base:
-
-### C90
-This code base is following strict C90 standard,
-with 2 extensions : 64-bit `long long` types, and variadic macros.
-This rule is applied strictly to code within `lib/` and `programs/`.
-Sub-project in `contrib/` are allowed to use other conventions.
-
-### C++ direct compatibility : symbol mangling
-All public symbol declarations must be wrapped in `extern “C” { … }`,
-so that this project can be compiled as C++98 code,
-and linked into C++ applications.
-
-### Minimal Frugal
-This design requirement is fundamental to preserve the portability of the code base.
-#### Dependencies
-- Reduce dependencies to the minimum possible level.
-  Any dependency should be considered “bad” by default,
-  and only tolerated because it provides a service in a better way than can be achieved locally.
-  The only external dependencies this repository tolerates are
-  standard C libraries, and in rare cases, system level headers.
-- Within `lib/`, this policy is even more drastic.
-  The only external dependencies allowed are `<assert.h>`, `<stdlib.h>`, `<string.h>`,
-  and even then, not directly.
-  In particular, no function shall ever allocate on heap directly,
-  and must use instead `ZSTD_malloc()` and equivalent.
-  Other accepted non-symbol headers are `<stddef.h>` and `<limits.h>`.
-- Within the project, there is a strict hierarchy of dependencies that must be respected.
-  `programs/` is allowed to depend on `lib/`, but only its public API.
-  Within `lib/`, `lib/common` doesn't depend on any other directory.
-  `lib/compress` and `lib/decompress` shall not depend on each other.
-  `lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`.
-#### Resources
-- Functions in `lib/` must use very little stack space,
-  several dozens of bytes max.
-  Everything larger must use the heap allocator,
-  or require a scratch buffer to be emplaced manually.
-
-### Naming
-* All public symbols are prefixed with `ZSTD_`
-  + private symbols, with a scope limited to their own unit, are free of this restriction.
-    However, since `libzstd` source code can be amalgamated,
-    each symbol name must attempt to be (and remain) unique.
-    Avoid too generic names that could become ground for future collisions.
-    This generally implies usage of some form of prefix.
-* For symbols (functions and variables), naming convention is `PREFIX_camelCase`.
-  + In some advanced cases, one can also find :
-    - `PREFIX_prefix2_camelCase`
-    - `PREFIX_camelCase_extendedQualifier`
-* Multi-words names generally consist of an action followed by object:
-  - for example : `ZSTD_createCCtx()`
-* Prefer positive actions
-  - `goBackward` rather than `notGoForward`
-* Type names (`struct`, etc.) follow similar convention,
-  except that they are allowed and even invited to start by an Uppercase letter.
-  Example : `ZSTD_CCtx`, `ZSTD_CDict`
-* Macro names are all Capital letters.
-  The same composition rules (`PREFIX_NAME_QUALIFIER`) apply.
-* File names are all lowercase letters.
-  The convention is `snake_case`.
-  File names **must** be unique across the entire code base,
-  even when they stand in clearly separated directories.
-
-### Qualifiers
-* This code base is `const` friendly, if not `const` fanatical.
-  Any variable that can be `const` (aka. read-only) **must** be `const`.
-  Any pointer which content will not be modified must be `const`.
-  This property is then controlled at compiler level.
-  `const` variables are an important signal to readers that this variable isn’t modified.
-  Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function.
-* If a function must be inlined, mention it explicitly,
-  using project's own portable macros, such as `FORCE_INLINE_ATTR`,
-  defined in `lib/common/compiler.h`.
-
-### Debugging
-* **Assertions** are welcome, and should be used very liberally,
-  to control any condition the code expects for its correct execution.
-  These assertion checks will be run in debug builds, and disabled in production.
-* For traces, this project provides its own debug macros,
-  in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`.
-
-### Code documentation
-* Avoid code documentation that merely repeats what the code is already stating.
-  Whenever applicable, prefer employing the code as the primary way to convey explanations.
-  Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`.
-  Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`.
-* At declaration level, the documentation explains how to use the function or variable
-  and when applicable why it's needed, of the scenarios where it can be useful.
-* At implementation level, the documentation explains the general outline of the algorithm employed,
-  and when applicable why this specific choice was preferred.
-
-### General layout
+## Coding Style  
 * 4 spaces for indentation rather than tabs
-* Code documentation shall directly precede function declaration or implementation
-* Function implementations and its code documentation should be preceded and followed by an empty line
-
 
 ## License
 By contributing to Zstandard, you agree that your contributions will be licensed

Makefile

@@ -1,16 +1,12 @@
 # ################################################################
-# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
 # in the COPYING file in the root directory of this source tree).
-# You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
-# verbose mode (print commands) on V=1 or VERBOSE=1
-Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
-
 PRGDIR   = programs
 ZSTDDIR  = lib
 BUILDIR  = build
@@ -21,20 +17,10 @@ FUZZDIR  = $(TESTDIR)/fuzz
 # Define nul output
 VOID = /dev/null
 
-# When cross-compiling from linux to windows, you might
-# need to specify this as "Windows." Fedora build fails
-# without it.
-#
-# Note: mingw-w64 build from linux to windows does not
-# fail on other tested distros (ubuntu, debian) even
-# without manually specifying the TARGET_SYSTEM.
-TARGET_SYSTEM ?= $(OS)
-CP ?= cp
-
-ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
-  EXT =.exe
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
 else
-  EXT =
+EXT =
 endif
 
 ## default: Build lib-release and zstd-release
@@ -50,27 +36,27 @@ allmost: allzstd zlibwrapper
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
 allzstd: lib
-	$(Q)$(MAKE) -C $(PRGDIR) all
-	$(Q)$(MAKE) -C $(TESTDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
 
 .PHONY: all32
 all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release lib-mt lib-nomt
-lib lib-release lib-mt lib-nomt:
-	$(Q)$(MAKE) -C $(ZSTDDIR) $@
+.PHONY: lib lib-release libzstd.a
+lib lib-release :
+	@$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
 zstd zstd-release:
-	$(Q)$(MAKE) -C $(PRGDIR) $@
-	$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) .
 
 .PHONY: zstdmt
 zstdmt:
-	$(Q)$(MAKE) -C $(PRGDIR) $@
-	$(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
 
 .PHONY: zlibwrapper
 zlibwrapper: lib
@@ -78,33 +64,24 @@ zlibwrapper: lib
 
 ## test: run long-duration tests
 .PHONY: test
-DEBUGLEVEL ?= 1
-test: MOREFLAGS += -g -Werror
+test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
 test:
-	DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
+	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
-	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
 
 ## shortest: same as `make check`
 .PHONY: shortest
 shortest:
-	$(Q)$(MAKE) -C $(TESTDIR) $@
+	$(MAKE) -C $(TESTDIR) $@
 
 ## check: run basic tests for `zstd` cli
 .PHONY: check
 check: shortest
 
-.PHONY: automated_benchmarking
-automated_benchmarking:
-	$(MAKE) -C $(TESTDIR) $@
-
-.PHONY: benchmarking
-benchmarking: automated_benchmarking
-
-## examples: build all examples in `examples/` directory
+## examples: build all examples in `/examples` directory
 .PHONY: examples
 examples: lib
-	$(MAKE) -C examples all
+	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
 
 ## manual: generate API documentation in html format
 .PHONY: manual
@@ -121,10 +98,8 @@ man:
 contrib: lib
 	$(MAKE) -C contrib/pzstd all
 	$(MAKE) -C contrib/seekable_format/examples all
-	$(MAKE) -C contrib/seekable_format/tests test
+	$(MAKE) -C contrib/adaptive-compression all
 	$(MAKE) -C contrib/largeNbDicts all
-	cd build/single_file_libs/ ; ./build_decoder_test.sh
-	cd build/single_file_libs/ ; ./build_library_test.sh
 
 .PHONY: cleanTabs
 cleanTabs:
@@ -132,35 +107,29 @@ cleanTabs:
 
 .PHONY: clean
 clean:
-	$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
-	$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
-	$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
-	$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	$(Q)$(MAKE) -C examples/ $@ > $(VOID)
-	$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
-	$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
-	$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
-	$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
-	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
-	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
-	$(Q)$(RM) -r lz4
+	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
+	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	@$(MAKE) -C examples/ $@ > $(VOID)
+	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
+	@$(MAKE) -C contrib/pzstd $@ > $(VOID)
+	@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
+	@$(MAKE) -C contrib/adaptive-compression $@ > $(VOID)
+	@$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
+	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	@$(RM) -r lz4
 	@echo Cleaning completed
 
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
 
 HOST_OS = POSIX
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
-MKDIR ?= mkdir -p
-
-HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
-EGREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
-EGREP_OPTIONS += --color=never
-endif
-EGREP = egrep $(EGREP_OPTIONS)
+EGREP = egrep --color=never
 
 # Print a two column output of targets and their description. To add a target description, put a
 # comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:
@@ -168,7 +137,7 @@ EGREP = egrep $(EGREP_OPTIONS)
 ## list: Print all targets and their descriptions (if provided)
 .PHONY: list
 list:
-	$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
+	@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
 		| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
 		| $(EGREP) -v  -e '^[^[:alnum:]]' | sort); \
 	{ \
@@ -181,36 +150,39 @@ list:
 	    done \
 	} | column -t -s $$'\t'
 
-.PHONY: install armtest usan asan uasan msan asan32
+.PHONY: install clangtest armtest usan asan uasan
 install:
-	$(Q)$(MAKE) -C $(ZSTDDIR) $@
-	$(Q)$(MAKE) -C $(PRGDIR) $@
+	@$(MAKE) -C $(ZSTDDIR) $@
+	@$(MAKE) -C $(PRGDIR) $@
 
 .PHONY: uninstall
 uninstall:
-	$(Q)$(MAKE) -C $(ZSTDDIR) $@
-	$(Q)$(MAKE) -C $(PRGDIR) $@
+	@$(MAKE) -C $(ZSTDDIR) $@
+	@$(MAKE) -C $(PRGDIR) $@
 
 .PHONY: travis-install
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
-.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
+.PHONY: gcc5build
 gcc5build: clean
 	gcc-5 -v
 	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: gcc6build
 gcc6build: clean
 	gcc-6 -v
 	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: gcc7build
 gcc7build: clean
 	gcc-7 -v
 	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
 
+.PHONY: clangbuild
 clangbuild: clean
 	clang -v
-	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
+	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 m32build: clean
 	gcc -v
@@ -220,15 +192,14 @@ armbuild: clean
 	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 aarch64build: clean
-	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 ppcbuild: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
 
 ppc64build: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
 
-.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
@@ -242,7 +213,7 @@ ppcfuzz: clean
 ppc64fuzz: clean
 	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
-.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
+.PHONY: cxxtest
 cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
 cxxtest: clean
 	$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)"   # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
@@ -255,6 +226,10 @@ gcc6test: clean
 	gcc-6 -v
 	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
 
+clangtest: clean
+	clang -v
+	$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
@@ -271,7 +246,6 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
 
-.PHONY: arm-ppc-compilation
 arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
@@ -287,26 +261,12 @@ uasanregressiontest:
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
 
-update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
-update_regressionResults:
-	$(MAKE) -C programs zstd
-	$(MAKE) -C tests/regression test
-	$(RM) -rf $(REGRESS_RESULTS_DIR)
-	$(MKDIR) $(REGRESS_RESULTS_DIR)
-	./tests/regression/test                         \
-        --cache  tests/regression/cache             \
-        --output $(REGRESS_RESULTS_DIR)/results.csv \
-        --zstd   programs/zstd
-	echo "Showing results differences"
-	! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv
-	echo "Updating results.csv"
-	$(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv
+# run UBsan with -fsanitize-recover=signed-integer-overflow
+# due to a bug in UBsan when doing pointer subtraction
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
 
-
-# run UBsan with -fsanitize-recover=pointer-overflow
-# this only works with recent compilers such as gcc 8+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
@@ -324,24 +284,21 @@ asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
 
-.PHONY: apt-install
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
 
-.PHONY: apt-add-repo
 apt-add-repo:
 	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 	sudo apt-get update -y -qq
 
-.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
 ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
 
@@ -376,47 +333,41 @@ lz4install:
 endif
 
 
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
-
 ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
+
 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
-	$(MKDIR) $(BUILDIR)/cmake/build
-	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
-	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
-	$(MAKE) -C $(BUILDIR)/cmake/build install;
-	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
-	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
+	mkdir $(BUILDIR)/cmake/build
+	cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
-c89build: clean
+c90build: clean
 	$(CC) -v
-	CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c90 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
 
 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
 
 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
 
 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
 
 bmix64build: clean
 	$(CC) -v
@@ -435,5 +386,5 @@ bmi32build: clean
 staticAnalyze: SCANBUILD ?= scan-build
 staticAnalyze:
 	$(CC) -v
-	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
+	CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
 endif

NEWS

@@ -0,0 +1,402 @@
+v1.3.7
+perf: slightly better decompression speed on clang (depending on hardware target)
+fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
+build: no longer build backtrace by default in release mode; restrict further automatic mode
+build: control backtrace support through build macro BACKTRACE
+misc: added man pages for zstdless and zstdgrep, by @samrussell
+
+v1.3.6
+perf: much faster dictionary builder, by @jenniferliu
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
+misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
+
+v1.3.5
+perf: much faster dictionary compression, by @felixhandte
+perf: small quality improvement for dictionary generation, by @terrelln
+perf: slightly improved high compression levels (notably level 19)
+mem : automatic memory release for long duration contexts
+cli : fix : overlapLog can be manually set
+cli : fix : decoding invalid lz4 frames
+api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
+api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
+build: select custom libzstd scope through control macros, by @GeorgeLu97
+build: OpenBSD patch, by @bket
+build: make and make all are compatible with -j
+doc : clarify zstd_compression_format.md, updated for IETF RFC process
+misc: pzstd compatible with reproducible compilation, by @lamby
+
+v1.3.4
+perf: faster speed (especially decoding speed) on recent cpus (haswell+)
+perf: much better performance associating --long with multi-threading, by @terrelln
+perf: better compression at levels 13-15
+cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
+cli : smoother status report in multi-threading mode
+cli : added command --fast=#, for faster compression modes
+cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
+api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
+api : compression levels can be negative, for even more speed
+api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
+api : ZSTDMT can accept new compression parameters during compression
+api : implemented all advanced dictionary decompression prototypes
+build: improved meson recipe, by Shawn Landden (@shawnl)
+build: VS2017 scripts, by @HaydnTrigg
+misc: all /contrib projects fixed
+misc: added /contrib/docker script by @gyscos
+
+v1.3.3
+perf: faster zstd_opt strategy (levels 16-19)
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+cli : fix : content size written in header by default
+cli : fix : improved LZ4 format support, by @felixhandte
+cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
+api : fix : support large skippable frames, by @terrelln
+api : fix : streaming interface was adding a useless 3-bytes null block to small frames
+api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
+build: fix : compilation under rhel6 and centos6, reported by @pixelb
+build: added `check` target
+
+v1.3.2
+new : long range mode, using --long command, by Stella Lau (@stellamplau)
+new : ability to generate and decode magicless frames (#591)
+changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
+fix : multi-threading compression works with custom allocators
+fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
+fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
+fix : 32-bits build can now decode large offsets (levels 21+)
+cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
+cli : improved --list output
+cli : new : can split input file for dictionary training, using command -B#
+cli : new : clean operation artefact on Ctrl-C interruption
+cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
+cli : fix : write file size in header in multiple-files mode
+api : added macro ZSTD_COMPRESSBOUND() for static allocation
+api : experimental : new advanced decompression API
+api : fix : sizeof_CCtx() used to over-estimate
+build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
+build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
+example : added streaming_memory_usage
+license : changed /examples license to BSD + GPLv2
+license : fix a few header files to reflect new license (#825)
+
+v1.3.1
+New license : BSD + GPLv2
+perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
+perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
+cli : improved and fixed --list command, by @ib (#772)
+cli : command -vV to list supported formats, by @ib (#771)
+build : fixed binary variants, reported by @svenha (#788)
+build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
+API exp : breaking change : ZSTD_getframeHeader() provides more information
+API exp : breaking change : pinned down values of error codes
+doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
+new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
+new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
+updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
+
+v1.3.0
+cli : new : `--list` command, by Paul Cruz
+cli : changed : xz/lzma support enabled by default
+cli : changed : `-t *` continue processing list after a decompression error
+API : added : ZSTD_versionString()
+API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : clarified memory estimation / measurement functions.
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
+tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
+v1.1.2
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
+API : zbuff : changed : prototypes now generate deprecation warnings
+lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
+lib : changed : only public ZSTD_ symbols are now exposed
+lib : changed : reduced usage  of stack memory
+lib : fixed : several corner case bugs, by Nick Terrell
+cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
+cli : new : preserve file attributes
+cli : new : added zstdless and zstdgrep tools
+cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
+cli : fixed : zstdcat
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+install : better compatibility with FreeBSD, by Dimitry Andric
+source tree : changed : zbuff source files moved to lib/deprecated
+
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
+Updated man page
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
+
+v1.1.0
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+added : NetBSD install target (#338)
+Improved : speed for batches of small files
+Improved : speed of zlib wrapper, by Przemyslaw Skibinski
+Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
+v0.8.0
+Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
+New : Build on FreeBSD and DragonFly, thanks to JrMarino
+Changed : modified API : ZSTD_compressEnd()
+Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
+Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
+Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
+Fixed : checksum correctly checked in single-pass mode
+Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
+Modified : minor compression level adaptations
+Updated : compression format specification to v0.2.0
+changed : zstd.h moved to /lib directory
+
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
+v0.7.4
+Added : homebrew for Mac, by Daniel Cade
+Added : more examples
+Fixed : segfault when using small dictionaries, reported by Felix Handte
+Modified : default compression level for CLI is now 3
+Updated : specification, to v0.1.1
+
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
+New : Command `--rm`, to remove source file after successful de/compression
+New : Visual build scripts, by Christophe Chevalier
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Frame checksum support
+New : Support pass-through mode (when using `-df`)
+API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
+API : create dictionary files from custom content, by Giuseppe Ottaviano
+API : support for custom malloc/free functions
+New : controllable Dictionary ID
+New : Support for skippable frames
+
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
+v0.6.0
+Stronger high compression modes, thanks to Przemyslaw Skibinski
+API : ZSTD_getFrameParams() provides size of decompressed content
+New : highest compression modes require `--ultra` command to fully unleash their capacity
+Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
+
+v0.5.1
+New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
+Changed : Dictionary builder integrated into libzstd and zstd cli
+Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
+Fix : high compression modes for big-endian platforms
+New : zstd cli : `-t` | `--test` command
+
+v0.5.0
+New : dictionary builder utility
+Changed : streaming & dictionary API
+Improved : better compression of small data
+
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
+v0.4.6
+fix : fast compression mode on Windows
+New : cmake configuration file, thanks to Artyom Dymchenko
+Improved : high compression mode on repetitive data
+New : block-level API
+New : ZSTD_duplicateCCtx()
+
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
+v0.4.0
+Command line utility compatible with high compression levels
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
+v0.3.6
+small blocks params
+
+v0.3.5
+minor generic compression improvements
+
+v0.3.4
+Faster fast cLevels
+
+v0.3.3
+Small compression ratio improvement
+
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
+v0.3
+HC mode : compression levels 2-26
+
+v0.2.2
+Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
+
+v0.2.1
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+
+v0.2.0
+**Breaking format change**
+Faster decompression speed
+Can still decode v0.1 format
+
+v0.1.3
+fix uninitialization warning, reported by Evan Nemerson
+
+v0.1.2
+frame concatenation support
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release

Package.swift

@@ -1,36 +0,0 @@
-// swift-tools-version:5.0
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
-    name: "zstd",
-    platforms: [
-        .macOS(.v10_10), .iOS(.v9), .tvOS(.v9)
-    ],
-    products: [
-        // Products define the executables and libraries a package produces, and make them visible to other packages.
-        .library(
-            name: "libzstd",
-            targets: [ "libzstd" ])
-    ],
-    dependencies: [
-        // Dependencies declare other packages that this package depends on.
-        // .package(url: /* package url */, from: "1.0.0"),
-    ],
-    targets: [
-        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
-        // Targets can depend on other targets in this package, and on products in packages this package depends on.
-        .target(
-            name: "libzstd",
-            path: "lib",
-            sources: [ "common", "compress", "decompress", "dictBuilder" ],
-            publicHeadersPath: ".",
-            cSettings: [
-                .headerSearchPath(".")
-            ])
-    ],
-    swiftLanguageVersions: [.v5],
-    cLanguageStandard: .gnu11,
-    cxxLanguageStandard: .gnucxx14
-)

README.md

@@ -4,38 +4,27 @@ __Zstandard__, or `zstd` as short version, is a fast lossless compression algori
 targeting real-time compression scenarios at zlib-level and better compression ratios.
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
 
-Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
-This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
+The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
-**Development branch status:**
+Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]
 
-[![Build Status][travisDevBadge]][travisLink]
-[![Build status][AppveyorDevBadge]][AppveyorLink]
-[![Build status][CircleDevBadge]][CircleLink]
-[![Build status][CirrusDevBadge]][CirrusLink]
-[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
-
-[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
-[travisLink]: https://travis-ci.com/facebook/zstd
+[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
+[travisLink]: https://travis-ci.org/facebook/zstd
 [AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd
-[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
-[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
-[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
-[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
 
-## Benchmarks
+### Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
-with a Core i7-9700K CPU @ 4.9GHz,
+on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
+with a Core i7-6700K CPU @ 4.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 9.3.0,
+compiled with [gcc] 7.3.0,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -44,24 +33,17 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
-| [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
-| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
-| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
-| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
-| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
-| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
-| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
+| **zstd 1.3.4 -1**       | 2.877 |   470 MB/s |  1380 MB/s |
+| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   400 MB/s |
+| brotli 1.0.2 -0         | 2.701 |   410 MB/s |   430 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
+| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
+| lz4 1.8.1               | 2.101 |   750 MB/s |  3700 MB/s |
+| snappy 1.1.4            | 2.091 |   530 MB/s |  1800 MB/s |
+| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
 
-[zlib]: http://www.zlib.net/
-[lz4]: http://www.lz4.org/
-
-The negative compression levels, specified with `--fast=#`,
-offer faster compression and decompression speed
-at the cost of compression ratio (compared to level 1).
+[zlib]:http://www.zlib.net/
+[LZ4]: http://www.lz4.org/
 
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
@@ -83,7 +65,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
 For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
 
 
-## The case for Small Data compression
+### The case for Small Data compression
 
 Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
 
@@ -107,24 +89,24 @@ Training works if there is some correlation in a family of small data samples. T
 Hence, deploying one dictionary per type of data will provide the greatest benefits.
 Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
 
-### Dictionary compression How To:
+#### Dictionary compression How To:
 
-1. Create the dictionary
+1) Create the dictionary
 
-   `zstd --train FullPathToTrainingSet/* -o dictionaryName`
+`zstd --train FullPathToTrainingSet/* -o dictionaryName`
 
-2. Compress with dictionary
+2) Compress with dictionary
 
-   `zstd -D dictionaryName FILE`
+`zstd -D dictionaryName FILE`
 
-3. Decompress with dictionary
+3) Decompress with dictionary
 
-   `zstd -D dictionaryName --decompress FILE.zst`
+`zstd -D dictionaryName --decompress FILE.zst`
 
 
-## Build instructions
+### Build instructions
 
-### Makefile
+#### Makefile
 
 If your system is compatible with standard `make` (or `gmake`),
 invoking `make` in root directory will generate `zstd` cli in root directory.
@@ -133,7 +115,7 @@ Other available options include:
 - `make install` : create and install zstd cli, library and man pages
 - `make check` : create and run `zstd`, tests its behavior on local platform
 
-### cmake
+#### cmake
 
 A `cmake` project generator is provided within `build/cmake`.
 It can generate Makefiles or other build scripts
@@ -141,29 +123,11 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
 By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 
-### Meson
+#### Meson
 
-A Meson project is provided within [`build/meson`](build/meson). Follow
-build instructions in that directory.
+A Meson project is provided within `contrib/meson`.
 
-You can also take a look at [`.travis.yml`](.travis.yml) file for an
-example about how Meson is used to build this project.
-
-Note that default build type is **release**.
-
-### VCPKG
-You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
-
-    git clone https://github.com/Microsoft/vcpkg.git
-    cd vcpkg
-    ./bootstrap-vcpkg.sh
-    ./vcpkg integrate install
-    ./vcpkg install zstd
-
-The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
-If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
-
-### Visual Studio (Windows)
+#### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities:
 - Projects for Visual Studio 2005, 2008 and 2010.
@@ -171,29 +135,19 @@ Going into `build` directory, you will find additional possibilities:
 - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
   which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
 
-### Buck
 
-You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
-The output binary will be in `buck-out/gen/programs/`.
-
-## Testing
-
-You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory.
-Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary.
-For information on CI testing, please refer to TESTING.md
-
-## Status
+### Status
 
 Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
 Zstandard is considered safe for production environments.
 
-## License
+### License
 
 Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
 
-## Contributing
+### Contributing
 
-The `dev` branch is the one where all contributions are merged before reaching `release`.
-If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
-Direct commit to `release` are not permitted.
+The "dev" branch is the one where all contributions are merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
+Direct commit to "master" are not permitted.
 For more information, please read [CONTRIBUTING](CONTRIBUTING.md).

TESTING.md

@@ -11,7 +11,7 @@ They consist of the following tests:
 - Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
 - Compilation on various versions of gcc, clang, and g++
 - `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
-- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
+- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
 
 Medium Tests
 ------------
@@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
 They consist of the following tests:
 - The following tests run with UBsan and Asan on x86_64 and x86, as well as with
   Msan on x86_64
-  - `tests/playTests.sh --test-large-data`
+  - `tests/playTests.sh --test-long-data`
   - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
 - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
 - Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
@@ -27,7 +27,7 @@ They consist of the following tests:
 
 Long Tests
 ----------
-Long tests run on all commits to `release` branch,
+Long tests run on all commits to `master` branch,
 and once a day on the current version of `dev` branch,
 on TravisCI.
 They consist of the following tests:
@@ -40,4 +40,5 @@ They consist of the following tests:
 - Versions test (ensuring `zstd` can decode files from all previous versions)
 - `pzstd` with asan and tsan, as well as in 32-bits mode
 - Testing `zstd` with legacy mode off
+- Testing `zbuff` (old streaming API)
 - Entire test suite and make install on macOS

appveyor.yml

@@ -1,20 +1,14 @@
-# Following tests are run _only_ on `release` branch
-# and on selected feature branch named `appveyorTest` or `visual*`
-
 -
   version: 1.0.{build}
   branches:
     only:
-    - release
     - master
-    - /appveyor*/
-    - /visual*/
   environment:
     matrix:
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "make allzstd MOREFLAGS=-static"
+      SCRIPT:   "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib"
       ARTIFACT: "true"
       BUILD:    "true"
     - COMPILER: "gcc"
@@ -23,15 +17,34 @@
       SCRIPT:   "make allzstd MOREFLAGS=-static"
       ARTIFACT: "true"
       BUILD:    "true"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
+      BUILD:    "true"
 
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   ""
+      TEST:     "cmake"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
       PLATFORM: "x64"
       CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
 
   install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@@ -73,9 +86,9 @@
           cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
           appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
           cp zstd.exe ..\bin\zstd.exe &&
-          git clone --depth 1 --branch release https://github.com/facebook/zstd &&
+          git clone --depth 1 --branch master https://github.com/facebook/zstd &&
           cd zstd &&
-          git archive --format=tar release -o zstd-src.tar &&
+          git archive --format=tar master -o zstd-src.tar &&
           ..\zstd -19 zstd-src.tar &&
           appveyor PushArtifact zstd-src.tar.zst &&
           certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
@@ -85,14 +98,55 @@
           appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
       )
     )
-  - if [%HOST%]==[cmake-visual] (
+  - if [%HOST%]==[visual] (
       ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
+      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
+      ECHO *** &&
+      msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
 
   test_script:
@@ -102,61 +156,66 @@
   - if [%TEST%]==[cmake] (
       mkdir build\cmake\build &&
       cd build\cmake\build &&
-      SET FUZZERTEST=-T2mn &&
-      SET ZSTREAM_TESTTIME=-T2mn &&
       cmake -G "Visual Studio 14 2015 Win64" .. &&
       cd ..\..\.. &&
       make clean
     )
-
-
-# The following tests are for regular pushes
-# into `dev` or some feature branch
-# There run less tests, for shorter feedback loop
+  - SET "FUZZERTEST=-T30s"
+  - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
+      CD tests &&
+      SET ZSTD=./zstd.exe &&
+      sh -e playTests.sh --test-large-data &&
+      fullbench.exe -i1 &&
+      fullbench.exe -i1 -P0 &&
+      fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
+    )
 
 -
   version: 1.0.{build}
   environment:
     matrix:
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Debug"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "x64"
-      CONFIGURATION: "Release"
-    - COMPILER: "visual"
-      HOST:     "visual"
-      PLATFORM: "Win32"
-      CONFIGURATION: "Release"
-
     - COMPILER: "gcc"
-      HOST:     "cygwin"
+      HOST:     "mingw"
       PLATFORM: "x64"
+      SCRIPT:   "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "CFLAGS=-Werror make -j allzstd"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
 
-    - COMPILER: "clang-cl"
-      HOST:     "cmake-visual"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
       PLATFORM: "x64"
       CONFIGURATION: "Release"
-      CMAKE_GENERATOR: "Visual Studio 15 2017"
-      CMAKE_GENERATOR_PLATFORM: "x64"
-      CMAKE_GENERATOR_TOOLSET: "LLVM"
-      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
 
   install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
   - SET PATH_ORIGINAL=%PATH%
-  - if [%HOST%]==[cygwin] (
-      ECHO Installing Cygwin Packages &&
-      C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
-        gcc,^
-        cmake,^
-        make
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
     )
   - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
       SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
@@ -164,42 +223,27 @@
 
   build_script:
   - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
-  - if [%HOST%]==[cygwin] (
-      set CHERE_INVOKING=yes &&
-      set CC=%COMPILER% &&
-      C:\cygwin64\bin\bash --login -c "
-        set -e;
-        cd build/cmake;
-        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .;
-        make VERBOSE=1 -j;
-        ctest -V -L Medium;
-      "
-    )
-  - if [%HOST%]==[cmake-visual] (
-      ECHO *** &&
-      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
-      PUSHD build\cmake &&
-      cmake -DBUILD_TESTING=ON . &&
-      cmake --build . --config %CONFIGURATION% -j4 &&
-      POPD &&
-      ECHO ***
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) ) &&
+      make -v &&
+      sh -c "%COMPILER% -v" &&
+      set "CC=%COMPILER%" &&
+      sh -c "%SCRIPT%"
     )
   - if [%HOST%]==[visual] (
       ECHO *** &&
-      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
-      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
-
-
-  test_script:
-  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET "FUZZERTEST=-T10s"
-  - if [%HOST%]==[mingw] (
-      set "CC=%COMPILER%" &&
-      make clean &&
-      make check
-    )

build/.gitignore

@@ -29,5 +29,3 @@ compile_commands.json
 CTestTestfile.cmake
 build
 lib
-!cmake/lib
-!meson/lib

build/VS2008/fullbench/fullbench.vcproj

@@ -364,18 +364,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
 				>
@@ -392,14 +380,6 @@
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
 				>
@@ -409,21 +389,13 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\benchfn.c"
+				RelativePath="..\..\..\programs\bench.c"
 				>
 			</File>
 			<File
 				RelativePath="..\..\..\tests\fullbench.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\programs\util.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\programs\timefn.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -463,7 +435,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -506,22 +478,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/fuzzer/fuzzer.vcproj

@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -328,14 +328,6 @@
 			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
-			<File
-				RelativePath="..\..\..\programs\util.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\programs\timefn.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\programs\datagen.c"
 				>
@@ -412,30 +404,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -483,7 +455,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -511,7 +483,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File
@@ -542,22 +514,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/zstd/zstd.vcproj

@@ -333,19 +333,7 @@
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
 			<File
-				RelativePath="..\..\..\programs\util.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\programs\timefn.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\programs\benchfn.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\programs\benchzstd.c"
+				RelativePath="..\..\..\programs\bench.c"
 				>
 			</File>
 			<File
@@ -424,30 +412,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\legacy\zstd_v01.c"
 				>
@@ -480,10 +448,6 @@
 				RelativePath="..\..\..\programs\zstdcli.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\programs\zstdcli_trace.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
 				>
@@ -559,7 +523,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File
@@ -575,7 +539,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -626,22 +590,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/zstdlib/zstdlib.vcproj

@@ -396,18 +396,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -432,14 +420,6 @@
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\legacy\zstd_v01.c"
 				>
@@ -495,7 +475,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -523,7 +503,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File
@@ -554,22 +534,6 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
-				>
-			</File>
-			<File
-				RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2010/datagen/datagen.vcxproj

@@ -155,7 +155,6 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\tests\datagencli.c" />
   </ItemGroup>
@@ -165,4 +164,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -166,16 +166,14 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
-    <ClCompile Include="..\..\..\programs\util.c" />
-    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\benchfn.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
-    <ClInclude Include="..\..\..\programs\benchfn.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <ItemGroup>
@@ -186,4 +184,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

build/VS2010/fullbench/fullbench.vcxproj

@@ -167,9 +167,6 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
@@ -178,39 +175,30 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
-    <ClCompile Include="..\..\..\programs\util.c" />
-    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\benchfn.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\common\pool.h" />
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
-    <ClInclude Include="..\..\..\programs\benchfn.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -90,7 +90,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -105,7 +105,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -122,7 +122,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <TreatWarningAsError>false</TreatWarningAsError>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -142,7 +142,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -167,9 +167,6 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -178,14 +175,10 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
-    <ClCompile Include="..\..\..\programs\util.c" />
-    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\tests\fuzzer.c" />
   </ItemGroup>
@@ -196,22 +189,17 @@
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
-    <ClInclude Include="..\..\..\lib\zdict.h" />
+    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -42,8 +39,9 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -61,11 +59,12 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\mem.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
+    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -77,17 +76,12 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="libzstd-dll.rc" />
@@ -212,6 +206,7 @@
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -236,6 +231,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <OmitFramePointers>true</OmitFramePointers>
+      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/VS2010/libzstd/libzstd.vcxproj

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -42,8 +39,9 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -61,11 +59,12 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\common\mem.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
+    <ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -77,42 +76,42 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>
     <RootNamespace>libzstd</RootNamespace>
-    <TargetName>libzstd_static</TargetName>
-    <CharacterSet>MultiByte</CharacterSet>
-    <ConfigurationType>StaticLibrary</ConfigurationType>
     <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
     <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -132,36 +131,35 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
+    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
+    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
+    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
+    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
-  <ItemDefinitionGroup>
-    <ClCompile>
-      <ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
-    </ClCompile>
-  </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <PrecompiledHeader>
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -181,7 +179,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -201,10 +199,11 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -222,13 +221,14 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <OmitFramePointers>true</OmitFramePointers>
+      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/VS2010/zstd/zstd.vcxproj

@@ -32,9 +32,6 @@
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -42,8 +39,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
-    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -55,40 +50,31 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
-    <ClCompile Include="..\..\..\programs\util.c" />
-    <ClCompile Include="..\..\..\programs\timefn.c" />
-    <ClCompile Include="..\..\..\programs\benchfn.c" />
-    <ClCompile Include="..\..\..\programs\benchzstd.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
-    <ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\common\pool.h" />
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
-    <ClInclude Include="..\..\..\lib\zdict.h" />
+    <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
-    <ClInclude Include="..\..\..\lib\zstd_errors.h" />
+    <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
-    <ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
-    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -97,7 +83,7 @@
     <ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
-    <ClInclude Include="..\..\..\programs\benchzstd.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\dibio.h" />
     <ClInclude Include="..\..\..\programs\fileio.h" />

build/VS_scripts/build.generic.cmd

@@ -19,10 +19,10 @@ GOTO build
 :display_help
 
 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
+echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
-echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
+echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141)
 
 EXIT /B 1
 
@@ -43,16 +43,6 @@ IF %msbuild_version% == VS2017 (
 	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
 )
 
-:: VS2019
-SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
-SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2019 (
-	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
-	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
-	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
-)
-
 SET project="%~p0\..\VS2010\zstd.sln"
 
 SET msbuild_params=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%msbuild_platform% /p:Configuration=%msbuild_configuration%

build/cmake/CMakeLists.txt

@@ -7,203 +7,104 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
-  
-# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. 
-# Set and use the newest cmake policies that are validated to work 
-set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") 
-set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
-if("${CMAKE_MAJOR_VERSION}" LESS 3)
-  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND 
-        "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
-    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
-else() 
-    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") 
-endif()
-cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
+PROJECT(zstd)
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
+SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
 
-set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
+# Ensure Release build even if not invoked via Makefile
+SET(CMAKE_BUILD_TYPE "Release")
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
-set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
-set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-# Parse version
-include(GetZstdLibraryVersion)
-GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
-
-if( CMAKE_MAJOR_VERSION LESS 3 )
-  ## Provide cmake 3+ behavior for older versions of cmake
-  project(zstd)
-  set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
-  set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
-  set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
-  set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
-  enable_language(C)   # Main library is in C
-  enable_language(CXX) # Testing contributed code also utilizes CXX
-else()
-  project(zstd
-    VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
-    LANGUAGES C   # Main library is in C
-              CXX # Testing contributed code also utilizes CXX
-    )
-endif()
-message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
-set(zstd_HOMEPAGE_URL "http://www.zstd.net")
-set(zstd_DESCRIPTION  "Zstandard is a real-time compression algorithm, providing high compression ratios.")
-
-# Set a default build type if none was specified
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-  message(STATUS "Setting build type to 'Release' as none was specified.")
-  set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
-  # Set the possible values of build type for cmake-gui
-  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
-endif()
-
-include(GNUInstallDirs)
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
+INCLUDE(GNUInstallDirs)
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags
 #-----------------------------------------------------------------------------
-include(AddZstdCompilationFlags)
+INCLUDE(AddZstdCompilationFlags)
 ADD_ZSTD_COMPILATION_FLAGS()
 
 # Always hide XXHash symbols
-add_definitions(-DXXH_NAMESPACE=ZSTD_)
+ADD_DEFINITIONS(-DXXH_NAMESPACE=ZSTD_)
 
 #-----------------------------------------------------------------------------
 # Installation variables
 #-----------------------------------------------------------------------------
-message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
-message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
+MESSAGE(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
+MESSAGE(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
 
 #-----------------------------------------------------------------------------
 # Options
 #-----------------------------------------------------------------------------
 
 # Legacy support
-option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
 
-if (ZSTD_LEGACY_SUPPORT)
-    message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=5)
-else ()
-    message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
-endif ()
-
-if (ANDROID)
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
-else()
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
-endif()
+IF (ZSTD_LEGACY_SUPPORT)
+    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
+ELSE (ZSTD_LEGACY_SUPPORT)
+    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
+ENDIF (ZSTD_LEGACY_SUPPORT)
 
 # Multi-threading support
-option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
+OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
 
-if (ZSTD_MULTITHREAD_SUPPORT)
-    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
-else ()
-    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
-endif ()
+IF (ZSTD_MULTITHREAD_SUPPORT)
+    MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
+ELSE (ZSTD_MULTITHREAD_SUPPORT)
+    MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
+ENDIF (ZSTD_MULTITHREAD_SUPPORT)
 
-option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
-option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
-
-# Respect the conventional CMake option for enabling tests if it was specified on the first configure
-if (BUILD_TESTING)
-    set(ZSTD_BUILD_TESTS_default ON)
-else()
-    set(ZSTD_BUILD_TESTS_default OFF)
-endif()
-option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
+OPTION(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
+OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
+OPTION(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
 if (MSVC)
-    option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
+    OPTION(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
 endif ()
 
 #-----------------------------------------------------------------------------
 # External dependencies
 #-----------------------------------------------------------------------------
-if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
-    set(THREADS_PREFER_PTHREAD_FLAG ON)
-    find_package(Threads REQUIRED)
-    if(CMAKE_USE_PTHREADS_INIT)
-        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
-    else()
-        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-    endif()
-endif ()
+IF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
+    SET(THREADS_PREFER_PTHREAD_FLAG ON)
+    FIND_PACKAGE(Threads REQUIRED)
+    IF(CMAKE_USE_PTHREADS_INIT)
+        SET(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+    ELSE()
+        MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+    ENDIF()
+ENDIF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
 
 #-----------------------------------------------------------------------------
 # Add source directories
 #-----------------------------------------------------------------------------
-add_subdirectory(lib)
+ADD_SUBDIRECTORY(lib)
 
-option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
+IF (ZSTD_BUILD_PROGRAMS)
+    IF (NOT ZSTD_BUILD_STATIC)
+        MESSAGE(SEND_ERROR "You need to build static library to build zstd CLI")
+    ENDIF (NOT ZSTD_BUILD_STATIC)
 
-if (ZSTD_BUILD_PROGRAMS)
-    if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "You need to build static library to build zstd CLI")
-    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "You need to build shared library to build zstd CLI")
-    endif ()
+    ADD_SUBDIRECTORY(programs)
+ENDIF (ZSTD_BUILD_PROGRAMS)
 
-    add_subdirectory(programs)
-endif ()
+IF (ZSTD_BUILD_TESTS)
+    IF (NOT ZSTD_BUILD_STATIC)
+        MESSAGE(SEND_ERROR "You need to build static library to build tests")
+    ENDIF (NOT ZSTD_BUILD_STATIC)
 
-if (ZSTD_BUILD_TESTS)
-    enable_testing()
-    if (NOT ZSTD_BUILD_STATIC)
-        message(SEND_ERROR "You need to build static library to build tests")
-    endif ()
+    ADD_SUBDIRECTORY(tests)
+ENDIF (ZSTD_BUILD_TESTS)
 
-    add_subdirectory(tests)
-endif ()
-
-if (ZSTD_BUILD_CONTRIB)
-    add_subdirectory(contrib)
-endif ()
+IF (ZSTD_BUILD_CONTRIB)
+    ADD_SUBDIRECTORY(contrib)
+ENDIF (ZSTD_BUILD_CONTRIB)
 
 #-----------------------------------------------------------------------------
 # Add clean-all target
 #-----------------------------------------------------------------------------
-add_custom_target(clean-all
+ADD_CUSTOM_TARGET(clean-all
    COMMAND ${CMAKE_BUILD_TOOL} clean
    COMMAND rm -rf ${CMAKE_BINARY_DIR}/
 )
-
-#-----------------------------------------------------------------------------
-# Generate Package Config files
-#
-# This section is based on the boiler plate code from:
-# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
-#-----------------------------------------------------------------------------
-include(CMakePackageConfigHelpers)
-write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    VERSION ${zstd_VERSION}
-    COMPATIBILITY SameMajorVersion
-    )
-
-# A Package Config file that works from the build directory
-export(EXPORT zstdExports
-    FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
-    NAMESPACE zstd::
-    )
-configure_file(zstdConfig.cmake
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
-    COPYONLY
-    )
-
-# A Package Config file that works from the installation directory
-set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
-install(EXPORT zstdExports
-    FILE zstdTargets.cmake
-    NAMESPACE zstd::
-    DESTINATION ${ConfigPackageLocation}
-    )
-install(FILES
-    zstdConfig.cmake
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    DESTINATION ${ConfigPackageLocation}
-    )

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -20,28 +20,19 @@ function(EnableCompilerFlag _flag _C _CXX)
     endif ()
 endfunction()
 
-macro(ADD_ZSTD_COMPILATION_FLAGS)
+MACRO(ADD_ZSTD_COMPILATION_FLAGS)
     if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         #Set c++11 by default
         EnableCompilerFlag("-std=c++11" false true)
         #Set c99 by default
         EnableCompilerFlag("-std=c99" true false)
-        if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
-            # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" true true)
-        else ()
-            EnableCompilerFlag("-Wall" true true)
-        endif ()
+        EnableCompilerFlag("-Wall" true true)
         EnableCompilerFlag("-Wextra" true true)
         EnableCompilerFlag("-Wundef" true true)
         EnableCompilerFlag("-Wshadow" true true)
         EnableCompilerFlag("-Wcast-align" true true)
         EnableCompilerFlag("-Wcast-qual" true true)
         EnableCompilerFlag("-Wstrict-prototypes" true false)
-        # Enable asserts in Debug mode
-        if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
-        endif ()
     elseif (MSVC) # Add specific compilation flags for Windows Visual
 
         set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
@@ -52,32 +43,25 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
         # UNICODE SUPPORT
         EnableCompilerFlag("/D_UNICODE" true true)
         EnableCompilerFlag("/DUNICODE" true true)
-        # Enable asserts in Debug mode
-        if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
-        endif ()
     endif ()
 
     # Remove duplicates compilation flags
-    foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+    FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
              CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
              CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
              CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-        if( ${flag_var} )
-            separate_arguments(${flag_var})
-            string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
-        endif()
-    endforeach ()
+        separate_arguments(${flag_var})
+        list(REMOVE_DUPLICATES ${flag_var})
+        string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
+    ENDFOREACH (flag_var)
 
     if (MSVC AND ZSTD_USE_STATIC_RUNTIME)
-        foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+        FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
                  CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
                  CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
                  CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-            if ( ${flag_var} )
-                string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
-            endif()
-        endforeach ()
+            STRING(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+        ENDFOREACH (flag_var)
     endif ()
 
-endmacro()
+ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)

build/cmake/CMakeModules/FindLibLZ4.cmake

@@ -1,49 +0,0 @@
-# Find LibLZ4
-#
-# Find LibLZ4 headers and library
-#
-#   Result Variables
-#
-#   LIBLZ4_FOUND             - True if lz4 is found
-#   LIBLZ4_INCLUDE_DIRS      - lz4 headers directories
-#   LIBLZ4_LIBRARIES         - lz4 libraries
-#   LIBLZ4_VERSION_MAJOR     - The major version of lz4
-#   LIBLZ4_VERSION_MINOR     - The minor version of lz4
-#   LIBLZ4_VERSION_RELEASE   - The release version of lz4
-#   LIBLZ4_VERSION_STRING    - version number string (e.g. 1.8.3)
-#
-#   Hints
-#
-#   Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library
-
-set(_LIBLZ4_ROOT_HINTS
-    ENV LZ4_ROOT_DIR)
-
-find_path(  LIBLZ4_INCLUDE_DIR lz4.h
-            HINTS ${_LIBLZ4_ROOT_HINTS})
-find_library(   LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
-                HINTS ${_LIBLZ4_ROOT_HINTS})
-
-if(LIBLZ4_INCLUDE_DIR)
-    file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")
-
-    string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
-    string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
-    string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")
-
-    set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
-    unset(LIBLZ4_HEADER_CONTENT)
-endif()
-
-include(FindPackageHandleStandardArgs)
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4    REQUIRED_VARS   LIBLZ4_INCLUDE_DIR
-                                                            LIBLZ4_LIBRARY
-                                            VERSION_VAR     LIBLZ4_VERSION_STRING
-                                            FAIL_MESSAGE    "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")
-
-if (LIBLZ4_FOUND)
-    set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
-    set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
-endif ()
-
-mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )

build/cmake/CMakeModules/GetZstdLibraryVersion.cmake

@@ -1,10 +1,9 @@
-function(GetZstdLibraryVersion _header _major _minor _patch)
+function(GetZstdLibraryVersion _header _major _minor _release)
     # Read file content
-    file(READ ${_header} CONTENT)
+    FILE(READ ${_header} CONTENT)
 
     string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}")
-    set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
-    set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
-    set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE)
+    SET(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
+    SET(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
+    SET(${_release} ${CMAKE_MATCH_3} PARENT_SCOPE)
 endfunction()
-

build/cmake/README.md

@@ -1,104 +0,0 @@
-# Cmake contributions
-
-Contributions to the cmake build configurations are welcome. Please
-use case sensitivity that matches modern (ie. cmake version 2.6 and above)
-conventions of using lower-case for commands, and upper-case for
-variables.
-
-## How to build
-
-As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
-To do this, you can create a new directory and build in it:
-```sh
-cd build/cmake
-mkdir builddir
-cd builddir
-cmake ..
-make
-```
-Then you can clean all cmake caches by simply delete the new directory:
-```sh
-rm -rf build/cmake/builddir
-```
-
-And of course, you can directly build in build/cmake:
-```sh
-cd build/cmake
-cmake
-make
-```
-
-To show cmake build options, you can:
-```sh
-cd build/cmake/builddir
-cmake -LH ..
-```
-
-Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
-```sh
-cd build/cmake/builddir
-cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
-make
-```
-
-### referring
-[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
-
-## CMake Style Recommendations
-
-### Indent all code correctly, i.e. the body of
-
- * if/else/endif
- * foreach/endforeach
- * while/endwhile
- * macro/endmacro
- * function/endfunction
-
-Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
-spaces for indenting as is used in the rest of the file. Do not use tabs.
-
-### Upper/lower casing
-
-Most important: use consistent upper- or lowercasing within one file !
-
-In general, the all-lowercase style is preferred.
-
-So, this is recommended:
-
-```
-add_executable(foo foo.c)
-```
-
-These forms are discouraged
-
-```
-ADD_EXECUTABLE(bar bar.c)
-Add_Executable(hello hello.c)
-aDd_ExEcUtAbLe(blub blub.c)
-```
-
-### End commands
-To make the code easier to read, use empty commands for endforeach(), endif(),
-endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
-
-For example, do this:
-
-```
-if(FOOVAR)
-   some_command(...)
-else()
-   another_command(...)
-endif()
-```
-
-and not this:
-
-```
-if(BARVAR)
-   some_other_command(...)
-endif(BARVAR)
-```
-
-### Other resources for best practices
-
-https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules

build/cmake/contrib/CMakeLists.txt

@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(contrib)
+PROJECT(contrib)
 
-add_subdirectory(pzstd)
-add_subdirectory(gen_html)
+ADD_SUBDIRECTORY(pzstd)
+ADD_SUBDIRECTORY(gen_html)

build/cmake/contrib/gen_html/CMakeLists.txt

@@ -7,24 +7,24 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(gen_html)
-include(GetZstdLibraryVersion)
+PROJECT(gen_html)
+INCLUDE(GetZstdLibraryVersion)
 
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
-set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
-include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
+SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+SET(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
+SET(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
 
-add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp)
+ADD_EXECUTABLE(gen_html ${GENHTML_DIR}/gen_html.cpp)
 
 GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
-set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
-add_custom_target(zstd_manual.html ALL
+SET(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
+ADD_CUSTOM_TARGET(zstd_manual.html ALL
                   ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
                   DEPENDS gen_html COMMENT "Update zstd manual")
 
-install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
+INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -7,32 +7,26 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(pzstd)
+PROJECT(pzstd)
 
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
-include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
+SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
 
-add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
-set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
-set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
+ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
+SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 
-if (ZSTD_BUILD_SHARED)
-    set(ZSTD_LIB libzstd_shared)
-else()
-    set(ZSTD_LIB libzstd_static)
-endif()
+SET(THREADS_PREFER_PTHREAD_FLAG ON)
+FIND_PACKAGE(Threads REQUIRED)
+IF (CMAKE_USE_PTHREADS_INIT)
+    TARGET_LINK_LIBRARIES(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
+ELSE()
+    MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+ENDIF()
 
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
-if (CMAKE_USE_PTHREADS_INIT)
-  target_link_libraries(pzstd ${ZSTD_LIB} ${CMAKE_THREAD_LIBS_INIT})
-else()
-    message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-endif()
-
-install(TARGETS pzstd RUNTIME DESTINATION "bin")
+INSTALL(TARGETS pzstd RUNTIME DESTINATION "bin")

build/cmake/lib/CMakeLists.txt

@@ -7,47 +7,82 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(libzstd C ASM)
+PROJECT(libzstd)
 
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
-option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
-option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
+OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
 
-if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
-    message(SEND_ERROR "You need to build at least one flavor of libzstd")
-endif()
+IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
+    MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd")
+ENDIF()
 
 # Define library directory, where sources and header files are located
-include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
+SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
-file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
-file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
-file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
-file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
+# Parse version
+INCLUDE(GetZstdLibraryVersion)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
+MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
 
-set(Sources
-        ${CommonSources}
-        ${CompressSources}
-        ${DecompressSources}
-        ${DictBuilderSources})
+SET(Sources
+        ${LIBRARY_DIR}/common/entropy_common.c
+        ${LIBRARY_DIR}/common/fse_decompress.c
+        ${LIBRARY_DIR}/common/threading.c
+        ${LIBRARY_DIR}/common/pool.c
+        ${LIBRARY_DIR}/common/zstd_common.c
+        ${LIBRARY_DIR}/common/error_private.c
+        ${LIBRARY_DIR}/common/xxhash.c
+        ${LIBRARY_DIR}/compress/hist.c
+        ${LIBRARY_DIR}/compress/fse_compress.c
+        ${LIBRARY_DIR}/compress/huf_compress.c
+        ${LIBRARY_DIR}/compress/zstd_compress.c
+        ${LIBRARY_DIR}/compress/zstdmt_compress.c
+        ${LIBRARY_DIR}/compress/zstd_fast.c
+        ${LIBRARY_DIR}/compress/zstd_double_fast.c
+        ${LIBRARY_DIR}/compress/zstd_lazy.c
+        ${LIBRARY_DIR}/compress/zstd_opt.c
+        ${LIBRARY_DIR}/compress/zstd_ldm.c
+        ${LIBRARY_DIR}/decompress/huf_decompress.c
+        ${LIBRARY_DIR}/decompress/zstd_decompress.c
+        ${LIBRARY_DIR}/dictBuilder/cover.c
+        ${LIBRARY_DIR}/dictBuilder/fastcover.c
+        ${LIBRARY_DIR}/dictBuilder/divsufsort.c
+        ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/deprecated/zbuff_common.c
+        ${LIBRARY_DIR}/deprecated/zbuff_compress.c
+        ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
 
-file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
-file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
-file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
-file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
-
-set(Headers
+SET(Headers
         ${LIBRARY_DIR}/zstd.h
-        ${CommonHeaders}
-        ${CompressHeaders}
-        ${DecompressHeaders}
-        ${DictBuilderHeaders})
+        ${LIBRARY_DIR}/common/debug.h
+        ${LIBRARY_DIR}/common/pool.h
+        ${LIBRARY_DIR}/common/threading.h
+        ${LIBRARY_DIR}/common/bitstream.h
+        ${LIBRARY_DIR}/common/error_private.h
+        ${LIBRARY_DIR}/common/zstd_errors.h
+        ${LIBRARY_DIR}/common/fse.h
+        ${LIBRARY_DIR}/common/huf.h
+        ${LIBRARY_DIR}/common/mem.h
+        ${LIBRARY_DIR}/common/zstd_internal.h
+        ${LIBRARY_DIR}/compress/hist.h
+        ${LIBRARY_DIR}/compress/zstd_compress_internal.h
+        ${LIBRARY_DIR}/compress/zstd_fast.h
+        ${LIBRARY_DIR}/compress/zstd_double_fast.h
+        ${LIBRARY_DIR}/compress/zstd_lazy.h
+        ${LIBRARY_DIR}/compress/zstd_opt.h
+        ${LIBRARY_DIR}/compress/zstd_ldm.h
+        ${LIBRARY_DIR}/compress/zstdmt_compress.h
+        ${LIBRARY_DIR}/dictBuilder/zdict.h
+        ${LIBRARY_DIR}/dictBuilder/cover.h
+        ${LIBRARY_DIR}/deprecated/zbuff.h)
 
-if (ZSTD_LEGACY_SUPPORT)
-    set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
-    include_directories(${LIBRARY_LEGACY_DIR})
+IF (ZSTD_LEGACY_SUPPORT)
+    SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
+    INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
 
-    set(Sources ${Sources}
+    SET(Sources ${Sources}
             ${LIBRARY_LEGACY_DIR}/zstd_v01.c
             ${LIBRARY_LEGACY_DIR}/zstd_v02.c
             ${LIBRARY_LEGACY_DIR}/zstd_v03.c
@@ -56,7 +91,7 @@ if (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v06.c
             ${LIBRARY_LEGACY_DIR}/zstd_v07.c)
 
-    set(Headers ${Headers}
+    SET(Headers ${Headers}
             ${LIBRARY_LEGACY_DIR}/zstd_legacy.h
             ${LIBRARY_LEGACY_DIR}/zstd_v01.h
             ${LIBRARY_LEGACY_DIR}/zstd_v02.h
@@ -65,119 +100,104 @@ if (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v05.h
             ${LIBRARY_LEGACY_DIR}/zstd_v06.h
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
-endif ()
+ENDIF (ZSTD_LEGACY_SUPPORT)
 
-if (MSVC)
-    set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
-    set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
-endif ()
+IF (MSVC)
+    SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
+    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
+ENDIF (MSVC)
 
 # Split project to static and shared libraries build
-set(library_targets)
-if (ZSTD_BUILD_SHARED)
-    add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
-    list(APPEND library_targets libzstd_shared)
-    if (ZSTD_MULTITHREAD_SUPPORT)
-        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
-        if (UNIX)
-            target_link_libraries(libzstd_shared ${THREADS_LIBS})
-        endif ()
-    endif()
-endif ()
-if (ZSTD_BUILD_STATIC)
-    add_library(libzstd_static STATIC ${Sources} ${Headers})
-    list(APPEND library_targets libzstd_static)
-    if (ZSTD_MULTITHREAD_SUPPORT)
-        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
-        if (UNIX)
-            target_link_libraries(libzstd_static ${THREADS_LIBS})
-        endif ()
-    endif ()
-endif ()
+IF (ZSTD_BUILD_SHARED)
+    ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
+    IF (ZSTD_MULTITHREAD_SUPPORT)
+        SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        IF (UNIX)
+            TARGET_LINK_LIBRARIES(libzstd_shared ${THREADS_LIBS})
+        ENDIF ()
+    ENDIF()
+ENDIF (ZSTD_BUILD_SHARED)
+IF (ZSTD_BUILD_STATIC)
+    ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
+    IF (ZSTD_MULTITHREAD_SUPPORT)
+        SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        IF (UNIX)
+            TARGET_LINK_LIBRARIES(libzstd_static ${THREADS_LIBS})
+        ENDIF ()
+    ENDIF ()
+ENDIF (ZSTD_BUILD_STATIC)
 
 # Add specific compile definitions for MSVC project
-if (MSVC)
-    if (ZSTD_BUILD_SHARED)
-        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
-    endif ()
-    if (ZSTD_BUILD_STATIC)
-        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
-    endif ()
-endif ()
+IF (MSVC)
+    IF (ZSTD_BUILD_SHARED)
+        SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
+    ENDIF (ZSTD_BUILD_SHARED)
+    IF (ZSTD_BUILD_STATIC)
+        SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
+    ENDIF (ZSTD_BUILD_STATIC)
+ENDIF (MSVC)
 
 # With MSVC static library needs to be renamed to avoid conflict with import library
-if (MSVC OR (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT MINGW))
-    set(STATIC_LIBRARY_BASE_NAME zstd_static)
-else ()
-    set(STATIC_LIBRARY_BASE_NAME zstd)
-endif ()
+IF (MSVC)
+    SET(STATIC_LIBRARY_BASE_NAME zstd_static)
+ELSE ()
+    SET(STATIC_LIBRARY_BASE_NAME zstd)
+ENDIF (MSVC)
 
 # Define static and shared library names
-if (ZSTD_BUILD_SHARED)
-    set_target_properties(
+IF (ZSTD_BUILD_SHARED)
+    SET_TARGET_PROPERTIES(
             libzstd_shared
             PROPERTIES
             OUTPUT_NAME zstd
-            VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
-            SOVERSION ${zstd_VERSION_MAJOR})
-endif ()
+            SOVERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE})
+ENDIF (ZSTD_BUILD_SHARED)
 
-if (ZSTD_BUILD_STATIC)
-    set_target_properties(
+IF (ZSTD_BUILD_STATIC)
+    SET_TARGET_PROPERTIES(
             libzstd_static
             PROPERTIES
-            POSITION_INDEPENDENT_CODE On
             OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
-endif ()
+ENDIF (ZSTD_BUILD_STATIC)
 
-if (UNIX OR MINGW)
+IF (UNIX)
     # pkg-config
-    set(PREFIX "${CMAKE_INSTALL_PREFIX}")
-    set(EXEC_PREFIX "\${prefix}")
-    set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
-    set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
-    set(VERSION "${zstd_VERSION}")
+    SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
+    SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
+    SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
+    ADD_CUSTOM_TARGET(libzstd.pc ALL
+            ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
+            -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
+            -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
+            COMMENT "Creating pkg-config file")
 
-    string(LENGTH "${PREFIX}" PREFIX_LENGTH)
-    string(SUBSTRING "${LIBDIR}" 0 ${PREFIX_LENGTH} LIBDIR_PREFIX)
-    string(SUBSTRING "${LIBDIR}" ${PREFIX_LENGTH} -1 LIBDIR_SUFFIX)
-    string(SUBSTRING "${INCLUDEDIR}" 0 ${PREFIX_LENGTH} INCLUDEDIR_PREFIX)
-    string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
-
-    if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
-    endif()
-    if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
-    endif()
-
-    configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-endif ()
+    INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${LIBDIR}/pkgconfig")
+ENDIF (UNIX)
 
 # install target
-install(FILES
-    "${LIBRARY_DIR}/zstd.h"
-    "${LIBRARY_DIR}/zdict.h"
-    "${LIBRARY_DIR}/zstd_errors.h"
-    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+INSTALL(FILES
+    ${LIBRARY_DIR}/zstd.h
+    ${LIBRARY_DIR}/deprecated/zbuff.h
+    ${LIBRARY_DIR}/dictBuilder/zdict.h
+    ${LIBRARY_DIR}/dictBuilder/cover.h
+    ${LIBRARY_DIR}/common/zstd_errors.h
+    DESTINATION "include")
 
-install(TARGETS ${library_targets}
-    EXPORT zstdExports
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    )
+IF (ZSTD_BUILD_SHARED)
+    INSTALL(TARGETS libzstd_shared RUNTIME DESTINATION "bin"
+                                   LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+                                   ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+ENDIF()
+IF (ZSTD_BUILD_STATIC)
+    INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+ENDIF (ZSTD_BUILD_STATIC)
 
 # uninstall target
-if (NOT TARGET uninstall)
-    configure_file(
-            "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
-            "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
-            IMMEDIATE @ONLY)
+CONFIGURE_FILE(
+        "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
+        "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
+        IMMEDIATE @ONLY)
 
-    add_custom_target(uninstall
-            COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
-endif ()
+ADD_CUSTOM_TARGET(uninstall
+        COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)

build/cmake/lib/cmake_uninstall.cmake.in

@@ -1,7 +1,7 @@
  
 if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
   message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
-endif()
+endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
 
 file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
 string(REGEX REPLACE "\n" ";" files "${files}")
@@ -15,8 +15,8 @@ foreach(file ${files})
       )
     if(NOT "${rm_retval}" STREQUAL 0)
       message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
-    endif()
-  else()
+    endif(NOT "${rm_retval}" STREQUAL 0)
+  else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
     message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
-  endif()
-endforeach()
+  endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
+endforeach(file)

build/cmake/lib/pkgconfig.cmake

@@ -0,0 +1 @@
+CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)

build/cmake/programs/CMakeLists.txt

@@ -7,131 +7,104 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(programs C)
+PROJECT(programs)
 
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
+SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-if (ZSTD_LEGACY_SUPPORT)
-    set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
-    include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
-endif ()
+IF (ZSTD_LEGACY_SUPPORT)
+    SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
+    INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
+ENDIF (ZSTD_LEGACY_SUPPORT)
 
-if (ZSTD_PROGRAMS_LINK_SHARED)
-    set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
-else ()
-    set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
-endif ()
+IF (MSVC)
+    SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
+    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
+ENDIF (MSVC)
 
-if (MSVC)
-    set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
-    set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
-endif ()
+ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+TARGET_LINK_LIBRARIES(zstd libzstd_static)
+IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
+    TARGET_LINK_LIBRARIES(zstd rt)
+ENDIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
+INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
 
-add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
-target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
-if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
-    target_link_libraries(zstd rt)
-endif ()
-install(TARGETS zstd
-  RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}")
+IF (UNIX)
+    ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
+    ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
+    INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
 
-if (UNIX)
-    add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
-    add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
-    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}")
-
-    add_custom_target(zstd.1 ALL
+    ADD_CUSTOM_TARGET(zstd.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
         COMMENT "Copying manpage zstd.1")
-    add_custom_target(zstdgrep.1 ALL
+    ADD_CUSTOM_TARGET(zstdgrep.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
         COMMENT "Copying manpage zstdgrep.1")
-    add_custom_target(zstdless.1 ALL
+    ADD_CUSTOM_TARGET(zstdless.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
         COMMENT "Copying manpage zstdless.1")
-    add_custom_target(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
-    add_custom_target(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
+    ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
+    ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
 
     # Define MAN_INSTALL_DIR if necessary
-    if (MAN_INSTALL_DIR)
-    else ()
-        set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1)
-    endif ()
+    IF (MAN_INSTALL_DIR)
+    ELSE (MAN_INSTALL_DIR)
+      SET(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1)
+    ENDIF (MAN_INSTALL_DIR)
 
-    install(FILES
-        ${CMAKE_CURRENT_BINARY_DIR}/zstd.1
-        ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1
-        ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1
-        ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1
-        ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
-        DESTINATION "${MAN_INSTALL_DIR}")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
 
-    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
-    target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
-    set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
-endif ()
+    ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
+    TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
+    SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
+ENDIF (UNIX)
 
 # Add multi-threading support definitions
 
-if (ZSTD_MULTITHREAD_SUPPORT)
-    set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+IF (ZSTD_MULTITHREAD_SUPPORT)
+    SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
 
-    if (UNIX)
-        target_link_libraries(zstd ${THREADS_LIBS})
+    IF (UNIX)
+        TARGET_LINK_LIBRARIES(zstd ${THREADS_LIBS})
 
-        add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
-        install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}")
-    endif ()
-endif ()
+        ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
+        INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
+    ENDIF (UNIX)
+ENDIF (ZSTD_MULTITHREAD_SUPPORT)
 
-option(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
-option(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
-option(ZSTD_LZ4_SUPPORT "LZ4 SUPPORT" OFF)
+OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
+OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
 
-# Add gzip support
-if (ZSTD_ZLIB_SUPPORT)
-    find_package(ZLIB REQUIRED)
+IF (ZSTD_ZLIB_SUPPORT)
+    FIND_PACKAGE(ZLIB REQUIRED)
 
-    if (ZLIB_FOUND)
-        include_directories(${ZLIB_INCLUDE_DIRS})
-        target_link_libraries(zstd ${ZLIB_LIBRARIES})
-        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
-    else ()
-        message(SEND_ERROR "zlib library is missing")
-    endif ()
-endif ()
+    IF (ZLIB_FOUND)
+        INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
+        TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
+        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
+    ELSE ()
+        MESSAGE(SEND_ERROR "zlib library is missing")
+    ENDIF ()
+ENDIF ()
 
-# Add lzma support
-if (ZSTD_LZMA_SUPPORT)
-    find_package(LibLZMA REQUIRED)
+IF (ZSTD_LZMA_SUPPORT)
+    FIND_PACKAGE(LibLZMA REQUIRED)
 
-    if (LIBLZMA_FOUND)
-        include_directories(${LIBLZMA_INCLUDE_DIRS})
-        target_link_libraries(zstd ${LIBLZMA_LIBRARIES})
-        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
-    else ()
-        message(SEND_ERROR "lzma library is missing")
-    endif ()
-endif ()
-
-# Add lz4 support
-if (ZSTD_LZ4_SUPPORT)
-    find_package(LibLZ4 REQUIRED)
-
-    if (LIBLZ4_FOUND)
-        include_directories(${LIBLZ4_INCLUDE_DIRS})
-        target_link_libraries(zstd ${LIBLZ4_LIBRARIES})
-        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZ4COMPRESS;ZSTD_LZ4DECOMPRESS")
-    else ()
-        message(SEND_ERROR "lz4 library is missing")
-    endif ()
-endif ()
+    IF (LIBLZMA_FOUND)
+        INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
+        TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
+        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
+    ELSE ()
+        MESSAGE(SEND_ERROR "lzma library is missing")
+    ENDIF ()
+ENDIF ()

build/cmake/tests/.gitignore

@@ -3,4 +3,5 @@ datagen
 fullbench
 fuzzer
 paramgrill
+zbufftest
 

build/cmake/tests/CMakeLists.txt

@@ -30,82 +30,26 @@
 #  - zstd homepage : http://www.zstd.net/
 # ################################################################
 
-project(tests)
+PROJECT(tests)
 
-# name: Cache variable name. The value is expected to be a semicolon-separated
-# list of command line flags
-# default_value: Value to initialize the option with. Can be space separated.
-function(AddTestFlagsOption name default_value doc)
-    string(STRIP "${default_value}" default_value)
-    string(REGEX REPLACE " +" ";" default_value "${default_value}")
-    set(${name} ${default_value} CACHE STRING "${doc}")
-    mark_as_advanced(${name})
-endfunction()
-
-set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
-include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
+SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+SET(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
+INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
-target_link_libraries(datagen libzstd_static)
+ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+TARGET_LINK_LIBRARIES(datagen libzstd_static)
 
-#
-# fullbench
-#
-add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
-set_property(TARGET fullbench APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
-target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
+ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
+TARGET_LINK_LIBRARIES(fullbench libzstd_static)
 
-#
-# fuzzer
-#
-add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
-set_property(TARGET fuzzer APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
-target_link_libraries(fuzzer libzstd_static)
-AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
-    "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
-add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
-# Disable the timeout since the run time is too long for the default timeout of
-# 1500 seconds and varies considerably between low-end and high-end CPUs.
-# set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
+ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
+TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
 
-#
-# zstreamtest
-#
-add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
-set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_OPTIONS "-Wno-deprecated-declarations")
-target_link_libraries(zstreamtest libzstd_static)
-AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
-    "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
-add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
-
-#
-# playTests.sh
-#
-AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}"
-    "Semicolon-separated list of flags to pass to the playTests.sh test")
-add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}")
-if (ZSTD_BUILD_PROGRAMS)
-    set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
-        "ZSTD_BIN=$<TARGET_FILE:zstd>"
-        "DATAGEN_BIN=$<TARGET_FILE:datagen>"
-        )
-else()
-    message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
-    set_tests_properties(playTests PROPERTIES DISABLED YES)
-endif()
-
-# Label the "Medium" set of tests (see TESTING.md)
-set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
-
-add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
-if (UNIX)
-    target_link_libraries(paramgrill libzstd_static m) #m is math library
-else()
-    target_link_libraries(paramgrill libzstd_static)
-endif ()
+IF (UNIX)
+    ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
+    TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
+ENDIF (UNIX)

build/cmake/zstdConfig.cmake

@@ -1 +0,0 @@
-include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")

build/meson/GetZstdLibraryVersion.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# #############################################################################
-# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-import re
-
-
-def find_version_tuple(filepath):
-  version_file_data = None
-  with open(filepath) as fd:
-    version_file_data = fd.read()
-
-  patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
-#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
-#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
-"""
-  regex = re.compile(patterns, re.MULTILINE)
-  version_match = regex.search(version_file_data)
-  if version_match:
-    return version_match.groups()
-  raise Exception("Unable to find version string")
-
-
-def main():
-  import argparse
-  parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
-  parser.add_argument('file', help='path to lib/zstd.h')
-  args = parser.parse_args()
-  version_tuple = find_version_tuple(args.file)
-  print('.'.join(version_tuple))
-
-
-if __name__ == '__main__':
-  main()

build/meson/InstallSymlink.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-# #############################################################################
-# Copyright (c) 2018-present  lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-# This file should be synced with https://github.com/lzutao/meson-symlink
-
-import os
-import pathlib  # since Python 3.4
-
-
-def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
-  if not install_dir.exists():
-    install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
-  if not install_dir.is_dir():
-    raise NotADirectoryError(install_dir)
-
-  new_dst = install_dir.joinpath(dst)
-  if new_dst.is_symlink() and os.readlink(new_dst) == src:
-    print('File exists: {!r} -> {!r}'.format(new_dst, src))
-    return
-  print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
-  new_dst.symlink_to(src, target_is_directory=dst_is_dir)
-
-
-def main():
-  import argparse
-  parser = argparse.ArgumentParser(description='Install a symlink',
-      usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
-            'example:\n'
-            '        {0} dash sh /bin'.format(pathlib.Path(__file__).name))
-  parser.add_argument('source', help='target to link')
-  parser.add_argument('dest', help='link name')
-  parser.add_argument('install_dir', help='installation directory')
-  parser.add_argument('-d', '--isdir',
-      action='store_true',
-      help='dest is a directory')
-  parser.add_argument('-m', '--mode',
-      help='directory mode on creating if not exist',
-      default='0o755')
-  args = parser.parse_args()
-
-  dir_mode = int(args.mode, 8)
-
-  meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
-  install_dir = pathlib.Path(meson_destdir, args.install_dir)
-  install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
-
-
-if __name__ == '__main__':
-  main()

build/meson/README.md

@@ -1,38 +0,0 @@
-Meson build system for zstandard
-================================
-
-Meson is a build system designed to optimize programmer productivity.
-It aims to do this by providing simple, out-of-the-box support for
-modern software development tools and practices, such as unit tests,
-coverage reports, Valgrind, CCache and the like.
-
-This Meson build system is provided with no guarantee and maintained
-by Dima Krasner \<dima@dimakrasner.com\>.
-
-It outputs one `libzstd`, either shared or static, depending on
-`default_library` option.
-
-## How to build
-
-`cd` to this meson directory (`build/meson`)
-
-```sh
-meson setup -Dbin_programs=true -Dbin_contrib=true builddir
-cd builddir
-ninja             # to build
-ninja install     # to install
-```
-
-You might want to install it in staging directory:
-
-```sh
-DESTDIR=./staging ninja install
-```
-
-To configure build options, use:
-
-```sh
-meson configure
-```
-
-See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).

build/meson/contrib/gen_html/meson.build

@@ -1,30 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-zstd_rootdir = '../../../..'
-
-gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
-  join_paths(zstd_rootdir, 'lib'),
-  join_paths(zstd_rootdir, 'lib/common'),
-  join_paths(zstd_rootdir, 'contrib/gen_html'))
-
-gen_html = executable('gen_html',
-  join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
-  include_directories: gen_html_includes,
-  native: true,
-  install: false)
-
-# Update zstd manual
-zstd_manual_html = custom_target('zstd_manual.html',
-  output : 'zstd_manual.html',
-  command : [gen_html,
-    zstd_version,
-    join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
-    '@OUTPUT@'],
-  install : false)

build/meson/contrib/meson.build

@@ -1,12 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-subdir('pzstd')
-subdir('gen_html')

build/meson/contrib/pzstd/meson.build

@@ -1,24 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-zstd_rootdir = '../../../..'
-
-pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
-  join_paths(zstd_rootdir, 'contrib/pzstd'))
-pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
-  join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
-  join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
-  join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
-pzstd = executable('pzstd',
-  pzstd_sources,
-  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-Wno-deprecated-declarations' ],
-  include_directories: pzstd_includes,
-  dependencies: [ libzstd_dep, thread_dep ],
-  install: true)

build/meson/lib/meson.build

@@ -1,137 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-zstd_rootdir = '../../..'
-
-libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
-  join_paths(zstd_rootdir, 'lib/common'),
-  join_paths(zstd_rootdir, 'lib/compress'),
-  join_paths(zstd_rootdir, 'lib/decompress'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder'))]
-
-libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
-  join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
-  join_paths(zstd_rootdir, 'lib/common/threading.c'),
-  join_paths(zstd_rootdir, 'lib/common/pool.c'),
-  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
-  join_paths(zstd_rootdir, 'lib/common/error_private.c'),
-  join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
-  join_paths(zstd_rootdir, 'lib/compress/hist.c'),
-  join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
-  join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
-  join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
-  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
-  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
-  join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
-  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')]
-
-# really we need anything that defines __GNUC__ as that is what ZSTD_ASM_SUPPORTED is gated on
-# but these are the two compilers that are supported in tree and actually handle this correctly
-# Otherwise, explicitly disable assmebly.
-if [compiler_gcc, compiler_clang].contains(cc_id)
-  libzstd_sources += join_paths(zstd_rootdir, 'lib/decompress/huf_decompress_amd64.S')
-else
-  add_project_arguments('-DZSTD_DISABLE_ASM', language: 'c')
-endif
-
-# Explicit define legacy support
-add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
-  language: 'c')
-
-if legacy_level == 0
-  message('Legacy support: DISABLED')
-else
-  # See ZSTD_LEGACY_SUPPORT of lib/README.md
-  message('Enable legacy support back to version 0.@0@'.format(legacy_level))
-
-  libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
-  foreach i : [1, 2, 3, 4, 5, 6, 7]
-    if legacy_level <= i
-      libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
-    endif
-  endforeach
-endif
-
-libzstd_deps = []
-if use_multi_thread
-  message('Enable multi-threading support')
-  add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
-  libzstd_deps = [ thread_dep ]
-endif
-
-libzstd_c_args = []
-if cc_id == compiler_msvc
-  if default_library_type != 'static'
-    libzstd_sources += [windows_mod.compile_resources(
-      join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
-    libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
-      '-DZSTD_HEAPMODE=0',
-      '-D_CONSOLE',
-      '-D_CRT_SECURE_NO_WARNINGS']
-  else
-    libzstd_c_args += ['-DZSTD_HEAPMODE=0',
-      '-D_CRT_SECURE_NO_WARNINGS']
-  endif
-endif
-
-mingw_ansi_stdio_flags = []
-if host_machine_os == os_windows and cc_id == compiler_gcc
-  mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
-endif
-libzstd_c_args += mingw_ansi_stdio_flags
-
-libzstd_debug_cflags = []
-if use_debug
-  libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
-  if cc_id == compiler_gcc or cc_id == compiler_clang
-    libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
-      '-Wdeclaration-after-statement', '-Wstrict-prototypes',
-      '-Wundef', '-Wpointer-arith', '-Wvla',
-      '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
-      '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
-  endif
-endif
-libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
-
-libzstd = library('zstd',
-  libzstd_sources,
-  include_directories: libzstd_includes,
-  c_args: libzstd_c_args,
-  gnu_symbol_visibility: 'hidden',
-  dependencies: libzstd_deps,
-  install: true,
-  version: zstd_libversion)
-
-libzstd_dep = declare_dependency(link_with: libzstd,
-  include_directories: libzstd_includes)
-
-pkgconfig.generate(libzstd,
-  name: 'libzstd',
-  filebase: 'libzstd',
-  description: 'fast lossless compression algorithm library',
-  version: zstd_libversion,
-  url: 'http://www.zstd.net/')
-
-install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
-  join_paths(zstd_rootdir, 'lib/zdict.h'),
-  join_paths(zstd_rootdir, 'lib/zstd_errors.h'))

build/meson/meson.build

@@ -1,148 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-project('zstd',
-  ['c', 'cpp'],
-  license: ['BSD', 'GPLv2'],
-  default_options : [
-    'c_std=gnu99',
-    'cpp_std=c++11',
-    'buildtype=release',
-    'warning_level=3',
-    # -Wdocumentation does not actually pass, nor do the test binaries,
-    # so this isn't safe
-    #'werror=true'
-  ],
-  version: 'DUMMY',
-  meson_version: '>=0.48.0')
-
-cc = meson.get_compiler('c')
-cxx = meson.get_compiler('cpp')
-pkgconfig = import('pkgconfig')
-windows_mod = import('windows')
-
-host_machine_os = host_machine.system()
-os_windows = 'windows'
-os_linux = 'linux'
-os_darwin = 'darwin'
-os_freebsd = 'freebsd'
-os_sun = 'sunos'
-
-cc_id = cc.get_id()
-compiler_gcc = 'gcc'
-compiler_clang = 'clang'
-compiler_msvc = 'msvc'
-
-zstd_version = meson.project_version()
-
-zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
-GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
-r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
-if r.returncode() == 0
-  zstd_version = r.stdout().strip()
-  message('Project version is now: @0@'.format(zstd_version))
-else
-  error('Cannot find project version in @0@'.format(zstd_h_file))
-endif
-
-zstd_libversion = zstd_version
-
-# =============================================================================
-# Installation directories
-# =============================================================================
-
-zstd_prefix = get_option('prefix')
-zstd_bindir = get_option('bindir')
-zstd_datadir = get_option('datadir')
-zstd_mandir = get_option('mandir')
-zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
-
-# =============================================================================
-# Project options
-# =============================================================================
-
-# Built-in options
-use_debug = get_option('debug')
-buildtype = get_option('buildtype')
-default_library_type = get_option('default_library')
-
-# Custom options
-debug_level = get_option('debug_level')
-legacy_level = get_option('legacy_level')
-use_backtrace = get_option('backtrace')
-use_static_runtime = get_option('static_runtime')
-
-bin_programs = get_option('bin_programs')
-bin_contrib = get_option('bin_contrib')
-bin_tests = get_option('bin_tests')
-
-feature_multi_thread = get_option('multi_thread')
-feature_zlib = get_option('zlib')
-feature_lzma = get_option('lzma')
-feature_lz4 = get_option('lz4')
-
-# =============================================================================
-# Dependencies
-# =============================================================================
-
-libm_dep = cc.find_library('m', required: bin_tests)
-thread_dep = dependency('threads', required: feature_multi_thread)
-use_multi_thread = thread_dep.found()
-# Arguments in dependency should be equivalent to those passed to pkg-config
-zlib_dep = dependency('zlib', required: feature_zlib)
-use_zlib = zlib_dep.found()
-lzma_dep = dependency('liblzma', required: feature_lzma)
-use_lzma = lzma_dep.found()
-lz4_dep = dependency('liblz4', required: feature_lz4)
-use_lz4 = lz4_dep.found()
-
-# =============================================================================
-# Compiler flags
-# =============================================================================
-
-add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
-
-if [compiler_gcc, compiler_clang].contains(cc_id)
-  common_warning_flags = [ '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
-  if cc_id == compiler_clang
-    common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
-  endif
-  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
-  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
-  add_project_arguments(cc_compile_flags, language : 'c')
-  add_project_arguments(cxx_compile_flags, language : 'cpp')
-elif cc_id == compiler_msvc
-  msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
-  if use_multi_thread
-    msvc_compile_flags += '/MP'
-  endif
-  if use_static_runtime
-    msvc_compile_flags += '/MT'
-  endif
-  add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
-endif
-
-# =============================================================================
-# Subdirs
-# =============================================================================
-
-subdir('lib')
-
-if bin_programs
-  subdir('programs')
-endif
-
-if bin_tests
-  subdir('tests')
-endif
-
-if bin_contrib
-  subdir('contrib')
-endif

build/meson/meson_options.txt

@@ -1,36 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
-
-option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
-  description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
-option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
-  description: 'Enable run-time debug. See lib/common/debug.h')
-option('backtrace', type: 'boolean', value: false,
-  description: 'Display a stack backtrace when execution generates a runtime exception')
-option('static_runtime', type: 'boolean', value: false,
-  description: 'Link to static run-time libraries on MSVC')
-
-option('bin_programs', type: 'boolean', value: true,
-  description: 'Enable programs build')
-option('bin_tests', type: 'boolean', value: false,
-  description: 'Enable tests build')
-option('bin_contrib', type: 'boolean', value: false,
-  description: 'Enable contrib build')
-
-option('multi_thread', type: 'feature', value: 'enabled',
-  description: 'Enable multi-threading when pthread is detected')
-option('zlib', type: 'feature', value: 'auto',
-  description: 'Enable zlib support')
-option('lzma', type: 'feature', value: 'auto',
-  description: 'Enable lzma support')
-option('lz4', type: 'feature', value: 'auto',
-  description: 'Enable lz4 support')

build/meson/programs/meson.build

@@ -1,107 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-zstd_rootdir = '../../..'
-
-zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/fileio.c'),
-  join_paths(zstd_rootdir, 'programs/benchfn.c'),
-  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
-  join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/dibio.c'),
-  join_paths(zstd_rootdir, 'programs/zstdcli_trace.c'),
-  # needed due to use of private symbol + -fvisibility=hidden
-  join_paths(zstd_rootdir, 'lib/common/xxhash.c')]
-
-zstd_c_args = libzstd_debug_cflags
-if use_multi_thread
-  zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
-endif
-
-zstd_deps = [ libzstd_dep ]
-if use_zlib
-  zstd_deps += [ zlib_dep ]
-  zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
-endif
-
-if use_lzma
-  zstd_deps += [ lzma_dep ]
-  zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
-endif
-
-if use_lz4
-  zstd_deps += [ lz4_dep ]
-  zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
-endif
-
-export_dynamic_on_windows = false
-# explicit backtrace enable/disable for Linux & Darwin
-if not use_backtrace
-  zstd_c_args += '-DBACKTRACE_ENABLE=0'
-elif use_debug and host_machine_os == os_windows  # MinGW target
-  zstd_c_args += '-DBACKTRACE_ENABLE=1'
-  export_dynamic_on_windows = true
-endif
-
-if cc_id == compiler_msvc
-  if default_library_type != 'static'
-    zstd_programs_sources += [windows_mod.compile_resources(
-      join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
-  endif
-endif
-
-zstd = executable('zstd',
-  zstd_programs_sources,
-  c_args: zstd_c_args,
-  dependencies: zstd_deps,
-  export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
-  install: true)
-
-zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/fileio.c')]
-
-# Minimal target, with only zstd compression and decompression.
-# No bench. No legacy.
-executable('zstd-frugal',
-  zstd_frugal_sources,
-  dependencies: libzstd_dep,
-  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
-  install: true)
-
-install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
-  join_paths(zstd_rootdir, 'programs/zstdless'),
-  install_dir: zstd_bindir)
-
-# =============================================================================
-# Programs and manpages installing
-# =============================================================================
-
-install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
-  join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
-  join_paths(zstd_rootdir, 'programs/zstdless.1'))
-
-InstallSymlink_py = '../InstallSymlink.py'
-zstd_man1_dir = join_paths(zstd_mandir, 'man1')
-bin_EXT = host_machine_os == os_windows ? '.exe' : ''
-man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
-
-foreach f : ['zstdcat', 'unzstd']
-  meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, f + bin_EXT, zstd_bindir)
-  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
-endforeach
-
-if use_multi_thread
-  meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, 'zstdmt' + bin_EXT, zstd_bindir)
-  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
-endif

build/meson/tests/meson.build

@@ -1,196 +0,0 @@
-# #############################################################################
-# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
-#                                lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-
-zstd_rootdir = '../../..'
-
-tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
-  os_freebsd, 'netbsd', 'dragonfly', os_sun]
-
-# =============================================================================
-# Test flags
-# =============================================================================
-
-FUZZER_FLAGS = ['--no-big-tests']
-FUZZERTEST = '-T200s'
-ZSTREAM_TESTTIME = '-T90s'
-DECODECORPUS_TESTTIME = '-T30'
-ZSTDRTTEST = ['--test-large-data']
-
-# =============================================================================
-# Executables
-# =============================================================================
-
-test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
-
-testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/util.c'),
-  join_paths(zstd_rootdir, 'programs/timefn.c'),
-  join_paths(zstd_rootdir, 'programs/benchfn.c'),
-  join_paths(zstd_rootdir, 'programs/benchzstd.c')]
-
-testcommon = static_library('testcommon',
-  testcommon_sources,
-  # needed due to use of private symbol + -fvisibility=hidden
-  objects: libzstd.extract_all_objects(recursive: false))
-
-testcommon_dep = declare_dependency(link_with: testcommon,
-  dependencies: libzstd_deps,
-  include_directories: libzstd_includes)
-
-datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
-datagen = executable('datagen',
-  datagen_sources,
-  c_args: [ '-DNDEBUG' ],
-  include_directories: test_includes,
-  dependencies: testcommon_dep,
-  install: false)
-
-fullbench_sources = [join_paths(zstd_rootdir, 'tests/fullbench.c')]
-fullbench = executable('fullbench',
-  fullbench_sources,
-  include_directories: test_includes,
-  dependencies: testcommon_dep,
-  install: false)
-
-fuzzer_sources = [join_paths(zstd_rootdir, 'tests/fuzzer.c')]
-fuzzer = executable('fuzzer',
-  fuzzer_sources,
-  include_directories: test_includes,
-  dependencies: [ testcommon_dep, thread_dep ],
-  install: false)
-
-zstreamtest_sources = [join_paths(zstd_rootdir, 'tests/seqgen.c'),
-  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
-zstreamtest = executable('zstreamtest',
-  zstreamtest_sources,
-  include_directories: test_includes,
-  dependencies: testcommon_dep,
-  install: false)
-
-paramgrill_sources = [join_paths(zstd_rootdir, 'tests/paramgrill.c')]
-paramgrill = executable('paramgrill',
-  paramgrill_sources,
-  include_directories: test_includes,
-  dependencies: [ testcommon_dep, libm_dep ],
-  install: false)
-
-roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
-roundTripCrash = executable('roundTripCrash',
-  roundTripCrash_sources,
-  dependencies: [ testcommon_dep ],
-  install: false)
-
-longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
-longmatch = executable('longmatch',
-  longmatch_sources,
-  dependencies: [ libzstd_dep ],
-  install: false)
-
-invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
-invalidDictionaries = executable('invalidDictionaries',
-  invalidDictionaries_sources,
-  dependencies: [ libzstd_dep ],
-  install: false)
-
-if 0 < legacy_level and legacy_level <= 4
-  legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
-  legacy = executable('legacy',
-    legacy_sources,
-    # Use -Dlegacy_level build option to control it
-    #c_args: '-DZSTD_LEGACY_SUPPORT=4',
-    dependencies: [ libzstd_dep ],
-    install: false)
-endif
-
-decodecorpus_sources = [join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
-decodecorpus = executable('decodecorpus',
-  decodecorpus_sources,
-  include_directories: test_includes,
-  dependencies: [ testcommon_dep, libm_dep ],
-  install: false)
-
-poolTests_sources = [join_paths(zstd_rootdir, 'tests/poolTests.c'),
-  join_paths(zstd_rootdir, 'lib/common/pool.c'),
-  join_paths(zstd_rootdir, 'lib/common/threading.c'),
-  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
-  join_paths(zstd_rootdir, 'lib/common/error_private.c')]
-poolTests = executable('poolTests',
-  poolTests_sources,
-  include_directories: test_includes,
-  dependencies: [ testcommon_dep, thread_dep ],
-  install: false)
-
-checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
-checkTag = executable('checkTag',
-  checkTag_sources,
-  dependencies: [ libzstd_dep ],
-  install: false)
-
-# =============================================================================
-# Tests (Use "meson test --list" to list all tests)
-# =============================================================================
-
-if tests_supported_oses.contains(host_machine_os)
-  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
-  valgrindTest_py = files('valgrindTest.py')
-  test('valgrindTest',
-    valgrindTest_py,
-    args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
-    depends: [zstd, datagen, fuzzer, fullbench],
-    timeout: 600) # Timeout should work on HDD drive
-endif
-
-if host_machine_os != os_windows
-  playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
-  test('test-zstd',
-    playTests_sh,
-    args: ZSTDRTTEST,
-    env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
-    depends: [datagen],
-    workdir: meson.current_build_dir(),
-    timeout: 2800) # Timeout should work on HDD drive
-endif
-
-test('test-fullbench-1',
-  fullbench,
-  args: ['-i1'],
-  depends: [datagen],
-  timeout: 60)
-test('test-fullbench-2',
-  fullbench,
-  args: ['-i1', '-P0'],
-  depends: [datagen],
-  timeout: 60)
-
-if use_zlib
-  test('test-fuzzer',
-    fuzzer,
-    args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
-    timeout: 480)
-endif
-
-test('test-zstream-1',
-  zstreamtest,
-  args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
-  timeout: 240)
-test('test-zstream-3',
-  zstreamtest,
-  args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
-  timeout: 120)
-test('test-longmatch', longmatch, timeout: 36)
-test('test-invalidDictionaries', invalidDictionaries) # should be fast
-if 0 < legacy_level and legacy_level <= 4
-  test('test-legacy', legacy) # should be fast
-endif
-test('test-decodecorpus',
-  decodecorpus,
-  args: ['-t', DECODECORPUS_TESTTIME],
-  timeout: 60)
-test('test-poolTests', poolTests) # should be fast

build/meson/tests/valgrindTest.py

@@ -1,90 +0,0 @@
-#!/usr/bin/env python3
-# #############################################################################
-# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# #############################################################################
-import os
-import subprocess
-import tempfile
-
-
-def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
-  VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
-
-  print('\n ---- valgrind tests : memory analyzer ----')
-
-  subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
-
-  if subprocess.call([*VALGRIND_ARGS, zstd],
-                     stdout=subprocess.DEVNULL) == 0:
-    raise subprocess.CalledProcessError('zstd without argument should have failed')
-
-  with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
-       subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
-                        stdin=p1.stdout,
-                        stdout=subprocess.DEVNULL) as p2:
-    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
-    p2.communicate()
-    if p2.returncode != 0:
-      raise subprocess.CalledProcessError()
-
-  with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
-       subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
-                        stdin=p1.stdout,
-                        stdout=subprocess.DEVNULL) as p2:
-    p1.stdout.close()
-    p2.communicate()
-    if p2.returncode != 0:
-      raise subprocess.CalledProcessError()
-
-  with tempfile.NamedTemporaryFile() as tmp_fd:
-    with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
-         subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
-                          stdin=p1.stdout) as p2:
-      p1.stdout.close()
-      p2.communicate()
-      if p2.returncode != 0:
-        raise subprocess.CalledProcessError()
-
-    subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
-                          stdout=subprocess.DEVNULL)
-
-    with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
-         subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
-                          stdin=p1.stdout,
-                          stdout=subprocess.DEVNULL) as p2:
-      p1.stdout.close()
-      p2.communicate()
-      if p2.returncode != 0:
-        raise subprocess.CalledProcessError()
-
-  subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
-  subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
-
-
-def main():
-  import argparse
-  parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
-  parser.add_argument('valgrind', help='valgrind path')
-  parser.add_argument('zstd', help='zstd path')
-  parser.add_argument('datagen', help='datagen path')
-  parser.add_argument('fuzzer', help='fuzzer path')
-  parser.add_argument('fullbench', help='fullbench path')
-
-  args = parser.parse_args()
-
-  valgrind = args.valgrind
-  zstd = args.zstd
-  datagen = args.datagen
-  fuzzer = args.fuzzer
-  fullbench = args.fullbench
-
-  valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
-
-
-if __name__ == '__main__':
-  main()

build/single_file_libs/.gitignore

@@ -1,9 +0,0 @@
-
-# build artifacts
-zstddeclib.c
-zstdenclib.c
-zstd.c
-zstd.h
-
-# test artifacts
-temp*

build/single_file_libs/README.md

@@ -1,33 +0,0 @@
-# Single File Zstandard Libraries
-
-The script `combine.sh` creates an _amalgamated_ source file that can be used with or without `zstd.h`. This isn't a _header-only_ file but it does offer a similar level of simplicity when integrating into a project.
-
-All it now takes to support Zstd in your own projects is the addition of a single file, two if using the header, with no configuration or further build steps.
-
-Decompressor
-------------
-
-This is the most common use case. The decompression library is small, adding, for example, 26kB to an Emscripten compiled WebAssembly project. Native implementations add a little more, 40-70kB depending on the compiler and platform.
-
-Create `zstddeclib.c` from the Zstd source using:
-```
-cd zstd/build/single_file_libs
-./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
-```
-Then add the resulting file to your project (see the [example files](examples)).
-
-`create_single_file_decoder.sh` will run the above script, creating the file `zstddeclib.c` (`build_decoder_test.sh` will also create `zstddeclib.c`, then compile and test the result).
-
-Full Library
-------------
-
-The same tool can amalgamate the entire Zstd library for ease of adding both compression and decompression to a project. The [roundtrip example](examples/roundtrip.c) uses the original `zstd.h` with the remaining source files combined into `zstd.c` (currently just over 1.2MB) created from `zstd-in.c`. As with the standalone decoder the most useful compile flags have already been rolled-in and the resulting file can be added to a project as-is.
-
-Create `zstd.c` from the Zstd source using:
-```
-cd zstd/build/single_file_libs
-./combine.sh -r ../../lib -o zstd.c zstd-in.c
-```
-It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`).
-
-`create_single_file_library.sh` will run the script to create `zstd.c` (`build_library_test.sh` will also create `zstd.c`, then compile and test the result).

build/single_file_libs/build_decoder_test.sh

@@ -1,91 +0,0 @@
-#!/bin/sh
-
-# Temporary compiled binary
-OUT_FILE="tempbin"
-
-# Optional temporary compiled WebAssembly
-OUT_WASM="temp.wasm"
-
-# Source files to compile using Emscripten.
-IN_FILES="examples/emscripten.c"
-
-# Emscripten build using emcc.
-emscripten_emcc_build() {
-  # Compile the the same example as above
-  CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
-  emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
-  # Did compilation work?
-  if [ $? -ne 0 ]; then
-    echo "Compiling ${IN_FILES}: FAILED"
-    exit 1
-  fi
-  echo "Compiling ${IN_FILES}: PASSED"
-  rm -f $OUT_WASM
-}
-
-# Emscripten build using docker.
-emscripten_docker_build() {
-  docker container run --rm \
-    --volume $PWD:/code \
-    --workdir /code \
-    emscripten/emsdk:latest \
-    emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
-  # Did compilation work?
-  if [ $? -ne 0 ]; then
-      echo "Compiling ${IN_FILES} (using docker): FAILED"
-    exit 1
-  fi
-  echo "Compiling ${IN_FILES} (using docker): PASSED"
-  rm -f $OUT_WASM
-}
-
-# Try Emscripten build using emcc or docker.
-try_emscripten_build() {
-  which emcc > /dev/null
-  if [ $? -eq 0 ]; then
-    emscripten_emcc_build
-    return $?
-  fi
-
-  which docker > /dev/null
-  if [ $? -eq 0 ]; then
-    emscripten_docker_build
-    return $?
-  fi
-
-  echo "(Skipping Emscripten test)"
-}
-
-# Amalgamate the sources
-./create_single_file_decoder.sh
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Single file decoder creation script: FAILED"
-  exit 1
-fi
-echo "Single file decoder creation script: PASSED"
-
-# Compile the generated output
-cc -Wall -Wextra -Wshadow -Werror -Os -g0 -o $OUT_FILE examples/simple.c
-# Did compilation work?
-if [ $? -ne 0 ]; then
-  echo "Compiling simple.c: FAILED"
-  exit 1
-fi
-echo "Compiling simple.c: PASSED"
-
-# Run then delete the compiled output
-./$OUT_FILE
-retVal=$?
-rm -f $OUT_FILE
-# Did the test work?
-if [ $retVal -ne 0 ]; then
-  echo "Running simple.c: FAILED"
-  exit 1
-fi
-echo "Running simple.c: PASSED"
-
-# Try Emscripten build if emcc or docker command is available.
-try_emscripten_build
-
-exit 0

build/single_file_libs/build_library_test.sh

@@ -1,97 +0,0 @@
-#!/bin/sh
-
-# Where to find the sources (only used to copy zstd.h)
-ZSTD_SRC_ROOT="../../lib"
-
-# Temporary compiled binary
-OUT_FILE="tempbin"
-
-# Optional temporary compiled WebAssembly
-OUT_WASM="temp.wasm"
-
-# Source files to compile using Emscripten.
-IN_FILES="zstd.c examples/roundtrip.c"
-
-# Emscripten build using emcc.
-emscripten_emcc_build() {
-  # Compile the the same example as above
-  CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
-  emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
-  # Did compilation work?
-  if [ $? -ne 0 ]; then
-    echo "Compiling ${IN_FILES}: FAILED"
-    exit 1
-  fi
-  echo "Compiling ${IN_FILES}: PASSED"
-  rm -f $OUT_WASM
-}
-
-# Emscripten build using docker.
-emscripten_docker_build() {
-  docker container run --rm \
-    --volume $PWD:/code \
-    --workdir /code \
-    emscripten/emsdk:latest \
-    emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
-  # Did compilation work?
-  if [ $? -ne 0 ]; then
-      echo "Compiling ${IN_FILES} (using docker): FAILED"
-    exit 1
-  fi
-  echo "Compiling ${IN_FILES} (using docker): PASSED"
-  rm -f $OUT_WASM
-}
-
-# Try Emscripten build using emcc or docker.
-try_emscripten_build() {
-  which emcc > /dev/null
-  if [ $? -eq 0 ]; then
-    emscripten_emcc_build
-    return $?
-  fi
-
-  which docker > /dev/null
-  if [ $? -eq 0 ]; then
-    emscripten_docker_build
-    return $?
-  fi
-
-  echo "(Skipping Emscripten test)"
-}
-
-# Amalgamate the sources
-./create_single_file_library.sh
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Single file library creation script: FAILED"
-  exit 1
-fi
-echo "Single file library creation script: PASSED"
-
-# Copy the header to here (for the tests)
-cp "$ZSTD_SRC_ROOT/zstd.h" zstd.h
-
-# Compile the generated output
-cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c
-# Did compilation work?
-if [ $? -ne 0 ]; then
-  echo "Compiling roundtrip.c: FAILED"
-  exit 1
-fi
-echo "Compiling roundtrip.c: PASSED"
-
-# Run then delete the compiled output
-./$OUT_FILE
-retVal=$?
-rm -f $OUT_FILE
-# Did the test work?
-if [ $retVal -ne 0 ]; then
-  echo "Running roundtrip.c: FAILED"
-  exit 1
-fi
-echo "Running roundtrip.c: PASSED"
-
-# Try Emscripten build if emcc or docker command is available.
-try_emscripten_build
-
-exit 0

build/single_file_libs/combine.sh

@@ -1,211 +0,0 @@
-#!/bin/sh -e
-
-# Tool to bundle multiple C/C++ source files, inlining any includes.
-# 
-# Note: this POSIX-compliant script is many times slower than the original bash
-# implementation (due to the grep calls) but it runs and works everywhere.
-# 
-# TODO: ROOTS, FOUND, etc., as arrays (since they fail on paths with spaces)
-# TODO: revert to Bash-only regex (the grep ones being too slow)
-# 
-# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
-
-# Common file roots
-ROOTS="."
-
-# -x option excluded includes
-XINCS=""
-
-# -k option includes to keep as include directives
-KINCS=""
-
-# Files previously visited
-FOUND=""
-
-# Optional destination file (empty string to write to stdout)
-DESTN=""
-
-# Whether the "#pragma once" directives should be written to the output
-PONCE=0
-
-# Prints the script usage then exits
-usage() {
-  echo "Usage: $0 [-r <path>] [-x <header>] [-k <header>] [-o <outfile>] infile"
-  echo "  -r file root search path"
-  echo "  -x file to completely exclude from inlining"
-  echo "  -k file to exclude from inlining but keep the include directive"
-  echo "  -p keep any '#pragma once' directives (removed by default)"
-  echo "  -o output file (otherwise stdout)"
-  echo "Example: $0 -r ../my/path - r ../other/path -o out.c in.c"
-  exit 1
-}
-
-# Tests that the grep implementation works as expected (older OSX grep fails)
-test_deps() {
-  if ! echo '#include "foo"' | grep -Eq '^\s*#\s*include\s*".+"'; then
-    echo "Aborting: the grep implementation fails to parse include lines"
-    exit 1
-  fi
-  if ! echo '"foo.h"' | sed -E 's/"([^"]+)"/\1/' | grep -Eq '^foo\.h$'; then
-    echo "Aborting: sed is unavailable or non-functional"
-    exit 1
-  fi
-}
-
-# Tests if list $1 has item $2 (returning zero on a match)
-list_has_item() {
-  if echo "$1" | grep -Eq "(^|\s*)$2(\$|\s*)"; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-# Adds a new line with the supplied arguments to $DESTN (or stdout)
-write_line() {
-  if [ -n "$DESTN" ]; then
-    printf '%s\n' "$@" >> "$DESTN"
-  else
-    printf '%s\n' "$@"
-  fi
-}
-
-log_line() {
-  echo $@ >&2
-}
-
-# Find this file!
-resolve_include() {
-  local srcdir=$1
-  local inc=$2
-  for root in $srcdir $ROOTS; do
-    if [ -f "$root/$inc" ]; then
-      # Try to reduce the file path into a canonical form (so that multiple)
-      # includes of the same file are successfully deduplicated, even if they
-      # are expressed differently.
-      local relpath="$(realpath --relative-to . "$root/$inc" 2>/dev/null)"
-      if [ "$relpath" != "" ]; then # not all realpaths support --relative-to
-        echo "$relpath"
-        return 0
-      fi
-      local relpath="$(realpath "$root/$inc" 2>/dev/null)"
-      if [ "$relpath" != "" ]; then # not all distros have realpath...
-        echo "$relpath"
-        return 0
-      fi
-      # Fallback on Python to reduce the path if the above fails.
-      local relpath=$(python -c "import os,sys; print os.path.relpath(sys.argv[1])" "$root/$inc" 2>/dev/null)
-      if [ "$relpath" != "" ]; then # not all distros have realpath...
-        echo "$relpath"
-        return 0
-      fi
-      # Worst case, fall back to just the root + relative include path. The
-      # problem with this is that it is possible to emit multiple different
-      # resolved paths to the same file, depending on exactly how its included.
-      # Since the main loop below keeps a list of the resolved paths it's
-      # already included, in order to avoid repeated includes, this failure to
-      # produce a canonical/reduced path can lead to multiple inclusions of the
-      # same file. But it seems like the resulting single file library still
-      # works (hurray include guards!), so I guess it's ok.
-      echo "$root/$inc"
-      return 0
-    fi
-  done
-  return 1
-}
-
-# Adds the contents of $1 with any of its includes inlined
-add_file() {
-  local file=$1
-  if [ -n "$file" ]; then
-    log_line "Processing: $file"
-    # Get directory of the current so we can resolve relative includes
-    local srcdir="$(dirname "$file")"
-    # Read the file
-    local line=
-    while IFS= read -r line; do
-      if echo "$line" | grep -Eq '^\s*#\s*include\s*".+"'; then
-        # We have an include directive so strip the (first) file
-        local inc=$(echo "$line" | grep -Eo '".*"' | sed -E 's/"([^"]+)"/\1/' | head -1)
-        local res_inc="$(resolve_include "$srcdir" "$inc")"
-        if list_has_item "$XINCS" "$inc"; then
-          # The file was excluded so error if the source attempts to use it
-          write_line "#error Using excluded file: $inc"
-          log_line "Excluding: $inc"
-        else
-          if ! list_has_item "$FOUND" "$res_inc"; then
-            # The file was not previously encountered
-            FOUND="$FOUND $res_inc"
-            if list_has_item "$KINCS" "$inc"; then
-              # But the include was flagged to keep as included
-              write_line "/**** *NOT* inlining $inc ****/"
-              write_line "$line"
-              log_line "Not Inlining: $inc"
-            else
-              # The file was neither excluded nor seen before so inline it
-              write_line "/**** start inlining $inc ****/"
-              add_file "$res_inc"
-              write_line "/**** ended inlining $inc ****/"
-            fi
-          else
-            write_line "/**** skipping file: $inc ****/"
-          fi
-        fi
-      else
-        # Skip any 'pragma once' directives, otherwise write the source line
-        local write=$PONCE
-        if [ $write -eq 0 ]; then
-          if echo "$line" | grep -Eqv '^\s*#\s*pragma\s*once\s*'; then
-            write=1
-          fi
-        fi
-        if [ $write -ne 0 ]; then
-          write_line "$line"
-        fi
-      fi
-    done < "$file"
-  else
-    write_line "#error Unable to find \"$1\""
-    log_line "Error: Unable to find: \"$1\""
-  fi
-}
-
-while getopts ":r:x:k:po:" opts; do
-  case $opts in
-  r)
-    ROOTS="$ROOTS $OPTARG"
-    ;;
-  x)
-    XINCS="$XINCS $OPTARG"
-    ;;
-  k)
-    KINCS="$KINCS $OPTARG"
-    ;;
-  p)
-    PONCE=1
-    ;;
-  o)
-    DESTN="$OPTARG"
-    ;;
-  *)
-    usage
-    ;;
-  esac
-done
-shift $((OPTIND-1))
-
-if [ -n "$1" ]; then
-  if [ -f "$1" ]; then
-    if [ -n "$DESTN" ]; then
-      printf "" > "$DESTN"
-    fi
-    test_deps
-    add_file "$1"
-  else
-    echo "Input file not found: \"$1\""
-    exit 1
-  fi
-else
-  usage
-fi
-exit 0

build/single_file_libs/create_single_file_decoder.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# Where to find the sources
-ZSTD_SRC_ROOT="../../lib"
-
-# Amalgamate the sources
-echo "Amalgamating files... this can take a while"
-./combine.sh -r "$ZSTD_SRC_ROOT" -o zstddeclib.c zstddeclib-in.c
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Combine script: FAILED"
-  exit 1
-fi
-echo "Combine script: PASSED"

build/single_file_libs/create_single_file_library.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# Where to find the sources
-ZSTD_SRC_ROOT="../../lib"
-
-# Amalgamate the sources
-echo "Amalgamating files... this can take a while"
-./combine.sh -r "$ZSTD_SRC_ROOT" -o zstd.c zstd-in.c
-# Did combining work?
-if [ $? -ne 0 ]; then
-  echo "Combine script: FAILED"
-  exit 1
-fi
-echo "Combine script: PASSED"

build/single_file_libs/examples/README.md

@@ -1,11 +0,0 @@
-# Single File ZStandard Examples
-
-The examples `#include` the generated `zstddeclib.c` directly but work equally as well when including `zstd.h` and compiling the amalgamated source separately.
-
-`simple.c` is the most basic example of decompressing content and verifying the result.
-
-`emscripten.c` is a bare-bones [Emscripten](https://github.com/emscripten-core/emscripten) compiled WebGL demo using Zstd to further compress a DXT1 texture (see the [original PNG image](testcard.png)). The 256x256 texture would normally be 32kB, but even when bundled with the Zstd decompressor the resulting WebAssembly weighs in at 41kB (`shell.html` is a support file to run the Wasm).
-
-`roundtrip.c` is an example to use with the optional [amalgamated library](../create_single_file_library.sh) showing compression the decompression.
-
-The example files in this directory are released under a [Creative Commons Zero license](https://creativecommons.org/publicdomain/zero/1.0/) (or Public Domain, whichever is applicable in your jurisdiction).

build/single_file_libs/examples/emscripten.c

@@ -1,340 +0,0 @@
-/**
- * \file emscripten.c
- * Emscripten example of using the single-file \c zstddeclib. Draws a rotating
- * textured quad with data from the in-line Zstd compressed DXT1 texture (DXT1
- * being hardware compression, further compressed with Zstd).
- * \n
- * Compile using:
- * \code
- *	export CC_FLAGS="-Wall -Wextra -Werror -Os -g0 -flto --llvm-lto 3 -lGL -DNDEBUG=1"
- *	export EM_FLAGS="-s WASM=1 -s ENVIRONMENT=web --shell-file shell.html --closure 1"
- *	emcc $CC_FLAGS $EM_FLAGS -o out.html emscripten.c
- * \endcode
- * 
- * \author Carl Woffenden, Numfum GmbH (released under a CC0 license)
- */
-
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <emscripten/emscripten.h>
-#include <emscripten/html5.h>
-
-#include <GLES2/gl2.h>
-#include <GLES2/gl2ext.h>
-
-#include "../zstddeclib.c"
-
-//************************* Test Data (DXT texture) **************************/
-
-/**
- * Zstd compressed DXT1 256x256 texture source.
- * \n
- * See \c testcard.png for the original.
- */
-static uint8_t const srcZstd[] = {
-#include "testcard-zstd.inl"
-};
-
-/**
- * Uncompressed size of \c #srcZstd.
- */
-#define DXT1_256x256 32768
-
-/**
- * Destination for decoding \c #srcZstd.
- */
-static uint8_t dstDxt1[DXT1_256x256] = {};
-
-#ifndef ZSTD_VERSION_MAJOR
-/**
- * For the case where the decompression library hasn't been included we add a
- * dummy function to fake the process and stop the buffers being optimised out.
- */
-size_t ZSTD_decompress(void* dst, size_t dstLen, const void* src, size_t srcLen) {
-	return (memcmp(dst, src, (srcLen < dstLen) ? srcLen : dstLen)) ? dstLen : 0;
-}
-#endif
-
-//*************************** Program and Shaders ***************************/
-
-/**
- * Program object ID.
- */
-static GLuint progId = 0;
-
-/**
- * Vertex shader ID.
- */
-static GLuint vertId = 0;
-
-/**
- * Fragment shader ID.
- */
-static GLuint fragId = 0;
-
-//********************************* Uniforms *********************************/
-
-/**
- * Quad rotation angle ID.
- */
-static GLint uRotId = -1;
-
-/**
- * Draw colour ID.
- */
-static GLint uTx0Id = -1;
-
-//******************************* Shader Source ******************************/
-
-/**
- * Vertex shader to draw texture mapped polys with an applied rotation.
- */
-static GLchar const vertShader2D[] =
-#if GL_ES_VERSION_2_0
-	"#version 100\n"
-	"precision mediump float;\n"
-#else
-	"#version 120\n"
-#endif
-	"uniform   float uRot;"	// rotation
-	"attribute vec2  aPos;"	// vertex position coords
-	"attribute vec2  aUV0;"	// vertex texture UV0
-	"varying   vec2  vUV0;"	// (passed to fragment shader)
-	"void main() {"
-	"	float cosA = cos(radians(uRot));"
-	"	float sinA = sin(radians(uRot));"
-	"	mat3 rot = mat3(cosA, -sinA, 0.0,"
-	"					sinA,  cosA, 0.0,"
-	"					0.0,   0.0,  1.0);"
-	"	gl_Position = vec4(rot * vec3(aPos, 1.0), 1.0);"
-	"	vUV0 = aUV0;"
-	"}";
-
-/**
- * Fragment shader for the above polys.
- */
-static GLchar const fragShader2D[] =
-#if GL_ES_VERSION_2_0
-	"#version 100\n"
-	"precision mediump float;\n"
-#else
-	"#version 120\n"
-#endif
-	"uniform sampler2D uTx0;"
-	"varying vec2      vUV0;" // (passed from fragment shader)
-	"void main() {"
-	"	gl_FragColor = texture2D(uTx0, vUV0);"
-	"}";
-
-/**
- * Helper to compile a shader.
- * 
- * \param type shader type
- * \param text shader source
- * \return the shader ID (or zero if compilation failed)
- */
-static GLuint compileShader(GLenum const type, const GLchar* text) {
-	GLuint shader = glCreateShader(type);
-	if (shader) {
-		glShaderSource (shader, 1, &text, NULL);
-		glCompileShader(shader);
-		GLint compiled;
-		glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
-		if (compiled) {
-			return shader;
-		} else {
-			GLint logLen;
-			glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &logLen);
-			if (logLen > 1) {
-				GLchar*  logStr = malloc(logLen);
-				glGetShaderInfoLog(shader, logLen, NULL, logStr);
-			#ifndef NDEBUG
-				printf("Shader compilation error: %s\n", logStr);
-			#endif
-				free(logStr);
-			}
-			glDeleteShader(shader);
-		}
-	}
-	return 0;
-}
-
-//********************************** Helpers *********************************/
-
-/**
- * Vertex position index.
- */
-#define GL_VERT_POSXY_ID 0
-
-/**
- * Vertex UV0 index.
- */
-#define GL_VERT_TXUV0_ID 1
-
- /**
-  * \c GL vec2 storage type.
-  */
-struct vec2 {
-	float x;
-	float y;
-};
-
-/**
- * Combined 2D vertex and 2D texture coordinates.
- */
-struct posTex2d {
-	struct vec2 pos;
-	struct vec2 uv0;
-};
-
-//****************************************************************************/
-
-/**
- * Current quad rotation angle (in degrees, updated per frame).
- */
-static float rotDeg = 0.0f;
-
-/**
- * Emscripten (single) GL context.
- */
-static EMSCRIPTEN_WEBGL_CONTEXT_HANDLE glCtx = 0;
-
-/**
- * Emscripten resize handler.
- */
-static EM_BOOL resize(int type, const EmscriptenUiEvent* e, void* data) {
-	double surfaceW;
-	double surfaceH;
-	if (emscripten_get_element_css_size   ("#canvas", &surfaceW, &surfaceH) == EMSCRIPTEN_RESULT_SUCCESS) {
-		emscripten_set_canvas_element_size("#canvas",  surfaceW,  surfaceH);
-		if (glCtx) {
-			glViewport(0, 0, (int) surfaceW, (int) surfaceH);
-		}
-	}
-	(void) type;
-	(void) data;
-	(void) e;
-	return EM_FALSE;
-}
-
-/**
- * Boilerplate to create a WebGL context.
- */
-static EM_BOOL initContext() {
-	// Default attributes
-	EmscriptenWebGLContextAttributes attr;
-	emscripten_webgl_init_context_attributes(&attr);
-	if ((glCtx = emscripten_webgl_create_context("#canvas", &attr))) {
-		// Bind the context and fire a resize to get the initial size
-		emscripten_webgl_make_context_current(glCtx);
-		emscripten_set_resize_callback(EMSCRIPTEN_EVENT_TARGET_DOCUMENT, NULL, EM_FALSE, resize);
-		resize(0, NULL, NULL);
-		return EM_TRUE;
-	}
-	return EM_FALSE;
-}
-
-/**
- * Called once per frame (clears the screen and draws the rotating quad).
- */
-static void tick() {
-	glClearColor(1.0f, 0.0f, 1.0f, 1.0f);
-	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
-
-	if (uRotId >= 0) {
-		glUniform1f(uRotId, rotDeg);
-		rotDeg += 0.1f;
-		if (rotDeg >= 360.0f) {
-			rotDeg -= 360.0f;
-		}
-	}
-
-	glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
-	glFlush();
-}
-
-/**
- * Creates the GL context, shaders and quad data, decompresses the Zstd data
- * and 'uploads' the resulting texture.
- * 
- * As a (naive) comparison, removing Zstd and building with "-Os -g0 s WASM=1
- * -lGL emscripten.c" results in a 15kB WebAssembly file; re-adding Zstd
- * increases the Wasm by 26kB.
- */
-int main() {
-	if (initContext()) {
-		// Compile shaders and set the initial GL state
-		if ((progId = glCreateProgram())) {
-			 vertId = compileShader(GL_VERTEX_SHADER,   vertShader2D);
-			 fragId = compileShader(GL_FRAGMENT_SHADER, fragShader2D);
-			 
-			 glBindAttribLocation(progId, GL_VERT_POSXY_ID, "aPos");
-			 glBindAttribLocation(progId, GL_VERT_TXUV0_ID, "aUV0");
-			 
-			 glAttachShader(progId, vertId);
-			 glAttachShader(progId, fragId);
-			 glLinkProgram (progId);
-			 glUseProgram  (progId);
-			 uRotId = glGetUniformLocation(progId, "uRot");
-			 uTx0Id = glGetUniformLocation(progId, "uTx0");
-			 if (uTx0Id >= 0) {
-				 glUniform1i(uTx0Id, 0);
-			 }
-			
-			 glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
-			 glEnable(GL_BLEND);
-			 glDisable(GL_DITHER);
-
-			 glCullFace(GL_BACK);
-			 glEnable(GL_CULL_FACE);
-		}
-		
-		GLuint vertsBuf = 0;
-		GLuint indexBuf = 0;
-		GLuint txName   = 0;
-		// Create the textured quad (vert positions then UVs)
-		struct posTex2d verts2d[] = {
-			{{-0.85f, -0.85f}, {0.0f, 0.0f}}, // BL
-			{{ 0.85f, -0.85f}, {1.0f, 0.0f}}, // BR
-			{{-0.85f,  0.85f}, {0.0f, 1.0f}}, // TL
-			{{ 0.85f,  0.85f}, {1.0f, 1.0f}}, // TR
-		};
-		uint16_t index2d[] = {
-			0, 1, 2,
-			2, 1, 3,
-		};
-		glGenBuffers(1, &vertsBuf);
-		glBindBuffer(GL_ARRAY_BUFFER, vertsBuf);
-		glBufferData(GL_ARRAY_BUFFER,
-			sizeof(verts2d), verts2d, GL_STATIC_DRAW);
-		glVertexAttribPointer(GL_VERT_POSXY_ID, 2,
-			GL_FLOAT, GL_FALSE, sizeof(struct posTex2d), 0);
-		glVertexAttribPointer(GL_VERT_TXUV0_ID, 2,
-			GL_FLOAT, GL_FALSE, sizeof(struct posTex2d),
-				(void*) offsetof(struct posTex2d, uv0));
-		glGenBuffers(1, &indexBuf);
-		glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indexBuf);
-		glBufferData(GL_ELEMENT_ARRAY_BUFFER,
-			sizeof(index2d), index2d, GL_STATIC_DRAW);
-		glEnableVertexAttribArray(GL_VERT_POSXY_ID);
-		glEnableVertexAttribArray(GL_VERT_TXUV0_ID);
-		
-		// Decode the Zstd data and create the texture
-		if (ZSTD_decompress(dstDxt1, DXT1_256x256, srcZstd, sizeof srcZstd) == DXT1_256x256) {
-			glGenTextures(1, &txName);
-			glBindTexture(GL_TEXTURE_2D, txName);
-			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-			glCompressedTexImage2D(GL_TEXTURE_2D, 0,
-				GL_COMPRESSED_RGB_S3TC_DXT1_EXT,
-					256, 256, 0, DXT1_256x256, dstDxt1);
-		} else {
-			printf("Failed to decode Zstd data\n");
-		}
-		emscripten_set_main_loop(tick, 0, EM_FALSE);
-		emscripten_exit_with_live_runtime();
-	}
-	return EXIT_FAILURE;
-}

build/single_file_libs/examples/roundtrip.c

@@ -1,83 +0,0 @@
-/*
- * \file roundtrip.c
- * In this example we include \c zstd.h and compile separately the amalgamated
- * \c zstd.c:
- * \code
- *	cc -Wall -Wextra -Werror -I. -Os -g0 zstd.c examples/roundtrip.c 
- * \endcode
- *
- * \author Carl Woffenden, Numfum GmbH (released under a CC0 license)
- */
-
-#include <stddef.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "zstd.h"
-
-//************************** Test Data (DXT texture) **************************/
-
-/**
- * Raw test data (borrowed from the Emscripten example).
- * \n
- * See \c testcard.png for the original.
- */
-static uint8_t const rawData[] = {
-#include "testcard-dxt1.inl"
-};
-
-#ifndef ZSTD_VERSION_MAJOR
-/*
- * For the case where the decompression library hasn't been included we add
- * dummy functions to fake the process and stop the buffers being optimised out.
- */
-size_t ZSTD_compressBound(size_t maxSrc) {
-	return maxSrc + 32;
-}
-int ZSTD_maxCLevel(void) {
-	return 20;
-}
-size_t ZSTD_compress(void* dst, size_t dstLen, const void* src, size_t srcLen, int level) {
-	return (memcmp(dst, src, (srcLen < dstLen) ? srcLen : dstLen)) ? level : dstLen;
-}
-unsigned ZSTD_isError(size_t code) {
-	return ((int) code) < 0;
-}
-size_t ZSTD_decompress(void* dst, size_t dstLen, const void* src, size_t srcLen) {
-	return (memcmp(dst, src, (srcLen < dstLen) ? srcLen : dstLen)) ? 0 : dstLen;
-}
-#endif
-
-//*****************************************************************************/
-
-/**
- * Simple single-file test to compress \c rawData, decompress the result, then
- * compare the decompressed version with the original.
- */
-int main() {
-	size_t bounds = ZSTD_compressBound(sizeof rawData);
-	void* compBuf = malloc(bounds);
-	void* testBuf = malloc(sizeof rawData);
-	int compare   = -1;
-	if (compBuf && testBuf) {
-		size_t compSize = ZSTD_compress(compBuf, bounds, rawData, sizeof rawData, ZSTD_maxCLevel());
-		if (!ZSTD_isError(compSize)) {
-			printf("Compression: PASSED (size: %lu, uncompressed: %lu)\n", (unsigned long) compSize, (unsigned long) (sizeof rawData));
-			size_t decSize = ZSTD_decompress(testBuf, sizeof rawData, compBuf, compSize);
-			if (!ZSTD_isError(decSize)) {
-				printf("Decompression: PASSED\n");
-				compare = memcmp(rawData, testBuf, decSize);
-				printf("Byte comparison: %s\n", (compare == 0) ? "PASSED" : "FAILED");
-			} else {
-				printf("Decompression: FAILED\n");
-			}
-		} else {
-			printf("Compression: FAILED\n");
-		}
-		free(compBuf);
-		free(testBuf);
-	}
-	return (compare == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
-}

build/single_file_libs/examples/shell.html

@@ -1,31 +0,0 @@
-<!doctype html>
-<html>
-<head>
-	<meta charset="utf-8">
-	<meta name="viewport" content="width=device-width, maximum-scale=1.0, user-scalable=no, viewport-fit=cover" />
-	<meta name="apple-mobile-web-app-capable" content="yes" />
-	<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
-	<title>Emscripten Shell</title>
-	<style>
-body   {background:#333; font-family:"Verdana","Helvetica Neue","Helvetica","Arial",sans-serif; margin:1em 0;}
-#canvas{position:absolute; top:0px; left:0px; border:none; margin:0; width: 100%; height: 100%; overflow: hidden; display: block;}
-	</style>
-</head>
-<body>
-	<canvas id="canvas" oncontextmenu="event.preventDefault()"></canvas>
-	<script>
-function trace(msg) {
-	console.log(msg);
-}
-
-var Module = {
-	print: trace, printErr: trace, setStatus: trace, monitorRunDependencies: trace,
-	canvas: (function() {
-		return document.getElementById("canvas");
-	})(),
-	preRun: [], postRun: [],
-};
-</script>
-{{{ SCRIPT }}}
-</body>
-</html>

build/single_file_libs/examples/simple.c

@@ -1,75 +0,0 @@
-/**
- * \file simple.c
- * Simple standalone example of using the single-file \c zstddeclib.
- * 
- * \note In this simple example we include the amalgamated source and compile
- * just this single file, but we could equally (and more conventionally)
- * include \c zstd.h and compile both this file and \c zstddeclib.c (the
- * resulting binaries differ slightly in size but perform the same).
- * 
- * \author Carl Woffenden, Numfum GmbH (released under a CC0 license)
- */
-
-#include <stddef.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "../zstddeclib.c"
-
-//************************* Test Data (DXT texture) **************************/
-
-/**
- * Raw 256x256 DXT1 data (used to compare the result).
- * \n
- * See \c testcard.png for the original.
- */
-static uint8_t const rawDxt1[] = {
-#include "testcard-dxt1.inl"
-};
-
-/**
- * Zstd compressed version of \c #rawDxt1.
- * \n
- * See \c testcard.png for the original.
- */
-static uint8_t const srcZstd[] = {
-#include "testcard-zstd.inl"
-};
-
-/**
- * Destination for decoding \c #srcZstd.
- */
-static uint8_t dstDxt1[sizeof rawDxt1] = {};
-
-#ifndef ZSTD_VERSION_MAJOR
-/**
- * For the case where the decompression library hasn't been included we add a
- * dummy function to fake the process and stop the buffers being optimised out.
- */
-size_t ZSTD_decompress(void* dst, size_t dstLen, const void* src, size_t srcLen) {
-	return (memcmp(dst, src, (srcLen < dstLen) ? srcLen : dstLen)) ? 0 : dstLen;
-}
-#endif
-
-//****************************************************************************/
-
-/**
- * Simple single-file test to decompress \c #srcZstd into \c # dstDxt1 then
- * compare the resulting bytes with \c #rawDxt1.
- * \n
- * As a (naive) comparison, removing Zstd and building with "-Os -g0 simple.c"
- * results in a 44kB binary (macOS 10.14, Clang 10); re-adding Zstd increases
- * the binary by 56kB (after calling \c strip).
- */
-int main() {
-	size_t size = ZSTD_decompress(dstDxt1, sizeof dstDxt1, srcZstd, sizeof srcZstd);
-	int compare = memcmp(rawDxt1, dstDxt1, sizeof dstDxt1);
-	printf("Decompressed size: %s\n", (size == sizeof dstDxt1) ? "PASSED" : "FAILED");
-	printf("Byte comparison: %s\n", (compare == 0) ? "PASSED" : "FAILED");
-	if (size == sizeof dstDxt1 && compare == 0) {
-		return EXIT_SUCCESS;
-	}
-	return EXIT_FAILURE;
-}

build/single_file_libs/examples/testcard-zstd.inl

@@ -1,261 +0,0 @@
-0x28, 0xb5, 0x2f, 0xfd, 0x60, 0x00, 0x7f, 0x6d, 0x61, 0x00, 0x0a, 0x66,
-0xec, 0x17, 0x48, 0x60, 0x1c, 0x5a, 0xc9, 0x5d, 0x1a, 0x38, 0x07, 0xe8,
-0xc5, 0x82, 0x99, 0x68, 0xe6, 0x95, 0x45, 0x58, 0x0d, 0x0c, 0xf3, 0x36,
-0xc8, 0xd9, 0x0f, 0x46, 0x2d, 0x68, 0x11, 0xf8, 0x31, 0x10, 0xa1, 0x1a,
-0x2f, 0x99, 0x5c, 0x84, 0xfd, 0x92, 0x02, 0xe6, 0x3b, 0x44, 0x9b, 0x01,
-0x5d, 0x92, 0xff, 0x38, 0x26, 0x00, 0x6a, 0x6b, 0xc3, 0x53, 0xb2, 0x0c,
-0x25, 0xf3, 0xd8, 0x59, 0x68, 0x9b, 0x14, 0x8a, 0x89, 0x75, 0x18, 0x03,
-0x1d, 0xc9, 0x0f, 0x63, 0x01, 0x73, 0x01, 0x72, 0x01, 0x4f, 0x66, 0x31,
-0x58, 0x0f, 0x97, 0x4b, 0x0c, 0x4c, 0x06, 0xac, 0x07, 0x0b, 0x68, 0xd4,
-0xad, 0x80, 0x64, 0x13, 0x74, 0xa1, 0x12, 0x16, 0x58, 0xcf, 0x1a, 0x95,
-0x5f, 0x0d, 0x26, 0x55, 0xd0, 0x9c, 0xf4, 0x52, 0x35, 0x2e, 0x20, 0xc1,
-0x06, 0x69, 0x03, 0x0a, 0x93, 0x83, 0x5e, 0x27, 0x9b, 0x4c, 0x6d, 0xee,
-0x87, 0x03, 0x30, 0x6c, 0x46, 0xd7, 0x50, 0x5c, 0xca, 0xe6, 0xa6, 0x4d,
-0xa8, 0xf6, 0xab, 0xd7, 0x0e, 0x27, 0x27, 0x90, 0xc4, 0xb2, 0xd1, 0x10,
-0xfa, 0x43, 0x82, 0xc8, 0xf2, 0xe5, 0xff, 0xff, 0xd5, 0x52, 0x62, 0x43,
-0x87, 0x26, 0x2a, 0x05, 0x70, 0x0e, 0xb0, 0x2f, 0xc4, 0x56, 0xef, 0xb5,
-0xca, 0xb8, 0x53, 0xb7, 0x96, 0x0e, 0xe7, 0x00, 0x2c, 0xa8, 0xda, 0x3b,
-0x07, 0x70, 0xa7, 0x78, 0x38, 0x60, 0x87, 0x7a, 0x01, 0x3b, 0x75, 0xec,
-0xfa, 0x77, 0xe2, 0x46, 0x94, 0x61, 0x8e, 0x0d, 0x0c, 0xfb, 0xe7, 0x8b,
-0x13, 0x50, 0x31, 0xa9, 0x27, 0xcd, 0x27, 0xef, 0x6b, 0xa6, 0xab, 0x9c,
-0x4d, 0x95, 0x6c, 0x3a, 0xbb, 0x8e, 0x96, 0x92, 0x18, 0x5a, 0x7c, 0x4f,
-0xff, 0x7b, 0x38, 0xf2, 0xdb, 0x86, 0xde, 0xff, 0x1f, 0x2f, 0x21, 0x86,
-0x7d, 0xbf, 0x45, 0xd0, 0x6e, 0x77, 0x0a, 0xee, 0x0a, 0xee, 0x14, 0x9a,
-0xb8, 0x84, 0xf3, 0xac, 0xbe, 0xc8, 0x7f, 0x8d, 0xff, 0xff, 0xcf, 0x2a,
-0xfb, 0x69, 0xfc, 0xfb, 0xfd, 0x7a, 0x10, 0x22, 0x36, 0xfc, 0xff, 0x3f,
-0xcf, 0xd0, 0xf1, 0x7f, 0xfe, 0xff, 0x3d, 0x24, 0xdf, 0x78, 0x4a, 0xff,
-0xda, 0x9c, 0x39, 0xcf, 0xef, 0xe7, 0xfd, 0x52, 0x98, 0xb5, 0x40, 0x92,
-0xee, 0xdd, 0x99, 0xf5, 0x53, 0x5b, 0x65, 0x6b, 0xb5, 0xd8, 0x7b, 0xae,
-0xfa, 0xc1, 0x0f, 0x0c, 0x7f, 0x4f, 0x55, 0xa3, 0xad, 0x2c, 0xa0, 0xbd,
-0xf7, 0x2a, 0x0e, 0xe8, 0xbd, 0xc7, 0x5e, 0xf5, 0xd8, 0x54, 0x9e, 0x56,
-0xa3, 0xd6, 0x59, 0xd5, 0xfe, 0x1f, 0xc0, 0x30, 0x8c, 0xfc, 0x46, 0x04,
-0xae, 0x60, 0xbc, 0xe8, 0xcf, 0xec, 0x3d, 0xde, 0xf9, 0xf0, 0xfe, 0xef,
-0x7d, 0xcc, 0xf7, 0x2b, 0xe5, 0x1b, 0x70, 0xff, 0xff, 0x7e, 0x3f, 0x6e,
-0xe4, 0x02, 0x07, 0xfc, 0x1b, 0x7a, 0xff, 0xe7, 0x58, 0xfc, 0x7e, 0x3a,
-0xdc, 0x97, 0xfd, 0x57, 0xef, 0xa3, 0xfc, 0x2a, 0xc7, 0x4d, 0xf3, 0xcb,
-0x9d, 0xce, 0xac, 0xfe, 0xeb, 0x2e, 0x86, 0xb9, 0x69, 0x54, 0xef, 0xf9,
-0x55, 0xcf, 0xff, 0x48, 0x24, 0x72, 0x3a, 0x9d, 0x72, 0x2f, 0x2f, 0x2f,
-0xff, 0x3f, 0xe7, 0x01, 0x6c, 0x4d, 0x6c, 0xcd, 0x2d, 0x5b, 0x53, 0xb7,
-0x59, 0x22, 0x08, 0x0b, 0xa7, 0x92, 0x15, 0x75, 0x93, 0xb0, 0x5d, 0xaf,
-0x2a, 0x63, 0x95, 0x1d, 0x25, 0xd2, 0xd2, 0xa8, 0x1c, 0x84, 0xc9, 0xdc,
-0x72, 0xba, 0xd7, 0xfc, 0x69, 0xf5, 0xc7, 0x19, 0xa9, 0xbe, 0xfa, 0x26,
-0x55, 0x25, 0x75, 0xb7, 0x60, 0xa3, 0xd8, 0x68, 0x54, 0xb7, 0x1b, 0xa5,
-0x54, 0x62, 0xb1, 0x49, 0xde, 0xe2, 0xac, 0xa2, 0xe8, 0x7b, 0xff, 0x5f,
-0x75, 0x4e, 0xb8, 0xa2, 0xdd, 0x6a, 0xb7, 0xda, 0x6e, 0x2e, 0x04, 0xcd,
-0x08, 0x2f, 0xec, 0x8e, 0x49, 0xaf, 0x49, 0x6f, 0x8b, 0x4f, 0x2e, 0x1a,
-0xc5, 0x62, 0x7b, 0x6b, 0x3e, 0x32, 0x3e, 0x32, 0xbe, 0x08, 0x35, 0x4d,
-0x63, 0x93, 0xa6, 0xc8, 0x42, 0xe6, 0x21, 0xcc, 0x59, 0xc8, 0x4c, 0xe5,
-0x86, 0xe1, 0x03, 0x06, 0xa4, 0xec, 0xff, 0xb7, 0x78, 0x7e, 0x62, 0x43,
-0xc7, 0x2c, 0x50, 0x30, 0x4a, 0xc8, 0x9b, 0xf3, 0xbf, 0xe6, 0x62, 0xa0,
-0x50, 0xa6, 0x9c, 0xe3, 0x6e, 0x5b, 0xaf, 0x77, 0x8b, 0xbb, 0xe1, 0x70,
-0xaa, 0xaa, 0xaa, 0x92, 0xb4, 0x52, 0xad, 0x14, 0x87, 0x93, 0x0b, 0xe6,
-0x82, 0x39, 0x11, 0xb9, 0x20, 0x9a, 0x16, 0x34, 0x22, 0x68, 0xb2, 0x68,
-0xb2, 0x76, 0xd3, 0xe8, 0x6e, 0xda, 0x6b, 0x62, 0x34, 0x2e, 0x8d, 0xbd,
-0x2d, 0x3d, 0x6b, 0x4c, 0x26, 0x33, 0xda, 0x33, 0xf3, 0x91, 0x51, 0x46,
-0x79, 0xbd, 0x3e, 0x39, 0x9f, 0xcf, 0xd2, 0xb8, 0x5c, 0xfa, 0x22, 0xf8,
-0x8e, 0xb6, 0xbe, 0x08, 0x40, 0x14, 0x49, 0x40, 0x14, 0xf2, 0x0c, 0x2d,
-0x30, 0x85, 0x3c, 0x63, 0x29, 0xd3, 0x98, 0x85, 0x6c, 0xb5, 0xdb, 0xad,
-0x5c, 0x63, 0x9e, 0x72, 0xcf, 0x43, 0xe6, 0xaf, 0x77, 0xa6, 0xe2, 0x21,
-0x0c, 0x4d, 0xd3, 0x49, 0x1e, 0xc2, 0x14, 0x6f, 0xee, 0xdb, 0x7b, 0x7b,
-0x08, 0xb3, 0xa4, 0x60, 0x3b, 0x9d, 0x6e, 0x8b, 0x37, 0x4b, 0x0a, 0x74,
-0x35, 0x33, 0xbc, 0xf9, 0x64, 0x85, 0x63, 0x32, 0x29, 0x20, 0x59, 0x0c,
-0x3c, 0x96, 0x67, 0x62, 0xb7, 0x8a, 0x92, 0x4d, 0xa0, 0xd3, 0xf3, 0xd1,
-0x85, 0x80, 0x38, 0xcb, 0x64, 0x60, 0xc9, 0xb5, 0xaf, 0x97, 0x8d, 0x20,
-0x45, 0x28, 0xb8, 0xab, 0xe8, 0xc9, 0x0a, 0x88, 0x1f, 0xd6, 0x47, 0x54,
-0xf1, 0xd3, 0xfb, 0x62, 0xa7, 0xfd, 0xf2, 0x8b, 0xfd, 0xb6, 0xe4, 0x2e,
-0xb6, 0x91, 0x73, 0x1c, 0xd0, 0x7b, 0xba, 0x83, 0xc9, 0xac, 0x51, 0x39,
-0x92, 0xc5, 0x4f, 0x30, 0x1e, 0x2e, 0xd5, 0xf1, 0xa8, 0xa6, 0xa5, 0x80,
-0x70, 0xb9, 0xbc, 0xb7, 0xc2, 0x52, 0x32, 0x6c, 0xe3, 0x3d, 0xed, 0x41,
-0xa4, 0x4b, 0x31, 0x2a, 0xe6, 0x62, 0x11, 0x19, 0x95, 0x73, 0x1d, 0xbf,
-0xe1, 0x6c, 0xfc, 0x47, 0x75, 0x6c, 0x37, 0x63, 0x02, 0xf8, 0x34, 0x40,
-0x9a, 0x00, 0x1d, 0xf7, 0x32, 0x56, 0x77, 0xda, 0x5b, 0x9f, 0x9f, 0x0f,
-0xbb, 0x91, 0x5b, 0xbd, 0xe7, 0x58, 0x82, 0x4a, 0x20, 0xcd, 0x4f, 0x47,
-0x15, 0xf3, 0x51, 0xf1, 0x43, 0x51, 0x10, 0x96, 0xae, 0xba, 0xf7, 0x21,
-0x50, 0xef, 0x55, 0x27, 0x0c, 0x1f, 0xe0, 0x54, 0xf8, 0xc9, 0x69, 0xef,
-0xb9, 0x53, 0xf7, 0x83, 0x73, 0x9d, 0xce, 0x86, 0x07, 0x83, 0x44, 0x61,
-0x37, 0x35, 0x35, 0x33, 0x4e, 0x33, 0x33, 0x3e, 0x9f, 0x50, 0x48, 0x24,
-0xe6, 0xd0, 0x79, 0x49, 0xc3, 0x2d, 0xa0, 0x46, 0x31, 0x9a, 0x72, 0xc3,
-0x84, 0xff, 0x7a, 0x95, 0xbb, 0x00, 0x22, 0xcc, 0x14, 0x00, 0x04, 0xac,
-0x60, 0x64, 0x86, 0xe4, 0x6f, 0xb1, 0x58, 0xf4, 0xdc, 0xb0, 0x05, 0x00,
-0x72, 0x38, 0xf8, 0xce, 0xce, 0x8e, 0xcf, 0x37, 0x33, 0x43, 0x24, 0x0a,
-0x85, 0x33, 0x35, 0x35, 0x37, 0xc2, 0xa8, 0x28, 0x27, 0x1b, 0x9d, 0xce,
-0x29, 0x18, 0xe4, 0xfc, 0x09, 0x53, 0xa5, 0x51, 0xad, 0x74, 0x79, 0x7b,
-0xb5, 0x4a, 0x65, 0x94, 0x36, 0x89, 0xf5, 0x62, 0x9b, 0xd8, 0x9a, 0xe8,
-0x2b, 0xff, 0xa1, 0x73, 0xfe, 0x2e, 0x2c, 0x41, 0x45, 0x37, 0xef, 0x6e,
-0x7b, 0x38, 0xca, 0xa5, 0xd2, 0xb8, 0x1c, 0x3b, 0x96, 0x21, 0xbb, 0x5d,
-0xad, 0xd1, 0xdb, 0x1c, 0xf6, 0x5e, 0x4b, 0xd9, 0x59, 0x1b, 0x67, 0xf5,
-0x7a, 0x7c, 0x9a, 0x91, 0x3e, 0x8e, 0xe3, 0xee, 0x7b, 0xb9, 0xa4, 0xb9,
-0xf5, 0x70, 0xee, 0x1d, 0x4e, 0x4f, 0xcc, 0xd6, 0x7b, 0x07, 0x71, 0x48,
-0xf2, 0x06, 0xd0, 0x10, 0x82, 0x21, 0xe4, 0x55, 0x2e, 0xa5, 0x1d, 0xbe,
-0x48, 0x8c, 0x69, 0xbb, 0x24, 0x40, 0x68, 0x9b, 0xba, 0x5a, 0x5a, 0xa7,
-0xe2, 0xd1, 0xac, 0xc2, 0xd8, 0x87, 0x9c, 0xe3, 0x78, 0xee, 0xa6, 0xcd,
-0xdd, 0x68, 0x6e, 0xdd, 0xdb, 0x2e, 0xc6, 0xbb, 0x8b, 0xe9, 0xc1, 0xd9,
-0xf6, 0x62, 0x7e, 0x72, 0x7e, 0x72, 0x34, 0xe4, 0x68, 0xc8, 0x11, 0x32,
-0x10, 0x32, 0x20, 0x32, 0xf0, 0x12, 0x19, 0x90, 0xe0, 0x45, 0x91, 0xe0,
-0x25, 0x83, 0xba, 0xc9, 0x20, 0x26, 0x87, 0xa5, 0x72, 0xa9, 0x64, 0x72,
-0x80, 0x21, 0x54, 0x13, 0xeb, 0x29, 0x18, 0x42, 0x34, 0x84, 0x94, 0x34,
-0x84, 0xa4, 0x1d, 0xa4, 0x1d, 0x82, 0x1c, 0xef, 0xaf, 0x0e, 0x63, 0x47,
-0x6d, 0x10, 0xb9, 0xec, 0xd8, 0x2d, 0x3b, 0x9e, 0x21, 0xb1, 0x67, 0x48,
-0x34, 0x24, 0x1a, 0x52, 0xdb, 0xa4, 0x6d, 0x62, 0xd3, 0xfa, 0xff, 0xfa,
-0x03, 0x34, 0xef, 0x9d, 0xc9, 0xe1, 0x5d, 0xec, 0xf5, 0xf1, 0x79, 0x8c,
-0x97, 0xd2, 0x24, 0xc1, 0x2d, 0xe0, 0x39, 0x16, 0x1e, 0xa9, 0x41, 0xc3,
-0xbf, 0x4a, 0xd9, 0x3c, 0xea, 0x77, 0x96, 0x55, 0xe6, 0x95, 0xc3, 0xf1,
-0x8e, 0x7b, 0x4f, 0xad, 0x61, 0xf8, 0xe7, 0x01, 0xad, 0x46, 0xf5, 0x2c,
-0xac, 0x55, 0x2c, 0x94, 0xaa, 0x46, 0xfb, 0x5e, 0xcd, 0xaa, 0x1f, 0x78,
-0x4f, 0x2f, 0xd1, 0xc9, 0x02, 0xd6, 0x2c, 0x67, 0xef, 0x3f, 0x54, 0xab,
-0xda, 0x03, 0x79, 0x1f, 0xab, 0xfd, 0x0c, 0x38, 0x3c, 0xbc, 0xe1, 0xd5,
-0x01, 0xf6, 0xfb, 0xfb, 0xf1, 0x70, 0xee, 0xfd, 0x90, 0x13, 0x97, 0xc4,
-0xbc, 0x08, 0xe7, 0x4b, 0x88, 0x34, 0xf7, 0x56, 0x1e, 0x0c, 0xdb, 0xe4,
-0x9c, 0x78, 0xf1, 0xf4, 0x62, 0x4c, 0xb5, 0xf7, 0xdd, 0xd9, 0x4c, 0x5a,
-0x69, 0xa6, 0x36, 0x27, 0x03, 0xbe, 0x86, 0xc2, 0x72, 0xa2, 0x60, 0x73,
-0xf1, 0xe0, 0x17, 0x50, 0xb5, 0x93, 0x81, 0xac, 0xf1, 0xc9, 0xd4, 0x66,
-0x73, 0x71, 0xce, 0x63, 0xa8, 0x60, 0x98, 0xda, 0x86, 0x46, 0x72, 0xec,
-0x54, 0xc1, 0xe6, 0x8a, 0x10, 0x23, 0x2d, 0x0c, 0xdd, 0x44, 0x0b, 0xa0,
-0x44, 0xa4, 0x9d, 0x0e, 0x64, 0x31, 0x30, 0x45, 0xb1, 0x97, 0x4c, 0x6d,
-0x9f, 0x43, 0x99, 0x71, 0xa8, 0x9a, 0xe6, 0xbd, 0x3a, 0xe1, 0xff, 0x7f,
-0x33, 0x61, 0xf1, 0x48, 0xda, 0x48, 0x28, 0x10, 0x23, 0x9b, 0x24, 0xeb,
-0xee, 0xbd, 0x35, 0x0e, 0x6e, 0x75, 0x23, 0x20, 0xd6, 0x95, 0x8c, 0xa5,
-0x24, 0xec, 0x44, 0x67, 0x52, 0x2e, 0x78, 0x2a, 0xba, 0x3e, 0x3a, 0x4e,
-0xc9, 0x5c, 0x23, 0xb0, 0xd5, 0xfb, 0x29, 0xaf, 0x9a, 0x0b, 0x90, 0x89,
-0xd8, 0xec, 0xa9, 0xa8, 0x13, 0xfc, 0x22, 0xfa, 0xf2, 0x74, 0x2f, 0x4e,
-0x35, 0xb0, 0x6d, 0x6c, 0xfd, 0xc4, 0xfe, 0xd0, 0x98, 0x3d, 0xe5, 0x43,
-0x0a, 0xd0, 0x33, 0x26, 0x3b, 0x12, 0x7d, 0x65, 0xa1, 0xff, 0xff, 0x6f,
-0x53, 0x0e, 0x28, 0x84, 0xa7, 0xa2, 0x2e, 0xf8, 0x4a, 0xb6, 0xa1, 0x47,
-0xf5, 0xd0, 0x13, 0x9d, 0xd9, 0x50, 0xef, 0x9f, 0x31, 0xb4, 0x13, 0x67,
-0xf9, 0x0a, 0x99, 0xb5, 0x80, 0xec, 0x4a, 0x1a, 0x59, 0x21, 0x3b, 0xce,
-0xc5, 0x7e, 0x96, 0x85, 0x92, 0xc5, 0xb0, 0x75, 0xaa, 0x41, 0x16, 0xa9,
-0xd3, 0xde, 0x13, 0x7d, 0xd9, 0x61, 0x30, 0x1c, 0x73, 0x61, 0x54, 0x90,
-0xcf, 0xd4, 0xe8, 0xfe, 0xbf, 0xbf, 0x36, 0x57, 0x26, 0x38, 0xab, 0x06,
-0xc4, 0x7e, 0x3c, 0x5b, 0x35, 0xd2, 0x7c, 0x2c, 0x0a, 0x82, 0xf2, 0xa8,
-0xf2, 0x8b, 0x48, 0xa9, 0x90, 0x00, 0x01, 0x10, 0x10, 0x14, 0x04, 0x00,
-0x32, 0xa2, 0x06, 0x82, 0x28, 0x90, 0xc2, 0xb4, 0x85, 0xda, 0x01, 0x52,
-0xe9, 0x18, 0x85, 0x60, 0x00, 0x00, 0x20, 0x0c, 0x00, 0x14, 0x41, 0x00,
-0x40, 0xa0, 0x04, 0x40, 0x00, 0x80, 0x50, 0x03, 0x01, 0x10, 0x18, 0x01,
-0x80, 0xd4, 0x14, 0x99, 0x01, 0xfd, 0x07, 0xf8, 0x16, 0x0e, 0xd9, 0x5d,
-0xa3, 0x70, 0xfe, 0xda, 0x17, 0xfa, 0xce, 0x46, 0x9a, 0x99, 0x81, 0x1a,
-0x39, 0xba, 0x63, 0xb1, 0x18, 0x11, 0x58, 0xd7, 0xc7, 0xba, 0x03, 0x3e,
-0x01, 0xf2, 0xf9, 0x4e, 0x12, 0xa3, 0x50, 0x7b, 0xaf, 0x7b, 0x60, 0x5c,
-0x83, 0x23, 0xd2, 0x60, 0x27, 0x84, 0xad, 0xb8, 0x02, 0xed, 0xfe, 0xb4,
-0x9c, 0x08, 0x9f, 0x49, 0xae, 0x55, 0x02, 0x94, 0xc0, 0x1b, 0x90, 0x75,
-0x0b, 0x90, 0xc4, 0xc7, 0x43, 0x9e, 0x67, 0x25, 0x70, 0x61, 0x0d, 0xb8,
-0x7a, 0x97, 0x43, 0xfc, 0xd1, 0x7e, 0x68, 0xed, 0x03, 0xb7, 0x1e, 0x75,
-0xe9, 0x4d, 0x7a, 0x23, 0x18, 0x37, 0x63, 0x6f, 0xab, 0x5f, 0x7c, 0x5b,
-0x1c, 0x05, 0xdf, 0x3f, 0x00, 0x86, 0x37, 0xa0, 0xfa, 0x0c, 0xe0, 0xed,
-0x35, 0x35, 0x2f, 0xd8, 0xd1, 0x75, 0xba, 0x37, 0x34, 0x7e, 0xb0, 0x84,
-0x2a, 0x01, 0x0c, 0x98, 0xed, 0x47, 0xf9, 0x86, 0x81, 0x74, 0x00, 0x5d,
-0x8b, 0x4c, 0x18, 0x8a, 0x31, 0xcd, 0xae, 0x07, 0x44, 0xb5, 0xd5, 0x07,
-0xa0, 0xdf, 0xf4, 0xfa, 0xa6, 0x42, 0xd0, 0x4f, 0x17, 0xd8, 0xdf, 0xb6,
-0x34, 0x44, 0xe3, 0x01, 0xc4, 0xb6, 0x2d, 0xb5, 0x56, 0xc6, 0x2a, 0x1f,
-0x05, 0x6c, 0x35, 0xe0, 0x09, 0x31, 0xef, 0x60, 0xfe, 0xaf, 0x07, 0x80,
-0x32, 0xa0, 0xe9, 0xd3, 0x96, 0x45, 0xa7, 0xaa, 0xb6, 0xfb, 0x03, 0x10,
-0xe3, 0x97, 0x96, 0x8d, 0x3a, 0x01, 0xdd, 0x58, 0x58, 0x78, 0x00, 0xab,
-0xff, 0x06, 0xa0, 0xd6, 0x01, 0x58, 0x08, 0xb7, 0xdc, 0x2d, 0xa7, 0xfb,
-0x22, 0xa8, 0x67, 0x00, 0xe3, 0xcf, 0x82, 0x43, 0xfc, 0x96, 0x1b, 0x40,
-0x63, 0xcf, 0x9d, 0x42, 0x5d, 0x66, 0x40, 0xaa, 0xaf, 0x28, 0x94, 0xd3,
-0x2a, 0xd4, 0x02, 0x13, 0xd2, 0xdf, 0x03, 0x9c, 0x60, 0x6b, 0x16, 0x94,
-0xb4, 0xbe, 0x62, 0xc2, 0x35, 0x60, 0x45, 0x09, 0x23, 0x5a, 0xe0, 0x85,
-0xb3, 0x03, 0x50, 0x68, 0x0c, 0x20, 0xa5, 0xf9, 0x94, 0xd2, 0x35, 0x80,
-0xad, 0x4c, 0x4e, 0x40, 0x41, 0x97, 0x92, 0x75, 0xbe, 0x0c, 0x03, 0x50,
-0x85, 0x08, 0xaf, 0x36, 0x00, 0x68, 0xaf, 0x09, 0xb3, 0x0c, 0x20, 0x4f,
-0x81, 0x6a, 0x6a, 0xf5, 0x0d, 0x70, 0x69, 0x00, 0x4c, 0xb4, 0x0f, 0x59,
-0xe7, 0x31, 0x0a, 0x45, 0x9f, 0xde, 0x90, 0xd6, 0x38, 0x80, 0x6b, 0x2c,
-0xb9, 0x2f, 0xd4, 0x01, 0x40, 0x14, 0xd5, 0xed, 0x8e, 0x01, 0x53, 0xbf,
-0x03, 0x18, 0x1e, 0xb0, 0xc1, 0x85, 0x32, 0xec, 0x78, 0x2b, 0xf0, 0xbb,
-0xbb, 0x6c, 0xf3, 0x4d, 0xdc, 0x73, 0x40, 0xfd, 0x10, 0x09, 0x9e, 0x20,
-0xe2, 0x12, 0x8c, 0xe0, 0xd2, 0xed, 0x80, 0x6b, 0xcc, 0x78, 0x20, 0x03,
-0xd0, 0x5e, 0x06, 0xf4, 0xb0, 0xc4, 0x0e, 0x15, 0x1d, 0x80, 0xb4, 0x76,
-0xdf, 0x49, 0x03, 0x50, 0x82, 0xad, 0xda, 0x8b, 0x5a, 0x61, 0xc2, 0x5e,
-0xb5, 0x1e, 0x46, 0xc0, 0xde, 0xaa, 0x0e, 0x15, 0x06, 0xd2, 0xf4, 0xb2,
-0xd1, 0xed, 0x38, 0x0a, 0x03, 0x18, 0x33, 0x1a, 0x80, 0x61, 0x3e, 0xec,
-0x7c, 0x74, 0xa8, 0x1d, 0x80, 0x1a, 0xce, 0x25, 0x1d, 0x41, 0xd1, 0xc1,
-0x03, 0x28, 0xb5, 0xaf, 0x72, 0x9c, 0x59, 0x7a, 0xe1, 0x7d, 0xc0, 0xa5,
-0x08, 0x1e, 0x18, 0x24, 0xfa, 0xbd, 0x99, 0x4a, 0x31, 0xa0, 0xea, 0xee,
-0xf8, 0x36, 0x60, 0x98, 0xc9, 0x10, 0xd1, 0xa7, 0x35, 0x00, 0x8d, 0x40,
-0x8e, 0x5a, 0x35, 0x0f, 0x80, 0xb1, 0xd4, 0x32, 0x79, 0x40, 0x34, 0x05,
-0x7e, 0x98, 0xc6, 0x80, 0x3e, 0x90, 0x01, 0x65, 0xf4, 0x80, 0x73, 0x08,
-0x64, 0xd7, 0x36, 0xc1, 0x7c, 0xc0, 0x5c, 0x75, 0x00, 0xc5, 0x09, 0x58,
-0x9c, 0x13, 0x01, 0x72, 0x37, 0x9b, 0x79, 0xe4, 0x05, 0xd1, 0x01, 0x04,
-0x98, 0x08, 0x74, 0xfd, 0xfc, 0x3f, 0x1c, 0x00, 0x73, 0x01, 0xfc, 0x1c,
-0xcc, 0x16, 0x43, 0x19, 0x1d, 0xac, 0x61, 0x4b, 0x11, 0xc2, 0xa0, 0xf2,
-0x01, 0x0b, 0x7b, 0x3b, 0xf4, 0xfc, 0x58, 0x5d, 0x2d, 0x5c, 0x01, 0x8c,
-0x62, 0x17, 0x78, 0xbe, 0x60, 0x8c, 0x01, 0x6f, 0x91, 0x49, 0x65, 0x54,
-0x92, 0xe9, 0x01, 0x1e, 0x10, 0x77, 0x35, 0x00, 0xa8, 0xd4, 0xc7, 0x71,
-0x07, 0xd8, 0xcd, 0xa3, 0x7d, 0x69, 0x20, 0xac, 0x07, 0x00, 0x35, 0xc7,
-0x62, 0xee, 0x8c, 0x7d, 0x0c, 0xb8, 0x43, 0x0e, 0x00, 0x08, 0xfb, 0xe7,
-0xec, 0x33, 0x37, 0x04, 0x80, 0x2d, 0x1d, 0xa6, 0x13, 0x34, 0x1b, 0x1d,
-0xc0, 0xca, 0x00, 0x92, 0xed, 0x2e, 0x56, 0xbe, 0x91, 0x80, 0x0c, 0x88,
-0xa6, 0x01, 0xdf, 0x7f, 0x90, 0x49, 0xed, 0x0c, 0xe0, 0x08, 0x73, 0x28,
-0x74, 0xc7, 0xe1, 0xb1, 0x03, 0x5d, 0xc5, 0xab, 0x61, 0x42, 0xdf, 0x03,
-0x43, 0xf3, 0x35, 0x04, 0xcf, 0xc6, 0x1d, 0x79, 0x07, 0x40, 0x22, 0xe4,
-0x68, 0x0d, 0x01, 0x95, 0xad, 0x72, 0x69, 0x00, 0x39, 0x9d, 0x53, 0x8f,
-0x13, 0x0d, 0xb0, 0x29, 0x79, 0x1a, 0x39, 0x20, 0x12, 0x28, 0x9b, 0x02,
-0x8f, 0x74, 0x90, 0x4c, 0xe8, 0xd1, 0x57, 0xf4, 0x01, 0x44, 0x04, 0xe0,
-0x0c, 0x82, 0x91, 0xc5, 0x4f, 0x8f, 0xc6, 0x00, 0x43, 0x85, 0x65, 0xc8,
-0xe6, 0x34, 0x1d, 0x80, 0xc0, 0xca, 0xdb, 0x57, 0x6c, 0x00, 0x72, 0x42,
-0x5f, 0xd0, 0x49, 0x57, 0x47, 0xd4, 0x97, 0x18, 0x18, 0x80, 0x68, 0x8e,
-0x0a, 0xf1, 0x6b, 0x34, 0xf1, 0x60, 0x2c, 0x41, 0x29, 0xd3, 0x3d, 0x55,
-0x95, 0xb1, 0x3c, 0xd4, 0x95, 0x42, 0xef, 0xe7, 0xca, 0x00, 0x2e, 0xce,
-0x25, 0xc2, 0xca, 0xf5, 0x00, 0x17, 0x3b, 0x8c, 0x42, 0x88, 0x03, 0xde,
-0x97, 0xe1, 0x3a, 0x74, 0xb0, 0x33, 0xe0, 0x8f, 0x47, 0xeb, 0x2a, 0x5f,
-0x36, 0x3e, 0x5a, 0xff, 0xc5, 0x80, 0xb9, 0x13, 0xa9, 0x1f, 0xf8, 0x86,
-0xc9, 0x51, 0xf8, 0x4c, 0xaa, 0xe1, 0x65, 0x80, 0xb0, 0x8b, 0x91, 0xec,
-0xcc, 0xbf, 0x70, 0x19, 0x98, 0x03, 0x10, 0xf0, 0x38, 0x40, 0xc4, 0x65,
-0xbe, 0x41, 0xb2, 0x58, 0x3f, 0xe0, 0xcc, 0x0e, 0x08, 0x2b, 0x73, 0xf4,
-0xdd, 0x86, 0x06, 0xa0, 0xc6, 0x8f, 0x1a, 0x32, 0x66, 0x50, 0x8e, 0xe1,
-0x59, 0x67, 0x00, 0xed, 0x66, 0x1d, 0xdd, 0xfa, 0x7b, 0xe2, 0x56, 0x89,
-0xd9, 0xa0, 0x4f, 0x41, 0x94, 0x28, 0xb8, 0xc6, 0xc7, 0x64, 0xde, 0x9b,
-0x64, 0x44, 0x33, 0x39, 0xb5, 0x6c, 0xb9, 0x42, 0xe7, 0x7e, 0x16, 0xd2,
-0x01, 0x12, 0x03, 0xb3, 0x48, 0x47, 0x6b, 0x75, 0x26, 0x19, 0x8c, 0xac,
-0x6f, 0xb1, 0x6f, 0xdc, 0x04, 0x27, 0x3a, 0x00, 0xd6, 0xae, 0xfa, 0xe1,
-0xf7, 0x30, 0xa4, 0xdb, 0xd5, 0x86, 0x5a, 0x07, 0x11, 0xde, 0xea, 0xf4,
-0xb0, 0x83, 0x16, 0xbb, 0xc6, 0x00, 0x6e, 0xf2, 0x6b, 0x40, 0x81, 0x01,
-0x67, 0x0e, 0xa9, 0x82, 0x23, 0x04, 0x34, 0xed, 0x02, 0xf5, 0xe4, 0x0e,
-0x58, 0xe8, 0x8a, 0x58, 0x57, 0xb0, 0x56, 0x65, 0x3d, 0x40, 0x64, 0x03,
-0x6e, 0x7b, 0x07, 0x20, 0x99, 0x90, 0x36, 0x95, 0x9f, 0xdf, 0x3d, 0xe8,
-0x00, 0xa0, 0x57, 0x8f, 0x6d, 0xa4, 0xb3, 0x1d, 0x7a, 0x06, 0xa8, 0x26,
-0x41, 0xb0, 0x8c, 0x9c, 0x10, 0x85, 0x6c, 0xb4, 0x31, 0xa6, 0x5b, 0x7a,
-0x10, 0x51, 0x15, 0x3c, 0xa2, 0x42, 0xd3, 0x23, 0x02, 0xc0, 0x17, 0x7e,
-0x03, 0x28, 0xba, 0xce, 0x9b, 0xae, 0xdd, 0x1a, 0x19, 0xd0, 0x15, 0xac,
-0xeb, 0x20, 0x3c, 0x3a, 0x00, 0xc6, 0xbb, 0x8a, 0xd5, 0x64, 0xc2, 0x21,
-0x1d, 0x6c, 0x15, 0x5a, 0xd3, 0x44, 0x98, 0x14, 0x95, 0xb3, 0xb7, 0xdd,
-0xa6, 0xea, 0x06, 0x54, 0x78, 0xc3, 0xe8, 0x79, 0x9b, 0x86, 0x29, 0x76,
-0x8b, 0x6b, 0xaa, 0x0d, 0xa8, 0x2f, 0x22, 0x2a, 0xeb, 0x68, 0x81, 0x6c,
-0x56, 0xfd, 0x79, 0xac, 0x79, 0x4b, 0xa0, 0x01, 0x3f, 0x17, 0x43, 0x82,
-0xb4, 0xd5, 0x00, 0x14, 0xb7, 0xf5, 0x00, 0xf4, 0x15, 0xa8, 0xd7, 0x4b,
-0xb1, 0xbc, 0xa8, 0x36, 0x98, 0xf0, 0x8c, 0xe7, 0xf4, 0x7b, 0x35, 0xd8,
-0xad, 0x0d, 0x5f, 0x9d, 0x96, 0xab, 0xed, 0x48, 0xe2, 0xdc, 0x1c, 0xbe,
-0x12, 0xfa, 0x41, 0x6f, 0xf5, 0x1e, 0xb6, 0x9f, 0xee, 0xac, 0x21, 0xf4,
-0xf6, 0x00, 0x38, 0xb1, 0x1f, 0xfd, 0xd0, 0x0e, 0xc7, 0xdd, 0xa0, 0x39,
-0x07, 0x8c, 0x35, 0x1f, 0x7e, 0xcc, 0xbf, 0xf6, 0xe0, 0x06, 0x66, 0x7d,
-0x10, 0x3f, 0xc5, 0x3e, 0xde, 0x42, 0xf9, 0x3d, 0x00, 0x54, 0x81, 0x67,
-0x8a, 0xe6, 0x63, 0x0d, 0x01, 0xd0, 0x31, 0xe0, 0x6e, 0xd0, 0xe1, 0x59,
-0xf6, 0x1b, 0xf7, 0x0d, 0x52, 0x06, 0x80, 0x61, 0x4f, 0xe8, 0x77, 0xdd,
-0x6f, 0x48, 0x20, 0x1d, 0xbb, 0x2a, 0x16, 0x8b, 0x54, 0x87, 0x92, 0x83,
-0xe6, 0x8f, 0x55, 0x59, 0x06, 0x00, 0xe9, 0xc5, 0xce, 0x21, 0x63, 0x87,
-0xaf, 0x86, 0xcc, 0xba, 0xd6, 0xe7, 0x00, 0xf6, 0x91, 0x92, 0x92, 0xea,
-0xe8, 0x42, 0x06, 0x69, 0x13, 0xf5, 0x00, 0xd0, 0xb0, 0xa7, 0xcb, 0x4c,
-0xb0, 0xd2, 0x2d, 0x28, 0x63, 0xf0, 0x6a, 0xc7, 0x80, 0x6a, 0x19, 0xb2,
-0x66, 0x51, 0xf3, 0xb1, 0x21, 0xa0, 0x48, 0xad, 0x1e, 0x80, 0x62, 0xaf,
-0x00, 0xf4, 0xa5, 0x4e, 0x83, 0x75, 0x1b, 0xfe, 0x00, 0xc4, 0xcf, 0x55,
-0xb2, 0x50, 0xa6, 0xeb, 0x38, 0xed, 0x8f, 0xd3, 0x1d, 0x00, 0xf6, 0xe3,
-0x90, 0x1c, 0x60, 0x9e, 0x8e, 0xeb, 0x0b, 0xba, 0x44, 0x06, 0x68, 0xbb,
-0xd3, 0x10, 0x63, 0x35, 0xe1, 0x86, 0x5c, 0x5c, 0x2b, 0x85, 0xa6, 0xe7,
-0x38, 0x2c, 0x18, 0x83, 0x1f, 0x8f, 0x9b, 0x8e, 0x4d, 0x26, 0xcd, 0x34,
-0x0c, 0x66, 0x1d, 0x70, 0xb7, 0x01, 0xe6, 0x02, 0xa8, 0x51, 0x63, 0xcf,
-0xbb, 0x03, 0xca, 0x85, 0xc6, 0x9c, 0xf6, 0xf1, 0x51, 0xe0, 0x60, 0x07,
-0x40, 0x86, 0xf0, 0x1e, 0x6e, 0xef, 0x61, 0x10, 0xd9, 0x36, 0xcc, 0xfc,
-0x58, 0xe2, 0x37, 0x0d, 0x58, 0xb7, 0xbe, 0xca, 0xc9, 0xd8, 0xcd, 0xaa,
-0xd5, 0x5b, 0x77, 0x83, 0xcb, 0x0e, 0x30, 0xce, 0xc8, 0xb8, 0xcd, 0xbf,
-0x1e, 0x63, 0x04, 0xad, 0xb7, 0xcd, 0x43, 0x62, 0x4c, 0xe0, 0x1a, 0xd4,
-0x21, 0xe2, 0xdd, 0x33, 0xdf, 0xb1, 0xdd, 0xdc, 0x01, 0x22, 0x18, 0xce,
-0xa1, 0xd8, 0xcb, 0x67, 0xd5, 0x38, 0x4a, 0xbc, 0xd5, 0x81, 0x3d, 0x03,
-0x98, 0x35, 0x60, 0x41, 0x85, 0x0c, 0x1d, 0xe7, 0x76, 0xf8, 0x11, 0x52,
-0x76, 0xf6, 0x06, 0x16, 0x02, 0x45, 0xc8, 0xd8, 0x2f, 0x5e, 0x57, 0xbc,
-0x3b, 0x89, 0x97, 0x09, 0x3e, 0x03, 0x34, 0x1a, 0x9d, 0x37, 0x87, 0x48,
-0x0a, 0xe0, 0xa7, 0x4f, 0x8c, 0x3a, 0xa2, 0xaf, 0xfd, 0x7b, 0x80, 0xcf,
-0xe5, 0x18, 0x61, 0x68, 0xba, 0x61, 0x8b, 0x09, 0xaa, 0xa3, 0x0c, 0x47,
-0x3c, 0x43, 0x03, 0xac, 0xa3, 0x2e, 0x5e, 0x72, 0x0c, 0x80, 0x19, 0x61,
-0xe6, 0x6e, 0x0e, 0xd9, 0xe8, 0xe8, 0xaf, 0x11, 0x9b, 0x4a, 0x73, 0x7a,
-0x61, 0x66, 0xf0, 0x54, 0x1d, 0x18, 0xc8, 0x23, 0x36, 0xbf, 0xb5, 0xf4,
-0x86, 0x54, 0xed, 0xb5, 0x91, 0xee, 0xb8, 0xbc, 0xde, 0xc3, 0x87, 0x9b,
-0x2f, 0x81, 0xf2, 0xee, 0xa3, 0xec, 0x02

build/single_file_libs/zstd-in.c

@@ -1,86 +0,0 @@
-/**
- * \file zstd.c
- * Single-file Zstandard library.
- *
- * Generate using:
- * \code
- *	combine.sh -r ../../lib -o zstd.c zstd-in.c
- * \endcode
- */
-/*
- * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-/*
- * Settings to bake for the single library file.
- *
- * Note: It's important that none of these affects 'zstd.h' (only the
- * implementation files we're amalgamating).
- *
- * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
- * defined in mem.h (breaking C99 compatibility).
- *
- * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
- * standalone xxHash usage (with global defines).
- *
- * Note: multithreading is enabled for all platforms apart from Emscripten.
- */
-#define DEBUGLEVEL 0
-#define MEM_MODULE
-#undef  XXH_NAMESPACE
-#define XXH_NAMESPACE ZSTD_
-#undef  XXH_PRIVATE_API
-#define XXH_PRIVATE_API
-#undef  XXH_INLINE_ALL
-#define XXH_INLINE_ALL
-#define ZSTD_LEGACY_SUPPORT 0
-#ifndef __EMSCRIPTEN__
-#define ZSTD_MULTITHREAD
-#endif
-#define ZSTD_TRACE 0
-/* TODO: Can't amalgamate ASM function */
-#define ZSTD_DISABLE_ASM 1
-
-/* Include zstd_deps.h first with all the options we need enabled. */
-#define ZSTD_DEPS_NEED_MALLOC
-#define ZSTD_DEPS_NEED_MATH64
-#include "common/zstd_deps.h"
-
-#include "common/debug.c"
-#include "common/entropy_common.c"
-#include "common/error_private.c"
-#include "common/fse_decompress.c"
-#include "common/threading.c"
-#include "common/pool.c"
-#include "common/zstd_common.c"
-
-#include "compress/fse_compress.c"
-#include "compress/hist.c"
-#include "compress/huf_compress.c"
-#include "compress/zstd_compress_literals.c"
-#include "compress/zstd_compress_sequences.c"
-#include "compress/zstd_compress_superblock.c"
-#include "compress/zstd_compress.c"
-#include "compress/zstd_double_fast.c"
-#include "compress/zstd_fast.c"
-#include "compress/zstd_lazy.c"
-#include "compress/zstd_ldm.c"
-#include "compress/zstd_opt.c"
-#ifdef ZSTD_MULTITHREAD
-#include "compress/zstdmt_compress.c"
-#endif
-
-#include "decompress/huf_decompress.c"
-#include "decompress/zstd_ddict.c"
-#include "decompress/zstd_decompress.c"
-#include "decompress/zstd_decompress_block.c"
-
-#include "dictBuilder/cover.c"
-#include "dictBuilder/divsufsort.c"
-#include "dictBuilder/fastcover.c"
-#include "dictBuilder/zdict.c"

build/single_file_libs/zstddeclib-in.c

@@ -1,58 +0,0 @@
-/**
- * \file zstddeclib.c
- * Single-file Zstandard decompressor.
- *
- * Generate using:
- * \code
- *	combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
- * \endcode
- */
-/*
- * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-/*
- * Settings to bake for the standalone decompressor.
- *
- * Note: It's important that none of these affects 'zstd.h' (only the
- * implementation files we're amalgamating).
- *
- * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also
- * defined in mem.h (breaking C99 compatibility).
- *
- * Note: the undefs for xxHash allow Zstd's implementation to coincide with with
- * standalone xxHash usage (with global defines).
- */
-#define DEBUGLEVEL 0
-#define MEM_MODULE
-#undef  XXH_NAMESPACE
-#define XXH_NAMESPACE ZSTD_
-#undef  XXH_PRIVATE_API
-#define XXH_PRIVATE_API
-#undef  XXH_INLINE_ALL
-#define XXH_INLINE_ALL
-#define ZSTD_LEGACY_SUPPORT 0
-#define ZSTD_STRIP_ERROR_STRINGS
-#define ZSTD_TRACE 0
-/* TODO: Can't amalgamate ASM function */
-#define ZSTD_DISABLE_ASM 1
-
-/* Include zstd_deps.h first with all the options we need enabled. */
-#define ZSTD_DEPS_NEED_MALLOC
-#include "common/zstd_deps.h"
-
-#include "common/debug.c"
-#include "common/entropy_common.c"
-#include "common/error_private.c"
-#include "common/fse_decompress.c"
-#include "common/zstd_common.c"
-
-#include "decompress/huf_decompress.c"
-#include "decompress/zstd_ddict.c"
-#include "decompress/zstd_decompress.c"
-#include "decompress/zstd_decompress_block.c"

contrib/VS2005/fullbench/fullbench.vcproj

@@ -390,7 +390,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File

contrib/VS2005/fuzzer/fuzzer.vcproj

@@ -426,7 +426,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -454,7 +454,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/VS2005/zstd/zstd.vcproj

@@ -454,7 +454,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -482,7 +482,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/VS2005/zstdlib/zstdlib.vcproj

@@ -371,6 +371,18 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\zdict.c"
 				>
@@ -446,7 +458,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zstd_errors.h"
+				RelativePath="..\..\..\lib\common\zstd_errors.h"
 				>
 			</File>
 			<File
@@ -474,7 +486,11 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\lib\zdict.h"
+				RelativePath="..\..\..\lib\common\zbuff.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\dictBuilder\zdict.h"
 				>
 			</File>
 			<File

contrib/adaptive-compression/.gitignore

@@ -0,0 +1,3 @@
+# binaries generated
+adapt
+datagen

contrib/adaptive-compression/Makefile

@@ -0,0 +1,76 @@
+
+ZSTDDIR = ../../lib
+PRGDIR  = ../../programs
+ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+
+MULTITHREAD_LDFLAGS = -pthread
+DEBUGFLAGS= -g -DZSTD_DEBUG=1
+CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS   ?= -O3
+CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+            -Wredundant-decls
+CFLAGS   += $(DEBUGFLAGS)
+CFLAGS   += $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
+
+all: adapt datagen
+
+adapt: $(ZSTD_FILES) adapt.c
+	$(CC) $(FLAGS) $^ -o $@
+
+adapt-debug: $(ZSTD_FILES) adapt.c
+	$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
+
+datagen : $(PRGDIR)/datagen.c datagencli.c
+	$(CC)      $(FLAGS) $^ -o $@
+
+test-adapt-correctness: datagen adapt
+	@./test-correctness.sh
+	@echo "test correctness complete"
+
+test-adapt-performance: datagen adapt
+	@./test-performance.sh
+	@echo "test performance complete"
+
+clean:
+	@$(RM) -f adapt datagen
+	@$(RM) -rf *.dSYM
+	@$(RM) -f tmp*
+	@$(RM) -f tests/*.zst
+	@$(RM) -f tests/tmp*
+	@echo "finished cleaning"
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX  ?= /usr/local
+DESTDIR ?=
+BINDIR  ?= $(PREFIX)/bin
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+
+install: adapt
+	@echo Installing binaries
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
+	@$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
+	@echo zstd-adaptive installation completed
+
+uninstall:
+	@$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
+	@echo zstd-adaptive programs successfully uninstalled
+endif

contrib/adaptive-compression/README.md

@@ -0,0 +1,91 @@
+### Summary
+
+`adapt` is a new compression tool targeted at optimizing performance across network connections and pipelines. The tool is aimed at sensing network speeds and adapting compression level based on network or pipe speeds.
+In situations where the compression level does not appropriately match the network/pipe speed, compression may be bottlenecking the entire pipeline or the files may not be compressed as much as they potentially could be, therefore losing efficiency. It also becomes quite impractical to manually measure and set an optimalcompression level (which could potentially change over time). 
+
+### Using `adapt`
+
+In order to build and use the tool, you can simply run `make adapt` in the `adaptive-compression` directory under `contrib`. This will generate an executable available for use. Another possible method of installation is running `make install`, which will create and install the binary as the command `zstd-adaptive`.
+
+Similar to many other compression utilities, `zstd-adaptive` can be invoked by using the following format:
+
+`zstd-adaptive [options] [file(s)]`
+
+Supported options for the above format are described below. 
+
+`zstd-adaptive` also supports reading from `stdin` and writing to `stdout`, which is potentially more useful. By default, if no files are given, `zstd-adaptive` reads from and writes to standard I/O. Therefore, you can simply insert it within a pipeline like so:
+
+`cat FILE | zstd-adaptive | ssh "cat - > tmp.zst"`
+
+If a file is provided, it is also possible to force writing to stdout using the `-c` flag like so:
+
+`zstd-adaptive -c FILE | ssh "cat - > tmp.zst"`
+
+Several options described below can be used to control the behavior of `zstd-adaptive`. More specifically, using the `-l#` and `-u#` flags will will set upper and lower bounds so that the compression level will always be within that range. The `-i#` flag can also be used to change the initial compression level. If an initial compression level is not provided, the initial compression level will be chosen such that it is within the appropriate range (it becomes equal to the lower bound). 
+
+### Options
+`-oFILE` : write output to `FILE`
+
+`-i#`    : provide initial compression level (must within the appropriate bounds)
+
+`-h`     : display help/information
+
+`-f`     : force the compression level to stay constant
+
+`-c`     : force write to `stdout`
+
+`-p`     : hide progress bar
+
+`-q`     : quiet mode -- do not show progress bar or other information
+
+`-l#`    : set a lower bound on the compression level (default is 1)
+
+`-u#`    : set an upper bound on the compression level (default is 22)
+### Benchmarking / Test results
+#### Artificial Tests
+These artificial tests were run by using the `pv` command line utility in order to limit pipe speeds (25 MB/s read and 5 MB/s write limits were chosen to mimic severe throughput constraints). A 40 GB backup file was sent through a pipeline, compressed, and written out to a file. Compression time, size, and ratio were computed. Data for `zstd -15` was excluded from these tests because the test runs quite long.
+
+<table>
+<tr><th> 25 MB/s read limit </th></tr>
+<tr><td>
+
+| Compressor Name | Ratio | Compressed Size | Compression Time |
+|:----------------|------:|----------------:|-----------------:| 
+| zstd -3         | 2.108 |       20.718 GB |      29m 48.530s |
+| zstd-adaptive   | 2.230 |       19.581 GB |      29m 48.798s |
+
+</td><tr>
+</table>
+
+<table>
+<tr><th> 5 MB/s write limit </th></tr>
+<tr><td>
+
+| Compressor Name | Ratio | Compressed Size | Compression Time |
+|:----------------|------:|----------------:|-----------------:| 
+| zstd -3         | 2.108 |       20.718 GB |   1h 10m 43.076s |
+| zstd-adaptive   | 2.249 |       19.412 GB |   1h 06m 15.577s |
+
+</td></tr>
+</table>
+
+The commands used for this test generally followed the form:
+
+`cat FILE | pv -L 25m -q | COMPRESSION | pv -q > tmp.zst # impose 25 MB/s read limit`
+
+`cat FILE | pv -q | COMPRESSION | pv -L 5m -q > tmp.zst  # impose 5 MB/s write limit`
+
+#### SSH Tests
+
+The following tests were performed by piping a relatively large backup file (approximately 80 GB) through compression and over SSH to be stored on a server. The test data includes statistics for time and compressed size  on `zstd` at several compression levels, as well as `zstd-adaptive`. The data highlights the potential advantages that `zstd-adaptive` has over using a low static compression level and the negative imapcts that using an excessively high static compression level can have on
+pipe throughput.
+
+| Compressor Name | Ratio | Compressed Size | Compression Time |
+|:----------------|------:|----------------:|-----------------:|
+| zstd -3         | 2.212 |       32.426 GB |   1h 17m 59.756s |
+| zstd -15        | 2.374 |       30.213 GB |   2h 56m 59.441s |
+| zstd-adaptive   | 2.315 |       30.993 GB |   1h 18m 52.860s |
+
+The commands used for this test generally followed the form: 
+
+`cat FILE | COMPRESSION | ssh dev "cat - > tmp.zst"`

contrib/adaptive-compression/datagencli.c

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+
+/*-************************************
+*  Dependencies
+**************************************/
+#include "util.h"      /* Compiler options */
+#include <stdio.h>     /* fprintf, stderr */
+#include "datagen.h"   /* RDG_generate */
+
+
+/*-************************************
+*  Constants
+**************************************/
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define SIZE_DEFAULT ((64 KB) + 1)
+#define SEED_DEFAULT 0
+#define COMPRESSIBILITY_DEFAULT 50
+
+
+/*-************************************
+*  Macros
+**************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static unsigned displayLevel = 2;
+
+
+/*-*******************************************************
+*  Command line
+*********************************************************/
+static int usage(const char* programName)
+{
+    DISPLAY( "Compressible data generator\n");
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -g#    : generate # data (default:%i)\n", SIZE_DEFAULT);
+    DISPLAY( " -s#    : Select seed (default:%i)\n", SEED_DEFAULT);
+    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n",
+                        COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -h     : display help and exit\n");
+    return 0;
+}
+
+
+int main(int argc, const char** argv)
+{
+    unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
+    double litProba = 0.0;
+    U64 size = SIZE_DEFAULT;
+    U32 seed = SEED_DEFAULT;
+    const char* const programName = argv[0];
+
+    int argNb;
+    for(argNb=1; argNb<argc; argNb++) {
+        const char* argument = argv[argNb];
+
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* Handle commands. Aggregated commands are allowed */
+        if (*argument=='-') {
+            argument++;
+            while (*argument!=0) {
+                switch(*argument)
+                {
+                case 'h':
+                    return usage(programName);
+                case 'g':
+                    argument++;
+                    size=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                        size *= 10, size += *argument++ - '0';
+                    if (*argument=='K') { size <<= 10; argument++; }
+                    if (*argument=='M') { size <<= 20; argument++; }
+                    if (*argument=='G') { size <<= 30; argument++; }
+                    if (*argument=='B') { argument++; }
+                    break;
+                case 's':
+                    argument++;
+                    seed=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                        seed *= 10, seed += *argument++ - '0';
+                    break;
+                case 'P':
+                    argument++;
+                    probaU32 = 0;
+                    while ((*argument>='0') && (*argument<='9'))
+                        probaU32 *= 10, probaU32 += *argument++ - '0';
+                    if (probaU32>100) probaU32 = 100;
+                    break;
+                case 'L':   /* hidden argument : Literal distribution probability */
+                    argument++;
+                    litProba=0.;
+                    while ((*argument>='0') && (*argument<='9'))
+                        litProba *= 10, litProba += *argument++ - '0';
+                    if (litProba>100.) litProba=100.;
+                    litProba /= 100.;
+                    break;
+                case 'v':
+                    displayLevel = 4;
+                    argument++;
+                    break;
+                default:
+                    return usage(programName);
+                }
+    }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
+
+    DISPLAYLEVEL(4, "Compressible data Generator \n");
+    if (probaU32!=COMPRESSIBILITY_DEFAULT)
+        DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
+    DISPLAYLEVEL(3, "Seed = %u \n", seed);
+
+    RDG_genStdout(size, (double)probaU32/100, litProba, seed);
+    DISPLAYLEVEL(1, "\n");
+
+    return 0;
+}

contrib/adaptive-compression/test-correctness.sh

@@ -0,0 +1,252 @@
+echo "correctness tests -- general"
+./datagen -s1 -g1GB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s2 -g500MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s3 -g250MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s4 -g125MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s5 -g50MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s6 -g25MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s7 -g10MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s8 -g5MB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s9 -g500KB > tmp
+./adapt -otmp.zst tmp
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- streaming"
+./datagen -s10 -g1GB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s11 -g100MB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s12 -g10MB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s13 -g1MB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s14 -g100KB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s15 -g10KB > tmp
+cat tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- read limit"
+./datagen -s16 -g1GB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s17 -g100MB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s18 -g10MB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s19 -g1MB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s20 -g100KB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s21 -g10KB > tmp
+pv -L 50m -q tmp | ./adapt > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- write limit"
+./datagen -s22 -g1GB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s23 -g100MB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s24 -g10MB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s25 -g1MB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s26 -g100KB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s27 -g10KB > tmp
+pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- read and write limits"
+./datagen -s28 -g1GB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s29 -g100MB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s30 -g10MB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s31 -g1MB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s32 -g100KB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s33 -g10KB > tmp
+pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- forced compression level"
+./datagen -s34 -g1GB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s35 -g100MB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s36 -g10MB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s37 -g1MB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s38 -g100KB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+./datagen -s39 -g10KB > tmp
+./adapt tmp -otmp.zst -i11 -f
+zstd -d tmp.zst -o tmp2
+diff -s -q tmp tmp2
+rm tmp*
+
+echo -e "\ncorrectness tests -- window size test"
+./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
+zstd -d tmp.zst
+rm tmp*
+
+echo -e "\ncorrectness tests -- testing bounds"
+./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
+rm tmp*
+
+./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
+rm tmp*
+make clean

contrib/adaptive-compression/test-performance.sh

@@ -0,0 +1,59 @@
+echo "testing time -- no limits set"
+./datagen -s1 -g1GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+rm tmp*
+
+./datagen -s2 -g2GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+rm tmp*
+
+./datagen -s3 -g4GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+rm tmp*
+
+echo -e "\ntesting compression ratio -- no limits set"
+./datagen -s4 -g1GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+ls -l tmp1.zst tmp2.zst
+rm tmp*
+
+./datagen -s5 -g2GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+ls -l tmp1.zst tmp2.zst
+rm tmp*
+
+./datagen -s6 -g4GB > tmp
+time ./adapt -otmp1.zst tmp
+time zstd -1 -o tmp2.zst tmp
+ls -l tmp1.zst tmp2.zst
+rm tmp*
+
+echo e "\ntesting performance at various compression levels -- no limits set"
+./datagen -s7 -g1GB > tmp
+echo "adapt"
+time ./adapt -i5 -f tmp -otmp1.zst
+echo "zstdcli"
+time zstd -5 tmp -o tmp2.zst
+ls -l tmp1.zst tmp2.zst
+rm tmp*
+
+./datagen -s8 -g1GB > tmp
+echo "adapt"
+time ./adapt -i10 -f tmp -otmp1.zst
+echo "zstdcli"
+time zstd -10 tmp -o tmp2.zst
+ls -l tmp1.zst tmp2.zst
+rm tmp*
+
+./datagen -s9 -g1GB > tmp
+echo "adapt"
+time ./adapt -i15 -f tmp -otmp1.zst
+echo "zstdcli"
+time zstd -15 tmp -o tmp2.zst
+ls -l tmp1.zst tmp2.zst
+rm tmp*

contrib/diagnose_corruption/.gitignore

@@ -1 +0,0 @@
-check_flipped_bits

contrib/diagnose_corruption/Makefile

@@ -1,35 +0,0 @@
-# ################################################################
-# Copyright (c) 2019-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# ################################################################
-
-.PHONY: all
-all: check_flipped_bits
-
-ZSTDLIBDIR ?= ../../lib
-
-CFLAGS     ?= -O3
-CFLAGS     += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \
-              -I$(ZSTDLIBDIR)/decompress
-CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
-              -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-              -Wstrict-prototypes -Wundef                                     \
-              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
-              -Wredundant-decls -Wmissing-prototypes
-CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
-
-.PHONY: $(ZSTDLIBDIR)/libzstd.a
-$(ZSTDLIBDIR)/libzstd.a:
-	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
-
-check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
-	$(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a
-
-.PHONY: clean
-clean:
-	rm -f check_flipped_bits

contrib/diagnose_corruption/check_flipped_bits.c

@@ -1,400 +0,0 @@
-/*
- * Copyright (c) 2019-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#define ZSTD_STATIC_LINKING_ONLY
-#include "zstd.h"
-#include "zstd_errors.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-typedef struct {
-  char *input;
-  size_t input_size;
-
-  char *perturbed; /* same size as input */
-
-  char *output;
-  size_t output_size;
-
-  const char *dict_file_name;
-  const char *dict_file_dir_name;
-  int32_t dict_id;
-  char *dict;
-  size_t dict_size;
-  ZSTD_DDict* ddict;
-
-  ZSTD_DCtx* dctx;
-
-  int success_count;
-  int error_counts[ZSTD_error_maxCode];
-} stuff_t;
-
-static void free_stuff(stuff_t* stuff) {
-  free(stuff->input);
-  free(stuff->output);
-  ZSTD_freeDDict(stuff->ddict);
-  free(stuff->dict);
-  ZSTD_freeDCtx(stuff->dctx);
-}
-
-static void usage(void) {
-  fprintf(stderr, "check_flipped_bits input_filename [-d dict] [-D dict_dir]\n");
-  fprintf(stderr, "\n");
-  fprintf(stderr, "Arguments:\n");
-  fprintf(stderr, "    -d file: path to a dictionary file to use.\n");
-  fprintf(stderr, "    -D dir : path to a directory, with files containing dictionaries, of the\n"
-                  "             form DICTID.zstd-dict, e.g., 12345.zstd-dict.\n");
-  exit(1);
-}
-
-static void print_summary(stuff_t* stuff) {
-  int error_code;
-  fprintf(stderr, "%9d successful decompressions\n", stuff->success_count);
-  for (error_code = 0; error_code < ZSTD_error_maxCode; error_code++) {
-    int count = stuff->error_counts[error_code];
-    if (count) {
-      fprintf(
-          stderr, "%9d failed decompressions with message: %s\n",
-          count, ZSTD_getErrorString(error_code));
-    }
-  }
-}
-
-static char* readFile(const char* filename, size_t* size) {
-  struct stat statbuf;
-  int ret;
-  FILE* f;
-  char *buf;
-  size_t bytes_read;
-
-  ret = stat(filename, &statbuf);
-  if (ret != 0) {
-    fprintf(stderr, "stat failed: %m\n");
-    return NULL;
-  }
-  if ((statbuf.st_mode & S_IFREG) != S_IFREG) {
-    fprintf(stderr, "Input must be regular file\n");
-    return NULL;
-  }
-
-  *size = statbuf.st_size;
-
-  f = fopen(filename, "r");
-  if (f == NULL) {
-    fprintf(stderr, "fopen failed: %m\n");
-    return NULL;
-  }
-
-  buf = malloc(*size);
-  if (buf == NULL) {
-    fprintf(stderr, "malloc failed\n");
-    fclose(f);
-    return NULL;
-  }
-
-  bytes_read = fread(buf, 1, *size, f);
-  if (bytes_read != *size) {
-    fprintf(stderr, "failed to read whole file\n");
-    fclose(f);
-    free(buf);
-    return NULL;
-  }
-
-  ret = fclose(f);
-  if (ret != 0) {
-    fprintf(stderr, "fclose failed: %m\n");
-    free(buf);
-    return NULL;
-  }
-
-  return buf;
-}
-
-static ZSTD_DDict* readDict(const char* filename, char **buf, size_t* size, int32_t* dict_id) {
-  ZSTD_DDict* ddict;
-  *buf = readFile(filename, size);
-  if (*buf == NULL) {
-    fprintf(stderr, "Opening dictionary file '%s' failed\n", filename);
-    return NULL;
-  }
-
-  ddict = ZSTD_createDDict_advanced(*buf, *size, ZSTD_dlm_byRef, ZSTD_dct_auto, ZSTD_defaultCMem);
-  if (ddict == NULL) {
-    fprintf(stderr, "Failed to create ddict.\n");
-    return NULL;
-  }
-  if (dict_id != NULL) {
-    *dict_id = ZSTD_getDictID_fromDDict(ddict);
-  }
-  return ddict;
-}
-
-static ZSTD_DDict* readDictByID(stuff_t *stuff, int32_t dict_id, char **buf, size_t* size) {
-  if (stuff->dict_file_dir_name == NULL) {
-    return NULL;
-  } else {
-    size_t dir_name_len = strlen(stuff->dict_file_dir_name);
-    int dir_needs_separator = 0;
-    size_t dict_file_name_alloc_size = dir_name_len + 1 /* '/' */ + 10 /* max int32_t len */ + strlen(".zstd-dict") + 1 /* '\0' */;
-    char *dict_file_name = malloc(dict_file_name_alloc_size);
-    ZSTD_DDict* ddict;
-    int32_t read_dict_id;
-    if (dict_file_name == NULL) {
-      fprintf(stderr, "malloc failed.\n");
-      return 0;
-    }
-
-    if (dir_name_len > 0 && stuff->dict_file_dir_name[dir_name_len - 1] != '/') {
-      dir_needs_separator = 1;
-    }
-
-    snprintf(
-      dict_file_name,
-      dict_file_name_alloc_size,
-      "%s%s%u.zstd-dict",
-      stuff->dict_file_dir_name,
-      dir_needs_separator ? "/" : "",
-      dict_id);
-
-    /* fprintf(stderr, "Loading dict %u from '%s'.\n", dict_id, dict_file_name); */
-
-    ddict = readDict(dict_file_name, buf, size, &read_dict_id);
-    if (ddict == NULL) {
-      fprintf(stderr, "Failed to create ddict from '%s'.\n", dict_file_name);
-      free(dict_file_name);
-      return 0;
-    }
-    if (read_dict_id != dict_id) {
-      fprintf(stderr, "Read dictID (%u) does not match expected (%u).\n", read_dict_id, dict_id);
-      free(dict_file_name);
-      ZSTD_freeDDict(ddict);
-      return 0;
-    }
-
-    free(dict_file_name);
-    return ddict;
-  }
-}
-
-static int init_stuff(stuff_t* stuff, int argc, char *argv[]) {
-  const char* input_filename;
-
-  if (argc < 2) {
-    usage();
-  }
-
-  input_filename = argv[1];
-  stuff->input_size = 0;
-  stuff->input = readFile(input_filename, &stuff->input_size);
-  if (stuff->input == NULL) {
-    fprintf(stderr, "Failed to read input file.\n");
-    return 0;
-  }
-
-  stuff->perturbed = malloc(stuff->input_size);
-  if (stuff->perturbed == NULL) {
-    fprintf(stderr, "malloc failed.\n");
-    return 0;
-  }
-  memcpy(stuff->perturbed, stuff->input, stuff->input_size);
-
-  stuff->output_size = ZSTD_DStreamOutSize();
-  stuff->output = malloc(stuff->output_size);
-  if (stuff->output == NULL) {
-    fprintf(stderr, "malloc failed.\n");
-    return 0;
-  }
-
-  stuff->dict_file_name = NULL;
-  stuff->dict_file_dir_name = NULL;
-  stuff->dict_id = 0;
-  stuff->dict = NULL;
-  stuff->dict_size = 0;
-  stuff->ddict = NULL;
-
-  if (argc > 2) {
-    if (!strcmp(argv[2], "-d")) {
-      if (argc > 3) {
-        stuff->dict_file_name = argv[3];
-      } else {
-        usage();
-      }
-    } else
-    if (!strcmp(argv[2], "-D")) {
-      if (argc > 3) {
-        stuff->dict_file_dir_name = argv[3];
-      } else {
-        usage();
-      }
-    } else {
-      usage();
-    }
-  }
-
-  if (stuff->dict_file_dir_name) {
-    int32_t dict_id = ZSTD_getDictID_fromFrame(stuff->input, stuff->input_size);
-    if (dict_id != 0) {
-      stuff->ddict = readDictByID(stuff, dict_id, &stuff->dict, &stuff->dict_size);
-      if (stuff->ddict == NULL) {
-        fprintf(stderr, "Failed to create cached ddict.\n");
-        return 0;
-      }
-      stuff->dict_id = dict_id;
-    }
-  } else
-  if (stuff->dict_file_name) {
-    stuff->ddict = readDict(stuff->dict_file_name, &stuff->dict, &stuff->dict_size, &stuff->dict_id);
-    if (stuff->ddict == NULL) {
-      fprintf(stderr, "Failed to create ddict from '%s'.\n", stuff->dict_file_name);
-      return 0;
-    }
-  }
-
-  stuff->dctx = ZSTD_createDCtx();
-  if (stuff->dctx == NULL) {
-    return 0;
-  }
-
-  stuff->success_count = 0;
-  memset(stuff->error_counts, 0, sizeof(stuff->error_counts));
-
-  return 1;
-}
-
-static int test_decompress(stuff_t* stuff) {
-  size_t ret;
-  ZSTD_inBuffer in = {stuff->perturbed, stuff->input_size, 0};
-  ZSTD_outBuffer out = {stuff->output, stuff->output_size, 0};
-  ZSTD_DCtx* dctx = stuff->dctx;
-  int32_t custom_dict_id = ZSTD_getDictID_fromFrame(in.src, in.size);
-  char *custom_dict = NULL;
-  size_t custom_dict_size = 0;
-  ZSTD_DDict* custom_ddict = NULL;
-
-  if (custom_dict_id != 0 && custom_dict_id != stuff->dict_id) {
-    /* fprintf(stderr, "Instead of dict %u, this perturbed blob wants dict %u.\n", stuff->dict_id, custom_dict_id); */
-    custom_ddict = readDictByID(stuff, custom_dict_id, &custom_dict, &custom_dict_size);
-  }
-
-  ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
-
-  if (custom_ddict != NULL) {
-    ZSTD_DCtx_refDDict(dctx, custom_ddict);
-  } else {
-    ZSTD_DCtx_refDDict(dctx, stuff->ddict);
-  }
-
-  while (in.pos != in.size) {
-    out.pos = 0;
-    ret = ZSTD_decompressStream(dctx, &out, &in);
-
-    if (ZSTD_isError(ret)) {
-      unsigned int code = ZSTD_getErrorCode(ret);
-      if (code >= ZSTD_error_maxCode) {
-        fprintf(stderr, "Received unexpected error code!\n");
-        exit(1);
-      }
-      stuff->error_counts[code]++;
-      /*
-      fprintf(
-          stderr, "Decompression failed: %s\n", ZSTD_getErrorName(ret));
-      */
-      if (custom_ddict != NULL) {
-        ZSTD_freeDDict(custom_ddict);
-        free(custom_dict);
-      }
-      return 0;
-    }
-  }
-
-  stuff->success_count++;
-
-  if (custom_ddict != NULL) {
-    ZSTD_freeDDict(custom_ddict);
-    free(custom_dict);
-  }
-  return 1;
-}
-
-static int perturb_bits(stuff_t* stuff) {
-  size_t pos;
-  size_t bit;
-  for (pos = 0; pos < stuff->input_size; pos++) {
-    unsigned char old_val = stuff->input[pos];
-    if (pos % 1000 == 0) {
-      fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
-    }
-    for (bit = 0; bit < 8; bit++) {
-      unsigned char new_val = old_val ^ (1 << bit);
-      stuff->perturbed[pos] = new_val;
-      if (test_decompress(stuff)) {
-        fprintf(
-            stderr,
-            "Flipping byte %zu bit %zu (0x%02x -> 0x%02x) "
-            "produced a successful decompression!\n",
-            pos, bit, old_val, new_val);
-      }
-    }
-    stuff->perturbed[pos] = old_val;
-  }
-  return 1;
-}
-
-static int perturb_bytes(stuff_t* stuff) {
-  size_t pos;
-  size_t new_val;
-  for (pos = 0; pos < stuff->input_size; pos++) {
-    unsigned char old_val = stuff->input[pos];
-    if (pos % 1000 == 0) {
-      fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
-    }
-    for (new_val = 0; new_val < 256; new_val++) {
-      stuff->perturbed[pos] = new_val;
-      if (test_decompress(stuff)) {
-        fprintf(
-            stderr,
-            "Changing byte %zu (0x%02x -> 0x%02x) "
-            "produced a successful decompression!\n",
-            pos, old_val, (unsigned char)new_val);
-      }
-    }
-    stuff->perturbed[pos] = old_val;
-  }
-  return 1;
-}
-
-int main(int argc, char* argv[]) {
-  stuff_t stuff;
-
-  if(!init_stuff(&stuff, argc, argv)) {
-    fprintf(stderr, "Failed to init.\n");
-    return 1;
-  }
-
-  if (test_decompress(&stuff)) {
-    fprintf(stderr, "Blob already decompresses successfully!\n");
-    return 1;
-  }
-
-  perturb_bits(&stuff);
-
-  perturb_bytes(&stuff);
-
-  print_summary(&stuff);
-
-  free_stuff(&stuff);
-
-  return 0;
-}

contrib/docker/README.md

@@ -5,7 +5,7 @@ The `Dockerfile` script requires a version of `docker` >= 17.05
 
 ## Installing docker
 
-The official docker install docs use a ppa with a modern version available:
+The officiel docker install docs use a ppa with a modern version available:
 https://docs.docker.com/install/linux/docker-ce/ubuntu/
 
 ## How to run

contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile

@@ -0,0 +1,44 @@
+ARG :=
+
+CC ?= gcc
+CFLAGS ?= -O3
+INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+
+RANDOM_FILE := ../randomDictBuilder/random.c
+IO_FILE := ../randomDictBuilder/io.c
+
+all: run clean
+
+.PHONY: run
+run: benchmark
+	echo "Benchmarking with $(ARG)"
+	./benchmark $(ARG)
+
+.PHONY: test
+test: benchmarkTest clean
+
+.PHONY: benchmarkTest
+benchmarkTest: benchmark test.sh
+	sh test.sh
+
+benchmark: benchmark.o io.o random.o libzstd.a
+	$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark
+
+benchmark.o: benchmark.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
+
+random.o: $(RANDOM_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
+
+io.o: $(IO_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
+
+libzstd.a:
+	$(MAKE) -C ../../../lib libzstd.a
+	mv ../../../lib/libzstd.a .
+
+.PHONY: clean
+clean:
+	rm -f *.o benchmark libzstd.a
+	$(MAKE) -C ../../../lib clean
+	echo "Cleaning is completed"

contrib/experimental_dict_builders/benchmarkDictBuilder/README.md

@@ -0,0 +1,849 @@
+Benchmarking Dictionary Builder
+
+### Permitted Argument:
+Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
+
+###Running Test:
+make test
+
+###Usage:
+Benchmark given input files: make ARG= followed by permitted arguments
+
+### Examples:
+make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
+
+###Benchmarking Result:
+- First Cover is optimize cover, second Cover uses optimized d and k from first one.
+- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. This is run for accel values from 1 to 10.
+- Fourth column is chosen d and fifth column is chosen k
+
+github:
+NODICT       0.000004       2.999642        
+RANDOM       0.024560       8.791189        
+LEGACY       0.727109       8.173529        
+COVER       40.565676       10.652243        8          1298
+COVER       3.608284       10.652243        8          1298
+FAST f=15 a=1       4.181024       10.570882        8          1154
+FAST f=15 a=1       0.040788       10.570882        8          1154
+FAST f=15 a=2       3.548352       10.574287        6          1970
+FAST f=15 a=2       0.035535       10.574287        6          1970
+FAST f=15 a=3       3.287364       10.613950        6          1010
+FAST f=15 a=3       0.032182       10.613950        6          1010
+FAST f=15 a=4       3.184976       10.573883        6          1058
+FAST f=15 a=4       0.029878       10.573883        6          1058
+FAST f=15 a=5       3.045513       10.580640        8          1154
+FAST f=15 a=5       0.022162       10.580640        8          1154
+FAST f=15 a=6       3.003296       10.583677        6          1010
+FAST f=15 a=6       0.028091       10.583677        6          1010
+FAST f=15 a=7       2.952655       10.622551        6          1106
+FAST f=15 a=7       0.02724       10.622551        6          1106
+FAST f=15 a=8       2.945674       10.614657        6          1010
+FAST f=15 a=8       0.027264       10.614657        6          1010
+FAST f=15 a=9       3.153439       10.564018        8          1154
+FAST f=15 a=9       0.020635       10.564018        8          1154
+FAST f=15 a=10       2.950416       10.511454        6          1010
+FAST f=15 a=10       0.026606       10.511454        6          1010
+FAST f=16 a=1       3.970029       10.681035        8          1154
+FAST f=16 a=1       0.038188       10.681035        8          1154
+FAST f=16 a=2       3.422892       10.484978        6          1874
+FAST f=16 a=2       0.034702       10.484978        6          1874
+FAST f=16 a=3       3.215836       10.632631        8          1154
+FAST f=16 a=3       0.026084       10.632631        8          1154
+FAST f=16 a=4       3.081353       10.626533        6          1106
+FAST f=16 a=4       0.030032       10.626533        6          1106
+FAST f=16 a=5       3.041241       10.545027        8          1922
+FAST f=16 a=5       0.022882       10.545027        8          1922
+FAST f=16 a=6       2.989390       10.638284        6          1874
+FAST f=16 a=6       0.028308       10.638284        6          1874
+FAST f=16 a=7       3.001581       10.797136        6          1106
+FAST f=16 a=7       0.027479       10.797136        6          1106
+FAST f=16 a=8       2.984107       10.658356        8          1058
+FAST f=16 a=8       0.021099       10.658356        8          1058
+FAST f=16 a=9       2.925788       10.523869        6          1010
+FAST f=16 a=9       0.026905       10.523869        6          1010
+FAST f=16 a=10       2.889605       10.745841        6          1874
+FAST f=16 a=10       0.026846       10.745841        6          1874
+FAST f=17 a=1       4.031953       10.672080        8          1202
+FAST f=17 a=1       0.040658       10.672080        8          1202
+FAST f=17 a=2       3.458107       10.589352        8          1106
+FAST f=17 a=2       0.02926       10.589352        8          1106
+FAST f=17 a=3       3.291189       10.662714        8          1154
+FAST f=17 a=3       0.026531       10.662714        8          1154
+FAST f=17 a=4       3.154950       10.549456        8          1346
+FAST f=17 a=4       0.024991       10.549456        8          1346
+FAST f=17 a=5       3.092271       10.541670        6          1202
+FAST f=17 a=5       0.038285       10.541670        6          1202
+FAST f=17 a=6       3.166146       10.729112        6          1874
+FAST f=17 a=6       0.038217       10.729112        6          1874
+FAST f=17 a=7       3.035467       10.810485        6          1106
+FAST f=17 a=7       0.036655       10.810485        6          1106
+FAST f=17 a=8       3.035668       10.530532        6          1058
+FAST f=17 a=8       0.037715       10.530532        6          1058
+FAST f=17 a=9       2.987917       10.589802        8          1922
+FAST f=17 a=9       0.02217       10.589802        8          1922
+FAST f=17 a=10       2.981647       10.722579        8          1106
+FAST f=17 a=10       0.021948       10.722579        8          1106
+FAST f=18 a=1       4.067144       10.634943        8          1154
+FAST f=18 a=1       0.041386       10.634943        8          1154
+FAST f=18 a=2       3.507377       10.546230        6          1970
+FAST f=18 a=2       0.037572       10.546230        6          1970
+FAST f=18 a=3       3.323015       10.648061        8          1154
+FAST f=18 a=3       0.028306       10.648061        8          1154
+FAST f=18 a=4       3.216735       10.705402        6          1010
+FAST f=18 a=4       0.030755       10.705402        6          1010
+FAST f=18 a=5       3.175794       10.588154        8          1874
+FAST f=18 a=5       0.025315       10.588154        8          1874
+FAST f=18 a=6       3.127459       10.751104        8          1106
+FAST f=18 a=6       0.023897       10.751104        8          1106
+FAST f=18 a=7       3.083017       10.780402        6          1106
+FAST f=18 a=7       0.029158       10.780402        6          1106
+FAST f=18 a=8       3.069700       10.547226        8          1346
+FAST f=18 a=8       0.024046       10.547226        8          1346
+FAST f=18 a=9       3.056591       10.674759        6          1010
+FAST f=18 a=9       0.028496       10.674759        6          1010
+FAST f=18 a=10       3.063588       10.737578        8          1106
+FAST f=18 a=10       0.023033       10.737578        8          1106
+FAST f=19 a=1       4.164041       10.650333        8          1154
+FAST f=19 a=1       0.042906       10.650333        8          1154
+FAST f=19 a=2       3.585409       10.577066        6          1058
+FAST f=19 a=2       0.038994       10.577066        6          1058
+FAST f=19 a=3       3.439643       10.639403        8          1154
+FAST f=19 a=3       0.028427       10.639403        8          1154
+FAST f=19 a=4       3.268869       10.554410        8          1298
+FAST f=19 a=4       0.026866       10.554410        8          1298
+FAST f=19 a=5       3.238225       10.615109        6          1010
+FAST f=19 a=5       0.03078       10.615109        6          1010
+FAST f=19 a=6       3.199558       10.609782        6          1874
+FAST f=19 a=6       0.030099       10.609782        6          1874
+FAST f=19 a=7       3.132395       10.794753        6          1106
+FAST f=19 a=7       0.028964       10.794753        6          1106
+FAST f=19 a=8       3.148446       10.554842        8          1298
+FAST f=19 a=8       0.024277       10.554842        8          1298
+FAST f=19 a=9       3.108324       10.668763        6          1010
+FAST f=19 a=9       0.02896       10.668763        6          1010
+FAST f=19 a=10       3.159863       10.757347        8          1106
+FAST f=19 a=10       0.023351       10.757347        8          1106
+FAST f=20 a=1       4.462698       10.661788        8          1154
+FAST f=20 a=1       0.047174       10.661788        8          1154
+FAST f=20 a=2       3.820269       10.678612        6          1106
+FAST f=20 a=2       0.040807       10.678612        6          1106
+FAST f=20 a=3       3.644955       10.648424        8          1154
+FAST f=20 a=3       0.031398       10.648424        8          1154
+FAST f=20 a=4       3.546257       10.559756        8          1298
+FAST f=20 a=4       0.029856       10.559756        8          1298
+FAST f=20 a=5       3.485248       10.646637        6          1010
+FAST f=20 a=5       0.033756       10.646637        6          1010
+FAST f=20 a=6       3.490438       10.775824        8          1106
+FAST f=20 a=6       0.028338       10.775824        8          1106
+FAST f=20 a=7       3.631289       10.801795        6          1106
+FAST f=20 a=7       0.035228       10.801795        6          1106
+FAST f=20 a=8       3.758936       10.545116        8          1346
+FAST f=20 a=8       0.027495       10.545116        8          1346
+FAST f=20 a=9       3.707024       10.677454        6          1010
+FAST f=20 a=9       0.031326       10.677454        6          1010
+FAST f=20 a=10       3.586593       10.756017        8          1106
+FAST f=20 a=10       0.027122       10.756017        8          1106
+FAST f=21 a=1       5.701396       10.655398        8          1154
+FAST f=21 a=1       0.067744       10.655398        8          1154
+FAST f=21 a=2       5.270542       10.650743        6          1106
+FAST f=21 a=2       0.052999       10.650743        6          1106
+FAST f=21 a=3       4.945294       10.652380        8          1154
+FAST f=21 a=3       0.052678       10.652380        8          1154
+FAST f=21 a=4       4.894079       10.543185        8          1298
+FAST f=21 a=4       0.04997       10.543185        8          1298
+FAST f=21 a=5       4.785417       10.630321        6          1010
+FAST f=21 a=5       0.045294       10.630321        6          1010
+FAST f=21 a=6       4.789381       10.664477        6          1874
+FAST f=21 a=6       0.046578       10.664477        6          1874
+FAST f=21 a=7       4.302955       10.805179        6          1106
+FAST f=21 a=7       0.041205       10.805179        6          1106
+FAST f=21 a=8       4.034630       10.551211        8          1298
+FAST f=21 a=8       0.040121       10.551211        8          1298
+FAST f=21 a=9       4.523868       10.799114        6          1010
+FAST f=21 a=9       0.043592       10.799114        6          1010
+FAST f=21 a=10       4.760736       10.750255        8          1106
+FAST f=21 a=10       0.043483       10.750255        8          1106
+FAST f=22 a=1       6.743064       10.640537        8          1154
+FAST f=22 a=1       0.086967       10.640537        8          1154
+FAST f=22 a=2       6.121739       10.626638        6          1970
+FAST f=22 a=2       0.066337       10.626638        6          1970
+FAST f=22 a=3       5.248851       10.640688        8          1154
+FAST f=22 a=3       0.054935       10.640688        8          1154
+FAST f=22 a=4       5.436579       10.588333        8          1298
+FAST f=22 a=4       0.064113       10.588333        8          1298
+FAST f=22 a=5       5.812815       10.652653        6          1010
+FAST f=22 a=5       0.058189       10.652653        6          1010
+FAST f=22 a=6       5.745472       10.666437        6          1874
+FAST f=22 a=6       0.057188       10.666437        6          1874
+FAST f=22 a=7       5.716393       10.806911        6          1106
+FAST f=22 a=7       0.056       10.806911        6          1106
+FAST f=22 a=8       5.698799       10.530784        8          1298
+FAST f=22 a=8       0.0583       10.530784        8          1298
+FAST f=22 a=9       5.710533       10.777391        6          1010
+FAST f=22 a=9       0.054945       10.777391        6          1010
+FAST f=22 a=10       5.685395       10.745023        8          1106
+FAST f=22 a=10       0.056526       10.745023        8          1106
+FAST f=23 a=1       7.836923       10.638828        8          1154
+FAST f=23 a=1       0.099522       10.638828        8          1154
+FAST f=23 a=2       6.627834       10.631061        6          1970
+FAST f=23 a=2       0.066769       10.631061        6          1970
+FAST f=23 a=3       5.602533       10.647288        8          1154
+FAST f=23 a=3       0.064513       10.647288        8          1154
+FAST f=23 a=4       6.005580       10.568747        8          1298
+FAST f=23 a=4       0.062022       10.568747        8          1298
+FAST f=23 a=5       5.481816       10.676921        6          1010
+FAST f=23 a=5       0.058959       10.676921        6          1010
+FAST f=23 a=6       5.460444       10.666194        6          1874
+FAST f=23 a=6       0.057687       10.666194        6          1874
+FAST f=23 a=7       5.659822       10.800377        6          1106
+FAST f=23 a=7       0.06783       10.800377        6          1106
+FAST f=23 a=8       6.826940       10.522167        8          1298
+FAST f=23 a=8       0.070533       10.522167        8          1298
+FAST f=23 a=9       6.804757       10.577799        8          1682
+FAST f=23 a=9       0.069949       10.577799        8          1682
+FAST f=23 a=10       6.774933       10.742093        8          1106
+FAST f=23 a=10       0.068395       10.742093        8          1106
+FAST f=24 a=1       8.444110       10.632783        8          1154
+FAST f=24 a=1       0.094357       10.632783        8          1154
+FAST f=24 a=2       7.289578       10.631061        6          1970
+FAST f=24 a=2       0.098515       10.631061        6          1970
+FAST f=24 a=3       8.619780       10.646289        8          1154
+FAST f=24 a=3       0.098041       10.646289        8          1154
+FAST f=24 a=4       8.508455       10.555199        8          1298
+FAST f=24 a=4       0.093885       10.555199        8          1298
+FAST f=24 a=5       8.471145       10.674363        6          1010
+FAST f=24 a=5       0.088676       10.674363        6          1010
+FAST f=24 a=6       8.426727       10.667228        6          1874
+FAST f=24 a=6       0.087247       10.667228        6          1874
+FAST f=24 a=7       8.356826       10.803027        6          1106
+FAST f=24 a=7       0.085835       10.803027        6          1106
+FAST f=24 a=8       6.756811       10.522049        8          1298
+FAST f=24 a=8       0.07107       10.522049        8          1298
+FAST f=24 a=9       6.548169       10.571882        8          1682
+FAST f=24 a=9       0.0713       10.571882        8          1682
+FAST f=24 a=10       8.238079       10.736453        8          1106
+FAST f=24 a=10       0.07004       10.736453        8          1106
+
+
+hg-commands:
+NODICT       0.000005       2.425276        
+RANDOM       0.046332       3.490331        
+LEGACY       0.720351       3.911682        
+COVER       45.507731       4.132653        8          386
+COVER       1.868810       4.132653        8          386
+FAST f=15 a=1       4.561427       3.866894        8          1202
+FAST f=15 a=1       0.048946       3.866894        8          1202
+FAST f=15 a=2       3.574462       3.892119        8          1538
+FAST f=15 a=2       0.033677       3.892119        8          1538
+FAST f=15 a=3       3.230227       3.888791        6          1346
+FAST f=15 a=3       0.034312       3.888791        6          1346
+FAST f=15 a=4       3.042388       3.899739        8          1010
+FAST f=15 a=4       0.024307       3.899739        8          1010
+FAST f=15 a=5       2.800148       3.896220        8          818
+FAST f=15 a=5       0.022331       3.896220        8          818
+FAST f=15 a=6       2.706518       3.882039        8          578
+FAST f=15 a=6       0.020955       3.882039        8          578
+FAST f=15 a=7       2.701820       3.885430        6          866
+FAST f=15 a=7       0.026074       3.885430        6          866
+FAST f=15 a=8       2.604445       3.906932        8          1826
+FAST f=15 a=8       0.021789       3.906932        8          1826
+FAST f=15 a=9       2.598568       3.870324        6          1682
+FAST f=15 a=9       0.026004       3.870324        6          1682
+FAST f=15 a=10       2.575920       3.920783        8          1442
+FAST f=15 a=10       0.020228       3.920783        8          1442
+FAST f=16 a=1       4.630623       4.001430        8          770
+FAST f=16 a=1       0.047497       4.001430        8          770
+FAST f=16 a=2       3.674721       3.974431        8          1874
+FAST f=16 a=2       0.035761       3.974431        8          1874
+FAST f=16 a=3       3.338384       3.978703        8          1010
+FAST f=16 a=3       0.029436       3.978703        8          1010
+FAST f=16 a=4       3.004412       3.983035        8          1010
+FAST f=16 a=4       0.025744       3.983035        8          1010
+FAST f=16 a=5       2.881892       3.987710        8          770
+FAST f=16 a=5       0.023211       3.987710        8          770
+FAST f=16 a=6       2.807410       3.952717        8          1298
+FAST f=16 a=6       0.023199       3.952717        8          1298
+FAST f=16 a=7       2.819623       3.994627        8          770
+FAST f=16 a=7       0.021806       3.994627        8          770
+FAST f=16 a=8       2.740092       3.954032        8          1826
+FAST f=16 a=8       0.0226       3.954032        8          1826
+FAST f=16 a=9       2.682564       3.969879        6          1442
+FAST f=16 a=9       0.026324       3.969879        6          1442
+FAST f=16 a=10       2.657959       3.969755        8          674
+FAST f=16 a=10       0.020413       3.969755        8          674
+FAST f=17 a=1       4.729228       4.046000        8          530
+FAST f=17 a=1       0.049703       4.046000        8          530
+FAST f=17 a=2       3.764510       3.991519        8          1970
+FAST f=17 a=2       0.038195       3.991519        8          1970
+FAST f=17 a=3       3.416992       4.006296        6          914
+FAST f=17 a=3       0.036244       4.006296        6          914
+FAST f=17 a=4       3.145626       3.979182        8          1970
+FAST f=17 a=4       0.028676       3.979182        8          1970
+FAST f=17 a=5       2.995070       4.050070        8          770
+FAST f=17 a=5       0.025707       4.050070        8          770
+FAST f=17 a=6       2.911833       4.040024        8          770
+FAST f=17 a=6       0.02453       4.040024        8          770
+FAST f=17 a=7       2.894796       4.015884        8          818
+FAST f=17 a=7       0.023956       4.015884        8          818
+FAST f=17 a=8       2.789962       4.039303        8          530
+FAST f=17 a=8       0.023219       4.039303        8          530
+FAST f=17 a=9       2.787625       3.996762        8          1634
+FAST f=17 a=9       0.023651       3.996762        8          1634
+FAST f=17 a=10       2.754796       4.005059        8          1058
+FAST f=17 a=10       0.022537       4.005059        8          1058
+FAST f=18 a=1       4.779117       4.038214        8          242
+FAST f=18 a=1       0.048814       4.038214        8          242
+FAST f=18 a=2       3.829753       4.045768        8          722
+FAST f=18 a=2       0.036541       4.045768        8          722
+FAST f=18 a=3       3.495053       4.021497        8          770
+FAST f=18 a=3       0.032648       4.021497        8          770
+FAST f=18 a=4       3.221395       4.039623        8          770
+FAST f=18 a=4       0.027818       4.039623        8          770
+FAST f=18 a=5       3.059369       4.050414        8          530
+FAST f=18 a=5       0.026296       4.050414        8          530
+FAST f=18 a=6       3.019292       4.010714        6          962
+FAST f=18 a=6       0.031104       4.010714        6          962
+FAST f=18 a=7       2.949322       4.031439        6          770
+FAST f=18 a=7       0.030745       4.031439        6          770
+FAST f=18 a=8       2.876425       4.032088        6          386
+FAST f=18 a=8       0.027407       4.032088        6          386
+FAST f=18 a=9       2.850958       4.053372        8          674
+FAST f=18 a=9       0.023799       4.053372        8          674
+FAST f=18 a=10       2.884352       4.020148        8          1730
+FAST f=18 a=10       0.024401       4.020148        8          1730
+FAST f=19 a=1       4.815669       4.061203        8          674
+FAST f=19 a=1       0.051425       4.061203        8          674
+FAST f=19 a=2       3.951356       4.013822        8          1442
+FAST f=19 a=2       0.039968       4.013822        8          1442
+FAST f=19 a=3       3.554682       4.050425        8          722
+FAST f=19 a=3       0.032725       4.050425        8          722
+FAST f=19 a=4       3.242585       4.054677        8          722
+FAST f=19 a=4       0.028194       4.054677        8          722
+FAST f=19 a=5       3.105909       4.064524        8          818
+FAST f=19 a=5       0.02675       4.064524        8          818
+FAST f=19 a=6       3.059901       4.036857        8          1250
+FAST f=19 a=6       0.026396       4.036857        8          1250
+FAST f=19 a=7       3.016151       4.068234        6          770
+FAST f=19 a=7       0.031501       4.068234        6          770
+FAST f=19 a=8       2.962902       4.077509        8          530
+FAST f=19 a=8       0.023333       4.077509        8          530
+FAST f=19 a=9       2.899607       4.067328        8          530
+FAST f=19 a=9       0.024553       4.067328        8          530
+FAST f=19 a=10       2.950978       4.059901        8          434
+FAST f=19 a=10       0.023852       4.059901        8          434
+FAST f=20 a=1       5.259834       4.027579        8          1634
+FAST f=20 a=1       0.061123       4.027579        8          1634
+FAST f=20 a=2       4.382150       4.025093        8          1634
+FAST f=20 a=2       0.048009       4.025093        8          1634
+FAST f=20 a=3       4.104323       4.060842        8          530
+FAST f=20 a=3       0.040965       4.060842        8          530
+FAST f=20 a=4       3.853340       4.023504        6          914
+FAST f=20 a=4       0.041072       4.023504        6          914
+FAST f=20 a=5       3.728841       4.018089        6          1634
+FAST f=20 a=5       0.037469       4.018089        6          1634
+FAST f=20 a=6       3.683045       4.069138        8          578
+FAST f=20 a=6       0.028011       4.069138        8          578
+FAST f=20 a=7       3.726973       4.063160        8          722
+FAST f=20 a=7       0.028437       4.063160        8          722
+FAST f=20 a=8       3.555073       4.057690        8          386
+FAST f=20 a=8       0.027588       4.057690        8          386
+FAST f=20 a=9       3.551095       4.067253        8          482
+FAST f=20 a=9       0.025976       4.067253        8          482
+FAST f=20 a=10       3.490127       4.068518        8          530
+FAST f=20 a=10       0.025971       4.068518        8          530
+FAST f=21 a=1       7.343816       4.064945        8          770
+FAST f=21 a=1       0.085035       4.064945        8          770
+FAST f=21 a=2       5.930894       4.048206        8          386
+FAST f=21 a=2       0.067349       4.048206        8          386
+FAST f=21 a=3       6.770775       4.063417        8          578
+FAST f=21 a=3       0.077104       4.063417        8          578
+FAST f=21 a=4       6.889409       4.066761        8          626
+FAST f=21 a=4       0.0717       4.066761        8          626
+FAST f=21 a=5       6.714896       4.051813        8          914
+FAST f=21 a=5       0.071026       4.051813        8          914
+FAST f=21 a=6       6.539890       4.047263        8          1922
+FAST f=21 a=6       0.07127       4.047263        8          1922
+FAST f=21 a=7       6.511052       4.068373        8          482
+FAST f=21 a=7       0.065467       4.068373        8          482
+FAST f=21 a=8       6.458788       4.071597        8          482
+FAST f=21 a=8       0.063817       4.071597        8          482
+FAST f=21 a=9       6.377591       4.052905        8          434
+FAST f=21 a=9       0.063112       4.052905        8          434
+FAST f=21 a=10       6.360752       4.047773        8          530
+FAST f=21 a=10       0.063606       4.047773        8          530
+FAST f=22 a=1       10.523471       4.040812        8          962
+FAST f=22 a=1       0.14214       4.040812        8          962
+FAST f=22 a=2       9.454758       4.059396        8          914
+FAST f=22 a=2       0.118343       4.059396        8          914
+FAST f=22 a=3       9.043197       4.043019        8          1922
+FAST f=22 a=3       0.109798       4.043019        8          1922
+FAST f=22 a=4       8.716261       4.044819        8          770
+FAST f=22 a=4       0.099687       4.044819        8          770
+FAST f=22 a=5       8.529472       4.070576        8          530
+FAST f=22 a=5       0.093127       4.070576        8          530
+FAST f=22 a=6       8.424241       4.070565        8          722
+FAST f=22 a=6       0.093703       4.070565        8          722
+FAST f=22 a=7       8.403391       4.070591        8          578
+FAST f=22 a=7       0.089763       4.070591        8          578
+FAST f=22 a=8       8.285221       4.089171        8          530
+FAST f=22 a=8       0.087716       4.089171        8          530
+FAST f=22 a=9       8.282506       4.047470        8          722
+FAST f=22 a=9       0.089773       4.047470        8          722
+FAST f=22 a=10       8.241809       4.064151        8          818
+FAST f=22 a=10       0.090413       4.064151        8          818
+FAST f=23 a=1       12.389208       4.051635        6          530
+FAST f=23 a=1       0.147796       4.051635        6          530
+FAST f=23 a=2       11.300910       4.042835        6          914
+FAST f=23 a=2       0.133178       4.042835        6          914
+FAST f=23 a=3       10.879455       4.047415        8          626
+FAST f=23 a=3       0.129571       4.047415        8          626
+FAST f=23 a=4       10.522718       4.038269        6          914
+FAST f=23 a=4       0.118121       4.038269        6          914
+FAST f=23 a=5       10.348043       4.066884        8          434
+FAST f=23 a=5       0.112098       4.066884        8          434
+FAST f=23 a=6       10.238630       4.048635        8          1010
+FAST f=23 a=6       0.120281       4.048635        8          1010
+FAST f=23 a=7       10.213255       4.061809        8          530
+FAST f=23 a=7       0.1121       4.061809        8          530
+FAST f=23 a=8       10.107879       4.074104        8          818
+FAST f=23 a=8       0.116544       4.074104        8          818
+FAST f=23 a=9       10.063424       4.064811        8          674
+FAST f=23 a=9       0.109045       4.064811        8          674
+FAST f=23 a=10       10.035801       4.054918        8          530
+FAST f=23 a=10       0.108735       4.054918        8          530
+FAST f=24 a=1       14.963878       4.073490        8          722
+FAST f=24 a=1       0.206344       4.073490        8          722
+FAST f=24 a=2       13.833472       4.036100        8          962
+FAST f=24 a=2       0.17486       4.036100        8          962
+FAST f=24 a=3       13.404631       4.026281        6          1106
+FAST f=24 a=3       0.153961       4.026281        6          1106
+FAST f=24 a=4       13.041164       4.065448        8          674
+FAST f=24 a=4       0.155509       4.065448        8          674
+FAST f=24 a=5       12.879412       4.054636        8          674
+FAST f=24 a=5       0.148282       4.054636        8          674
+FAST f=24 a=6       12.773736       4.081376        8          530
+FAST f=24 a=6       0.142563       4.081376        8          530
+FAST f=24 a=7       12.711310       4.059834        8          770
+FAST f=24 a=7       0.149321       4.059834        8          770
+FAST f=24 a=8       12.635459       4.052050        8          1298
+FAST f=24 a=8       0.15095       4.052050        8          1298
+FAST f=24 a=9       12.558104       4.076516        8          722
+FAST f=24 a=9       0.144361       4.076516        8          722
+FAST f=24 a=10       10.661348       4.062137        8          818
+FAST f=24 a=10       0.108232       4.062137        8          818
+
+
+hg-changelog:
+NODICT       0.000017       1.377590        
+RANDOM       0.186171       2.097487        
+LEGACY       1.670867       2.058907        
+COVER       173.561948       2.189685        8          98
+COVER       4.811180       2.189685        8          98
+FAST f=15 a=1       18.685906       2.129682        8          434
+FAST f=15 a=1       0.173376       2.129682        8          434
+FAST f=15 a=2       12.928259       2.131890        8          482
+FAST f=15 a=2       0.102582       2.131890        8          482
+FAST f=15 a=3       11.132343       2.128027        8          386
+FAST f=15 a=3       0.077122       2.128027        8          386
+FAST f=15 a=4       10.120683       2.125797        8          434
+FAST f=15 a=4       0.065175       2.125797        8          434
+FAST f=15 a=5       9.479092       2.127697        8          386
+FAST f=15 a=5       0.057905       2.127697        8          386
+FAST f=15 a=6       9.159523       2.127132        8          1682
+FAST f=15 a=6       0.058604       2.127132        8          1682
+FAST f=15 a=7       8.724003       2.129914        8          434
+FAST f=15 a=7       0.0493       2.129914        8          434
+FAST f=15 a=8       8.595001       2.127137        8          338
+FAST f=15 a=8       0.0474       2.127137        8          338
+FAST f=15 a=9       8.356405       2.125512        8          482
+FAST f=15 a=9       0.046126       2.125512        8          482
+FAST f=15 a=10       8.207111       2.126066        8          338
+FAST f=15 a=10       0.043292       2.126066        8          338
+FAST f=16 a=1       18.464436       2.144040        8          242
+FAST f=16 a=1       0.172156       2.144040        8          242
+FAST f=16 a=2       12.844825       2.148171        8          194
+FAST f=16 a=2       0.099619       2.148171        8          194
+FAST f=16 a=3       11.082568       2.140837        8          290
+FAST f=16 a=3       0.079165       2.140837        8          290
+FAST f=16 a=4       10.066749       2.144405        8          386
+FAST f=16 a=4       0.068411       2.144405        8          386
+FAST f=16 a=5       9.501121       2.140720        8          386
+FAST f=16 a=5       0.061316       2.140720        8          386
+FAST f=16 a=6       9.179332       2.139478        8          386
+FAST f=16 a=6       0.056322       2.139478        8          386
+FAST f=16 a=7       8.849438       2.142412        8          194
+FAST f=16 a=7       0.050493       2.142412        8          194
+FAST f=16 a=8       8.810919       2.143454        8          434
+FAST f=16 a=8       0.051304       2.143454        8          434
+FAST f=16 a=9       8.553900       2.140339        8          194
+FAST f=16 a=9       0.047285       2.140339        8          194
+FAST f=16 a=10       8.398027       2.143130        8          386
+FAST f=16 a=10       0.046386       2.143130        8          386
+FAST f=17 a=1       18.644657       2.157192        8          98
+FAST f=17 a=1       0.173884       2.157192        8          98
+FAST f=17 a=2       13.071242       2.159830        8          146
+FAST f=17 a=2       0.10388       2.159830        8          146
+FAST f=17 a=3       11.332366       2.153654        6          194
+FAST f=17 a=3       0.08983       2.153654        6          194
+FAST f=17 a=4       10.362413       2.156813        8          242
+FAST f=17 a=4       0.070389       2.156813        8          242
+FAST f=17 a=5       9.808159       2.155098        6          338
+FAST f=17 a=5       0.072661       2.155098        6          338
+FAST f=17 a=6       9.451165       2.153845        6          146
+FAST f=17 a=6       0.064959       2.153845        6          146
+FAST f=17 a=7       9.163097       2.155424        6          242
+FAST f=17 a=7       0.064323       2.155424        6          242
+FAST f=17 a=8       9.047276       2.156640        8          242
+FAST f=17 a=8       0.053382       2.156640        8          242
+FAST f=17 a=9       8.807671       2.152396        8          146
+FAST f=17 a=9       0.049617       2.152396        8          146
+FAST f=17 a=10       8.649827       2.152370        8          146
+FAST f=17 a=10       0.047849       2.152370        8          146
+FAST f=18 a=1       18.809502       2.168116        8          98
+FAST f=18 a=1       0.175226       2.168116        8          98
+FAST f=18 a=2       13.756502       2.170870        6          242
+FAST f=18 a=2       0.119507       2.170870        6          242
+FAST f=18 a=3       12.059748       2.163094        6          98
+FAST f=18 a=3       0.093912       2.163094        6          98
+FAST f=18 a=4       11.410294       2.172372        8          98
+FAST f=18 a=4       0.073048       2.172372        8          98
+FAST f=18 a=5       10.560297       2.166388        8          98
+FAST f=18 a=5       0.065136       2.166388        8          98
+FAST f=18 a=6       10.071390       2.162672        8          98
+FAST f=18 a=6       0.059402       2.162672        8          98
+FAST f=18 a=7       10.084214       2.166624        6          194
+FAST f=18 a=7       0.073276       2.166624        6          194
+FAST f=18 a=8       9.953226       2.167454        8          98
+FAST f=18 a=8       0.053659       2.167454        8          98
+FAST f=18 a=9       8.982461       2.161593        6          146
+FAST f=18 a=9       0.05955       2.161593        6          146
+FAST f=18 a=10       8.986092       2.164373        6          242
+FAST f=18 a=10       0.059135       2.164373        6          242
+FAST f=19 a=1       18.908277       2.176021        8          98
+FAST f=19 a=1       0.177316       2.176021        8          98
+FAST f=19 a=2       13.471313       2.176103        8          98
+FAST f=19 a=2       0.106344       2.176103        8          98
+FAST f=19 a=3       11.571406       2.172812        8          98
+FAST f=19 a=3       0.083293       2.172812        8          98
+FAST f=19 a=4       10.632775       2.177770        6          146
+FAST f=19 a=4       0.079864       2.177770        6          146
+FAST f=19 a=5       10.030190       2.175574        6          146
+FAST f=19 a=5       0.07223       2.175574        6          146
+FAST f=19 a=6       9.717818       2.169997        8          98
+FAST f=19 a=6       0.060049       2.169997        8          98
+FAST f=19 a=7       9.397531       2.172770        8          146
+FAST f=19 a=7       0.057188       2.172770        8          146
+FAST f=19 a=8       9.281061       2.175822        8          98
+FAST f=19 a=8       0.053711       2.175822        8          98
+FAST f=19 a=9       9.165242       2.169849        6          146
+FAST f=19 a=9       0.059898       2.169849        6          146
+FAST f=19 a=10       9.048763       2.173394        8          98
+FAST f=19 a=10       0.049757       2.173394        8          98
+FAST f=20 a=1       21.166917       2.183923        6          98
+FAST f=20 a=1       0.205425       2.183923        6          98
+FAST f=20 a=2       15.642753       2.182349        6          98
+FAST f=20 a=2       0.135957       2.182349        6          98
+FAST f=20 a=3       14.053730       2.173544        6          98
+FAST f=20 a=3       0.11266       2.173544        6          98
+FAST f=20 a=4       15.270019       2.183656        8          98
+FAST f=20 a=4       0.107892       2.183656        8          98
+FAST f=20 a=5       15.497927       2.174661        6          98
+FAST f=20 a=5       0.100305       2.174661        6          98
+FAST f=20 a=6       13.973505       2.172391        8          98
+FAST f=20 a=6       0.087565       2.172391        8          98
+FAST f=20 a=7       14.083296       2.172443        8          98
+FAST f=20 a=7       0.078062       2.172443        8          98
+FAST f=20 a=8       12.560048       2.175581        8          98
+FAST f=20 a=8       0.070282       2.175581        8          98
+FAST f=20 a=9       13.078645       2.173975        6          146
+FAST f=20 a=9       0.081041       2.173975        6          146
+FAST f=20 a=10       12.823328       2.177778        8          98
+FAST f=20 a=10       0.074522       2.177778        8          98
+FAST f=21 a=1       29.825370       2.183057        6          98
+FAST f=21 a=1       0.334453       2.183057        6          98
+FAST f=21 a=2       29.476474       2.182752        8          98
+FAST f=21 a=2       0.286602       2.182752        8          98
+FAST f=21 a=3       25.937186       2.175867        8          98
+FAST f=21 a=3       0.17626       2.175867        8          98
+FAST f=21 a=4       20.413865       2.179780        8          98
+FAST f=21 a=4       0.206085       2.179780        8          98
+FAST f=21 a=5       20.541889       2.178328        6          146
+FAST f=21 a=5       0.199157       2.178328        6          146
+FAST f=21 a=6       21.090670       2.174443        6          146
+FAST f=21 a=6       0.190645       2.174443        6          146
+FAST f=21 a=7       20.221569       2.177384        6          146
+FAST f=21 a=7       0.184278       2.177384        6          146
+FAST f=21 a=8       20.322357       2.179456        6          98
+FAST f=21 a=8       0.178458       2.179456        6          98
+FAST f=21 a=9       20.683912       2.174396        6          146
+FAST f=21 a=9       0.190829       2.174396        6          146
+FAST f=21 a=10       20.840865       2.174905        8          98
+FAST f=21 a=10       0.172515       2.174905        8          98
+FAST f=22 a=1       36.822827       2.181612        6          98
+FAST f=22 a=1       0.437389       2.181612        6          98
+FAST f=22 a=2       30.616902       2.183142        8          98
+FAST f=22 a=2       0.324284       2.183142        8          98
+FAST f=22 a=3       28.472482       2.178130        8          98
+FAST f=22 a=3       0.236538       2.178130        8          98
+FAST f=22 a=4       25.847028       2.181878        8          98
+FAST f=22 a=4       0.263744       2.181878        8          98
+FAST f=22 a=5       27.095881       2.180775        8          98
+FAST f=22 a=5       0.24988       2.180775        8          98
+FAST f=22 a=6       25.939172       2.170916        8          98
+FAST f=22 a=6       0.240033       2.170916        8          98
+FAST f=22 a=7       27.064194       2.177849        8          98
+FAST f=22 a=7       0.242383       2.177849        8          98
+FAST f=22 a=8       25.140221       2.178216        8          98
+FAST f=22 a=8       0.237601       2.178216        8          98
+FAST f=22 a=9       25.505283       2.177455        6          146
+FAST f=22 a=9       0.223217       2.177455        6          146
+FAST f=22 a=10       24.529362       2.176705        6          98
+FAST f=22 a=10       0.222876       2.176705        6          98
+FAST f=23 a=1       39.127310       2.183006        6          98
+FAST f=23 a=1       0.417338       2.183006        6          98
+FAST f=23 a=2       32.468161       2.183524        6          98
+FAST f=23 a=2       0.351645       2.183524        6          98
+FAST f=23 a=3       31.577620       2.172604        6          98
+FAST f=23 a=3       0.319659       2.172604        6          98
+FAST f=23 a=4       30.129247       2.183932        6          98
+FAST f=23 a=4       0.307239       2.183932        6          98
+FAST f=23 a=5       29.103376       2.183529        6          146
+FAST f=23 a=5       0.285533       2.183529        6          146
+FAST f=23 a=6       29.776045       2.174367        8          98
+FAST f=23 a=6       0.276846       2.174367        8          98
+FAST f=23 a=7       28.940407       2.178022        6          146
+FAST f=23 a=7       0.274082       2.178022        6          146
+FAST f=23 a=8       29.256009       2.179462        6          98
+FAST f=23 a=8       0.26949       2.179462        6          98
+FAST f=23 a=9       29.347312       2.170407        8          98
+FAST f=23 a=9       0.265034       2.170407        8          98
+FAST f=23 a=10       29.140081       2.171762        8          98
+FAST f=23 a=10       0.259183       2.171762        8          98
+FAST f=24 a=1       44.871179       2.182115        6          98
+FAST f=24 a=1       0.509433       2.182115        6          98
+FAST f=24 a=2       38.694867       2.180549        8          98
+FAST f=24 a=2       0.406695       2.180549        8          98
+FAST f=24 a=3       38.363769       2.172821        8          98
+FAST f=24 a=3       0.359581       2.172821        8          98
+FAST f=24 a=4       36.580797       2.184142        8          98
+FAST f=24 a=4       0.340614       2.184142        8          98
+FAST f=24 a=5       33.125701       2.183301        8          98
+FAST f=24 a=5       0.324874       2.183301        8          98
+FAST f=24 a=6       34.776068       2.173019        6          146
+FAST f=24 a=6       0.340397       2.173019        6          146
+FAST f=24 a=7       34.417625       2.176561        6          146
+FAST f=24 a=7       0.308223       2.176561        6          146
+FAST f=24 a=8       35.470291       2.182161        6          98
+FAST f=24 a=8       0.307724       2.182161        6          98
+FAST f=24 a=9       34.927252       2.172682        6          146
+FAST f=24 a=9       0.300598       2.172682        6          146
+FAST f=24 a=10       33.238355       2.173395        6          98
+FAST f=24 a=10       0.249916       2.173395        6          98
+
+
+hg-manifest:
+NODICT       0.000004       1.866377        
+RANDOM       0.696346       2.309436        
+LEGACY       7.064527       2.506977        
+COVER       876.312865       2.582528        8          434
+COVER       35.684533       2.582528        8          434
+FAST f=15 a=1       76.618201       2.404013        8          1202
+FAST f=15 a=1       0.700722       2.404013        8          1202
+FAST f=15 a=2       49.213058       2.409248        6          1826
+FAST f=15 a=2       0.473393       2.409248        6          1826
+FAST f=15 a=3       41.753197       2.409677        8          1490
+FAST f=15 a=3       0.336848       2.409677        8          1490
+FAST f=15 a=4       38.648295       2.407996        8          1538
+FAST f=15 a=4       0.283952       2.407996        8          1538
+FAST f=15 a=5       36.144936       2.402895        8          1874
+FAST f=15 a=5       0.270128       2.402895        8          1874
+FAST f=15 a=6       35.484675       2.394873        8          1586
+FAST f=15 a=6       0.251637       2.394873        8          1586
+FAST f=15 a=7       34.280599       2.397311        8          1778
+FAST f=15 a=7       0.23984       2.397311        8          1778
+FAST f=15 a=8       32.122572       2.396089        6          1490
+FAST f=15 a=8       0.251508       2.396089        6          1490
+FAST f=15 a=9       29.909842       2.390092        6          1970
+FAST f=15 a=9       0.251233       2.390092        6          1970
+FAST f=15 a=10       30.102938       2.400086        6          1682
+FAST f=15 a=10       0.23688       2.400086        6          1682
+FAST f=16 a=1       67.750401       2.475460        6          1346
+FAST f=16 a=1       0.796035       2.475460        6          1346
+FAST f=16 a=2       52.812027       2.480860        6          1730
+FAST f=16 a=2       0.480384       2.480860        6          1730
+FAST f=16 a=3       44.179259       2.469304        8          1970
+FAST f=16 a=3       0.332657       2.469304        8          1970
+FAST f=16 a=4       37.612728       2.478208        6          1970
+FAST f=16 a=4       0.32498       2.478208        6          1970
+FAST f=16 a=5       35.056222       2.475568        6          1298
+FAST f=16 a=5       0.302824       2.475568        6          1298
+FAST f=16 a=6       34.713012       2.486079        8          1730
+FAST f=16 a=6       0.24755       2.486079        8          1730
+FAST f=16 a=7       33.713687       2.477180        6          1682
+FAST f=16 a=7       0.280358       2.477180        6          1682
+FAST f=16 a=8       31.571412       2.475418        8          1538
+FAST f=16 a=8       0.241241       2.475418        8          1538
+FAST f=16 a=9       31.608069       2.478263        8          1922
+FAST f=16 a=9       0.241764       2.478263        8          1922
+FAST f=16 a=10       31.358002       2.472263        8          1442
+FAST f=16 a=10       0.221661       2.472263        8          1442
+FAST f=17 a=1       66.185775       2.536085        6          1346
+FAST f=17 a=1       0.713549       2.536085        6          1346
+FAST f=17 a=2       50.365000       2.546105        8          1298
+FAST f=17 a=2       0.467846       2.546105        8          1298
+FAST f=17 a=3       42.712843       2.536250        8          1298
+FAST f=17 a=3       0.34047       2.536250        8          1298
+FAST f=17 a=4       39.514227       2.535555        8          1442
+FAST f=17 a=4       0.302989       2.535555        8          1442
+FAST f=17 a=5       35.189292       2.524925        8          1202
+FAST f=17 a=5       0.273451       2.524925        8          1202
+FAST f=17 a=6       35.791683       2.523466        8          1202
+FAST f=17 a=6       0.268261       2.523466        8          1202
+FAST f=17 a=7       37.416136       2.526625        6          1010
+FAST f=17 a=7       0.277558       2.526625        6          1010
+FAST f=17 a=8       37.084707       2.533274        6          1250
+FAST f=17 a=8       0.285104       2.533274        6          1250
+FAST f=17 a=9       34.183814       2.532765        8          1298
+FAST f=17 a=9       0.235133       2.532765        8          1298
+FAST f=17 a=10       31.149235       2.528722        8          1346
+FAST f=17 a=10       0.232679       2.528722        8          1346
+FAST f=18 a=1       72.942176       2.559857        6          386
+FAST f=18 a=1       0.718618       2.559857        6          386
+FAST f=18 a=2       51.690440       2.559572        8          290
+FAST f=18 a=2       0.403978       2.559572        8          290
+FAST f=18 a=3       45.344908       2.561040        8          962
+FAST f=18 a=3       0.357205       2.561040        8          962
+FAST f=18 a=4       39.804522       2.558446        8          1010
+FAST f=18 a=4       0.310526       2.558446        8          1010
+FAST f=18 a=5       38.134888       2.561811        8          626
+FAST f=18 a=5       0.273743       2.561811        8          626
+FAST f=18 a=6       35.091890       2.555518        8          722
+FAST f=18 a=6       0.260135       2.555518        8          722
+FAST f=18 a=7       34.639523       2.562938        8          290
+FAST f=18 a=7       0.234294       2.562938        8          290
+FAST f=18 a=8       36.076431       2.563567        8          1586
+FAST f=18 a=8       0.274075       2.563567        8          1586
+FAST f=18 a=9       36.376433       2.560950        8          722
+FAST f=18 a=9       0.240106       2.560950        8          722
+FAST f=18 a=10       32.624790       2.559340        8          578
+FAST f=18 a=10       0.234704       2.559340        8          578
+FAST f=19 a=1       70.513761       2.572441        8          194
+FAST f=19 a=1       0.726112       2.572441        8          194
+FAST f=19 a=2       59.263032       2.574560        8          482
+FAST f=19 a=2       0.451554       2.574560        8          482
+FAST f=19 a=3       51.509594       2.571546        6          194
+FAST f=19 a=3       0.393014       2.571546        6          194
+FAST f=19 a=4       55.393906       2.573386        8          482
+FAST f=19 a=4       0.38819       2.573386        8          482
+FAST f=19 a=5       43.201736       2.567589        8          674
+FAST f=19 a=5       0.292155       2.567589        8          674
+FAST f=19 a=6       42.911687       2.572666        6          434
+FAST f=19 a=6       0.303988       2.572666        6          434
+FAST f=19 a=7       44.687591       2.573613        6          290
+FAST f=19 a=7       0.308721       2.573613        6          290
+FAST f=19 a=8       37.372868       2.571039        6          194
+FAST f=19 a=8       0.287137       2.571039        6          194
+FAST f=19 a=9       36.074230       2.566473        6          482
+FAST f=19 a=9       0.280721       2.566473        6          482
+FAST f=19 a=10       33.731720       2.570306        8          194
+FAST f=19 a=10       0.224073       2.570306        8          194
+FAST f=20 a=1       79.670634       2.581146        6          290
+FAST f=20 a=1       0.899986       2.581146        6          290
+FAST f=20 a=2       58.827141       2.579782        8          386
+FAST f=20 a=2       0.602288       2.579782        8          386
+FAST f=20 a=3       51.289004       2.579627        8          722
+FAST f=20 a=3       0.446091       2.579627        8          722
+FAST f=20 a=4       47.711068       2.581508        8          722
+FAST f=20 a=4       0.473007       2.581508        8          722
+FAST f=20 a=5       47.402929       2.578062        6          434
+FAST f=20 a=5       0.497131       2.578062        6          434
+FAST f=20 a=6       54.797102       2.577365        8          482
+FAST f=20 a=6       0.515061       2.577365        8          482
+FAST f=20 a=7       51.370877       2.583050        8          386
+FAST f=20 a=7       0.402878       2.583050        8          386
+FAST f=20 a=8       51.437931       2.574875        6          242
+FAST f=20 a=8       0.453094       2.574875        6          242
+FAST f=20 a=9       44.105456       2.576700        6          242
+FAST f=20 a=9       0.456633       2.576700        6          242
+FAST f=20 a=10       44.447580       2.578305        8          338
+FAST f=20 a=10       0.409121       2.578305        8          338
+FAST f=21 a=1       113.031686       2.582449        6          242
+FAST f=21 a=1       1.456971       2.582449        6          242
+FAST f=21 a=2       97.700932       2.582124        8          194
+FAST f=21 a=2       1.072078       2.582124        8          194
+FAST f=21 a=3       96.563648       2.585479        8          434
+FAST f=21 a=3       0.949528       2.585479        8          434
+FAST f=21 a=4       90.597813       2.582366        6          386
+FAST f=21 a=4       0.76944       2.582366        6          386
+FAST f=21 a=5       86.815980       2.579043        8          434
+FAST f=21 a=5       0.858167       2.579043        8          434
+FAST f=21 a=6       91.235820       2.578378        8          530
+FAST f=21 a=6       0.684274       2.578378        8          530
+FAST f=21 a=7       84.392788       2.581243        8          386
+FAST f=21 a=7       0.814386       2.581243        8          386
+FAST f=21 a=8       82.052310       2.582547        8          338
+FAST f=21 a=8       0.822633       2.582547        8          338
+FAST f=21 a=9       74.696074       2.579319        8          194
+FAST f=21 a=9       0.811028       2.579319        8          194
+FAST f=21 a=10       76.211170       2.578766        8          290
+FAST f=21 a=10       0.809715       2.578766        8          290
+FAST f=22 a=1       138.976871       2.580478        8          194
+FAST f=22 a=1       1.748932       2.580478        8          194
+FAST f=22 a=2       120.164097       2.583633        8          386
+FAST f=22 a=2       1.333239       2.583633        8          386
+FAST f=22 a=3       111.986474       2.582566        6          194
+FAST f=22 a=3       1.305734       2.582566        6          194
+FAST f=22 a=4       108.548148       2.583068        6          194
+FAST f=22 a=4       1.314026       2.583068        6          194
+FAST f=22 a=5       103.173017       2.583495        6          290
+FAST f=22 a=5       1.228664       2.583495        6          290
+FAST f=22 a=6       108.421262       2.582349        8          530
+FAST f=22 a=6       1.076773       2.582349        8          530
+FAST f=22 a=7       103.284127       2.581022        8          386
+FAST f=22 a=7       1.112117       2.581022        8          386
+FAST f=22 a=8       96.330279       2.581073        8          290
+FAST f=22 a=8       1.109303       2.581073        8          290
+FAST f=22 a=9       97.651348       2.580075        6          194
+FAST f=22 a=9       0.933032       2.580075        6          194
+FAST f=22 a=10       101.660621       2.584886        8          194
+FAST f=22 a=10       0.796823       2.584886        8          194
+FAST f=23 a=1       159.322978       2.581474        6          242
+FAST f=23 a=1       2.015878       2.581474        6          242
+FAST f=23 a=2       134.331775       2.581619        8          194
+FAST f=23 a=2       1.545845       2.581619        8          194
+FAST f=23 a=3       127.724552       2.579888        6          338
+FAST f=23 a=3       1.444496       2.579888        6          338
+FAST f=23 a=4       126.077675       2.578137        6          242
+FAST f=23 a=4       1.364394       2.578137        6          242
+FAST f=23 a=5       124.914027       2.580843        8          338
+FAST f=23 a=5       1.116059       2.580843        8          338
+FAST f=23 a=6       122.874153       2.577637        6          338
+FAST f=23 a=6       1.164584       2.577637        6          338
+FAST f=23 a=7       123.099257       2.582715        6          386
+FAST f=23 a=7       1.354042       2.582715        6          386
+FAST f=23 a=8       122.026753       2.577681        8          194
+FAST f=23 a=8       1.210966       2.577681        8          194
+FAST f=23 a=9       121.164312       2.584599        6          290
+FAST f=23 a=9       1.174859       2.584599        6          290
+FAST f=23 a=10       117.462222       2.580358        8          194
+FAST f=23 a=10       1.075258       2.580358        8          194
+FAST f=24 a=1       169.539659       2.581642        6          194
+FAST f=24 a=1       1.916804       2.581642        6          194
+FAST f=24 a=2       160.539270       2.580421        6          290
+FAST f=24 a=2       1.71087       2.580421        6          290
+FAST f=24 a=3       155.455874       2.580449        6          242
+FAST f=24 a=3       1.60307       2.580449        6          242
+FAST f=24 a=4       147.630320       2.582953        6          338
+FAST f=24 a=4       1.396364       2.582953        6          338
+FAST f=24 a=5       133.767428       2.580589        6          290
+FAST f=24 a=5       1.19933       2.580589        6          290
+FAST f=24 a=6       146.437535       2.579453        8          194
+FAST f=24 a=6       1.385405       2.579453        8          194
+FAST f=24 a=7       147.227507       2.584155        8          386
+FAST f=24 a=7       1.48942       2.584155        8          386
+FAST f=24 a=8       138.005773       2.584115        8          194
+FAST f=24 a=8       1.352       2.584115        8          194
+FAST f=24 a=9       141.442625       2.582902        8          290
+FAST f=24 a=9       1.39647       2.582902        8          290
+FAST f=24 a=10       142.157446       2.582701        8          434
+FAST f=24 a=10       1.498889       2.582701        8          434

contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c

@@ -0,0 +1,442 @@
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h>   /* strcmp, strlen */
+#include <errno.h>    /* errno */
+#include <ctype.h>
+#include <time.h>
+#include "random.h"
+#include "dictBuilder.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#include "io.h"
+#include "util.h"
+#include "zdict.h"
+
+
+
+/*-*************************************
+*  Console display
+***************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
+#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (displayLevel>=4) fflush(stderr); } } }
+
+
+/*-*************************************
+*  Exceptions
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAY("Error %i : ", error);                                        \
+    DISPLAY(__VA_ARGS__);                                                 \
+    DISPLAY("\n");                                                        \
+    exit(error);                                                          \
+}
+
+
+/*-*************************************
+*  Constants
+***************************************/
+static const unsigned g_defaultMaxDictSize = 110 KB;
+#define DEFAULT_CLEVEL 3
+#define DEFAULT_DISPLAYLEVEL 2
+
+
+/*-*************************************
+*  Struct
+***************************************/
+typedef struct {
+  const void* dictBuffer;
+  size_t dictSize;
+} dictInfo;
+
+
+/*-*************************************
+* Dictionary related operations
+***************************************/
+/** createDictFromFiles() :
+ *  Based on type of param given, train dictionary using the corresponding algorithm
+ *  @return dictInfo containing dictionary buffer and dictionary size
+ */
+dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
+                  ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams,
+                  ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) {
+    unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel :
+                                  coverParams ? coverParams->zParams.notificationLevel :
+                                  legacyParams ? legacyParams->zParams.notificationLevel :
+                                  fastParams ? fastParams->zParams.notificationLevel :
+                                  DEFAULT_DISPLAYLEVEL;   /* no dict */
+    void* const dictBuffer = malloc(maxDictSize);
+
+    dictInfo* dInfo = NULL;
+
+    /* Checks */
+    if (!dictBuffer)
+        EXM_THROW(12, "not enough memory for trainFromFiles");   /* should not happen */
+
+    {   size_t dictSize;
+        if(randomParams) {
+          dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, *randomParams);
+        }else if(coverParams) {
+          /* Run the optimize version if either k or d is not provided */
+          if (!coverParams->d || !coverParams->k){
+            dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, coverParams);
+          } else {
+            dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, *coverParams);
+          }
+        } else if(legacyParams) {
+          dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer,
+                                               info->samplesSizes, info->nbSamples, *legacyParams);
+        } else if(fastParams) {
+          /* Run the optimize version if either k or d is not provided */
+          if (!fastParams->d || !fastParams->k) {
+            dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, fastParams);
+          } else {
+            dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
+                                                  info->samplesSizes, info->nbSamples, *fastParams);
+          }
+        } else {
+          dictSize = 0;
+        }
+        if (ZDICT_isError(dictSize)) {
+            DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize));   /* should not happen */
+            free(dictBuffer);
+            return dInfo;
+        }
+        dInfo = (dictInfo *)malloc(sizeof(dictInfo));
+        dInfo->dictBuffer = dictBuffer;
+        dInfo->dictSize = dictSize;
+    }
+    return dInfo;
+}
+
+
+/** compressWithDict() :
+ *  Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
+ *  @return compression ratio
+ */
+double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
+  /* Local variables */
+  size_t totalCompressedSize = 0;
+  size_t totalOriginalSize = 0;
+  const unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0;
+  double cRatio;
+  size_t dstCapacity;
+  int i;
+
+  /* Pointers */
+  ZSTD_CDict *cdict = NULL;
+  ZSTD_CCtx* cctx = NULL;
+  size_t *offsets = NULL;
+  void* dst = NULL;
+
+  /* Allocate dst with enough space to compress the maximum sized sample */
+  {
+    size_t maxSampleSize = 0;
+    for (i = 0; i < srcInfo->nbSamples; i++) {
+      maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
+    }
+    dstCapacity = ZSTD_compressBound(maxSampleSize);
+    dst = malloc(dstCapacity);
+  }
+
+  /* Calculate offset for each sample */
+  offsets = (size_t *)malloc((srcInfo->nbSamples + 1) * sizeof(size_t));
+  offsets[0] = 0;
+  for (i = 1; i <= srcInfo->nbSamples; i++) {
+    offsets[i] = offsets[i - 1] + srcInfo->samplesSizes[i - 1];
+  }
+
+  /* Create the cctx */
+  cctx = ZSTD_createCCtx();
+  if(!cctx || !dst) {
+    cRatio = -1;
+    goto _cleanup;
+  }
+
+  /* Create CDict if there's a dictionary stored on buffer */
+  if (hasDict) {
+    cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel);
+    if(!cdict) {
+      cRatio = -1;
+      goto _cleanup;
+    }
+  }
+
+  /* Compress each sample and sum their sizes*/
+  const BYTE *const samples = (const BYTE *)srcInfo->srcBuffer;
+  for (i = 0; i < srcInfo->nbSamples; i++) {
+    size_t compressedSize;
+    if(hasDict) {
+      compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict);
+    } else {
+      compressedSize = ZSTD_compressCCtx(cctx, dst, dstCapacity,samples + offsets[i], srcInfo->samplesSizes[i], compressionLevel);
+    }
+    if (ZSTD_isError(compressedSize)) {
+      cRatio = -1;
+      goto _cleanup;
+    }
+    totalCompressedSize += compressedSize;
+  }
+
+  /* Sum orignal sizes */
+  for (i = 0; i<srcInfo->nbSamples; i++) {
+    totalOriginalSize += srcInfo->samplesSizes[i];
+  }
+
+  /* Calculate compression ratio */
+  DISPLAYLEVEL(2, "original size is %lu\n", totalOriginalSize);
+  DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize);
+  cRatio = (double)totalOriginalSize/(double)totalCompressedSize;
+
+_cleanup:
+  free(dst);
+  free(offsets);
+  ZSTD_freeCCtx(cctx);
+  ZSTD_freeCDict(cdict);
+  return cRatio;
+}
+
+
+/** FreeDictInfo() :
+ *  Free memory allocated for dictInfo
+ */
+void freeDictInfo(dictInfo* info) {
+  if (!info) return;
+  if (info->dictBuffer) free((void*)(info->dictBuffer));
+  free(info);
+}
+
+
+
+/*-********************************************************
+  *  Benchmarking functions
+**********************************************************/
+/** benchmarkDictBuilder() :
+ *  Measure how long a dictionary builder takes and compression ratio with the dictionary built
+ *  @return 0 if benchmark successfully, 1 otherwise
+ */
+int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam,
+                        ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam,
+                        ZDICT_fastCover_params_t *fastParam) {
+  /* Local variables */
+  const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel :
+                                coverParam ? coverParam->zParams.notificationLevel :
+                                legacyParam ? legacyParam->zParams.notificationLevel :
+                                fastParam ? fastParam->zParams.notificationLevel:
+                                DEFAULT_DISPLAYLEVEL;   /* no dict */
+  const char* name = randomParam ? "RANDOM" :
+                    coverParam ? "COVER" :
+                    legacyParam ? "LEGACY" :
+                    fastParam ? "FAST":
+                    "NODICT";    /* no dict */
+  const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel :
+                          coverParam ? coverParam->zParams.compressionLevel :
+                          legacyParam ? legacyParam->zParams.compressionLevel :
+                          fastParam ? fastParam->zParams.compressionLevel:
+                          DEFAULT_CLEVEL;   /* no dict */
+  int result = 0;
+
+  /* Calculate speed */
+  const UTIL_time_t begin = UTIL_getTime();
+  dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam);
+  const U64 timeMicro = UTIL_clockSpanMicro(begin);
+  const double timeSec = timeMicro / (double)SEC_TO_MICRO;
+  if (!dInfo) {
+    DISPLAYLEVEL(1, "%s does not train successfully\n", name);
+    result = 1;
+    goto _cleanup;
+  }
+  DISPLAYLEVEL(1, "%s took %f seconds to execute \n", name, timeSec);
+
+  /* Calculate compression ratio */
+  const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel);
+  if (cRatio < 0) {
+    DISPLAYLEVEL(1, "Compressing with %s dictionary does not work\n", name);
+    result = 1;
+    goto _cleanup;
+
+  }
+  DISPLAYLEVEL(1, "Compression ratio with %s dictionary is %f\n", name, cRatio);
+
+_cleanup:
+  freeDictInfo(dInfo);
+  return result;
+}
+
+
+
+int main(int argCount, const char* argv[])
+{
+  const int displayLevel = DEFAULT_DISPLAYLEVEL;
+  const char* programName = argv[0];
+  int result = 0;
+
+  /* Initialize arguments to default values */
+  unsigned k = 200;
+  unsigned d = 8;
+  unsigned f;
+  unsigned accel;
+  unsigned i;
+  const unsigned cLevel = DEFAULT_CLEVEL;
+  const unsigned dictID = 0;
+  const unsigned maxDictSize = g_defaultMaxDictSize;
+
+  /* Initialize table to store input files */
+  const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
+  unsigned filenameIdx = 0;
+
+  char* fileNamesBuf = NULL;
+  unsigned fileNamesNb = filenameIdx;
+  const int followLinks = 0;
+  const char** extendedFileList = NULL;
+
+  /* Parse arguments */
+  for (i = 1; i < argCount; i++) {
+    const char* argument = argv[i];
+    if (longCommandWArg(&argument, "in=")) {
+      filenameTable[filenameIdx] = argument;
+      filenameIdx++;
+      continue;
+    }
+    DISPLAYLEVEL(1, "benchmark: Incorrect parameters\n");
+    return 1;
+  }
+
+  /* Get the list of all files recursively (because followLinks==0)*/
+  extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
+                                        &fileNamesNb, followLinks);
+  if (extendedFileList) {
+    unsigned u;
+    for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
+    free((void*)filenameTable);
+    filenameTable = extendedFileList;
+    filenameIdx = fileNamesNb;
+  }
+
+  /* get sampleInfo */
+  size_t blockSize = 0;
+  sampleInfo* srcInfo= getSampleInfo(filenameTable,
+                    filenameIdx, blockSize, maxDictSize, displayLevel);
+
+  /* set up zParams */
+  ZDICT_params_t zParams;
+  zParams.compressionLevel = cLevel;
+  zParams.notificationLevel = displayLevel;
+  zParams.dictID = dictID;
+
+  /* with no dict */
+  {
+    const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL);
+    if(noDictResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for random */
+  {
+    ZDICT_random_params_t randomParam;
+    randomParam.zParams = zParams;
+    randomParam.k = k;
+    const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\n", randomParam.k);
+    if(randomResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for legacy */
+  {
+    ZDICT_legacy_params_t legacyParam;
+    legacyParam.zParams = zParams;
+    legacyParam.selectivityLevel = 9;
+    const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL);
+    DISPLAYLEVEL(2, "selectivityLevel=%u\n", legacyParam.selectivityLevel);
+    if(legacyResult) {
+      result = 1;
+      goto _cleanup;
+    }
+  }
+
+  /* for cover */
+  {
+    /* for cover (optimizing k and d) */
+    ZDICT_cover_params_t coverParam;
+    memset(&coverParam, 0, sizeof(coverParam));
+    coverParam.zParams = zParams;
+    coverParam.splitPoint = 1.0;
+    coverParam.steps = 40;
+    coverParam.nbThreads = 1;
+    const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
+    if(coverOptResult) {
+      result = 1;
+      goto _cleanup;
+    }
+
+    /* for cover (with k and d provided) */
+    const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
+    DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
+    if(coverResult) {
+      result = 1;
+      goto _cleanup;
+    }
+
+  }
+
+  /* for fastCover */
+  for (f = 15; f < 25; f++){
+    DISPLAYLEVEL(2, "current f is %u\n", f);
+    for (accel = 1; accel < 11; accel++) {
+      DISPLAYLEVEL(2, "current accel is %u\n", accel);
+      /* for fastCover (optimizing k and d) */
+      ZDICT_fastCover_params_t fastParam;
+      memset(&fastParam, 0, sizeof(fastParam));
+      fastParam.zParams = zParams;
+      fastParam.f = f;
+      fastParam.steps = 40;
+      fastParam.nbThreads = 1;
+      fastParam.accel = accel;
+      const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+      DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+      if(fastOptResult) {
+        result = 1;
+        goto _cleanup;
+      }
+
+      /* for fastCover (with k and d provided) */
+      for (i = 0; i < 5; i++) {
+        const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
+        DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
+        if(fastResult) {
+          result = 1;
+          goto _cleanup;
+        }
+      }
+    }
+  }
+
+
+  /* Free allocated memory */
+_cleanup:
+  UTIL_freeFileList(extendedFileList, fileNamesBuf);
+  freeSampleInfo(srcInfo);
+  return result;
+}

contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h

@@ -0,0 +1,6 @@
+/* ZDICT_trainFromBuffer_legacy() :
+ * issue : samplesBuffer need to be followed by a noisy guard band.
+ * work around : duplicate the buffer, and add the noise */
+size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
+                                    const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                                    ZDICT_legacy_params_t params);

contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh

@@ -0,0 +1,2 @@
+echo "Benchmark with in=../../lib/common"
+./benchmark in=../../../lib/common

contrib/experimental_dict_builders/fastCover/Makefile

@@ -0,0 +1,54 @@
+ARG :=
+
+CC ?= gcc
+CFLAGS ?= -O3 -g
+INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
+
+IO_FILE := ../randomDictBuilder/io.c
+
+TEST_INPUT := ../../../lib
+TEST_OUTPUT := fastCoverDict
+
+all: main run clean
+
+.PHONY: test
+test: main testrun testshell clean
+
+.PHONY: run
+run:
+	echo "Building a fastCover dictionary with given arguments"
+	./main $(ARG)
+
+main: main.o io.o fastCover.o libzstd.a
+	$(CC) $(CFLAGS) main.o io.o fastCover.o libzstd.a -o main
+
+main.o: main.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c main.c
+
+fastCover.o: fastCover.c
+	$(CC) $(CFLAGS) $(INCLUDES) -c fastCover.c
+
+io.o: $(IO_FILE)
+	$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
+
+libzstd.a:
+	$(MAKE) MOREFLAGS=-g -C ../../../lib libzstd.a
+	mv ../../../lib/libzstd.a .
+
+.PHONY: testrun
+testrun: main
+	echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) "
+	./main in=$(TEST_INPUT) out=$(TEST_OUTPUT)
+	zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q
+	rm -f $(TEST_OUTPUT)
+
+.PHONY: testshell
+testshell: test.sh
+	sh test.sh
+	echo "Finish running test.sh"
+
+.PHONY: clean
+clean:
+	rm -f *.o main libzstd.a
+	$(MAKE) -C ../../../lib clean
+	echo "Cleaning is completed"

contrib/experimental_dict_builders/fastCover/README.md

@@ -0,0 +1,24 @@
+FastCover Dictionary Builder
+
+### Permitted Arguments:
+Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
+Output Dictionary (out=dictName): if not provided, default to fastCoverDict
+Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0
+Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB
+Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200
+Size of Dmer (d=#): either 6 or 8; if not provided, default to 8
+Number of steps (steps=#): positive number, if not provided, default to 32
+Percentage of samples used for training(split=#): positive number; if not provided, default to 100
+
+
+###Running Test:
+make test
+
+
+###Usage:
+To build a FASTCOVER dictionary with the provided arguments: make ARG= followed by arguments
+If k or d is not provided, the optimize version of FASTCOVER is run.
+
+### Examples:
+make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520"
+make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"

contrib/experimental_dict_builders/fastCover/fastCover.c

@@ -0,0 +1,809 @@
+/*-*************************************
+*  Dependencies
+***************************************/
+#include <stdio.h>  /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h> /* memset */
+#include <time.h>   /* clock */
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "fastCover.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#include "zdict.h"
+
+
+/*-*************************************
+*  Constants
+***************************************/
+#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+#define FASTCOVER_MAX_F 32
+#define DEFAULT_SPLITPOINT 1.0
+
+/*-*************************************
+*  Console display
+***************************************/
+static int g_displayLevel = 2;
+#define DISPLAY(...)                                                           \
+  {                                                                            \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fflush(stderr);                                                            \
+  }
+#define LOCALDISPLAYLEVEL(displayLevel, l, ...)                                \
+  if (displayLevel >= l) {                                                     \
+    DISPLAY(__VA_ARGS__);                                                      \
+  } /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */
+#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
+
+#define LOCALDISPLAYUPDATE(displayLevel, l, ...)                               \
+  if (displayLevel >= l) {                                                     \
+    if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) {             \
+      g_time = clock();                                                        \
+      DISPLAY(__VA_ARGS__);                                                    \
+    }                                                                          \
+  }
+#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/*-*************************************
+* Hash Functions
+***************************************/
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u  << (64-48)) * prime6bytes) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+
+
+/**
+ * Hash the d-byte value pointed to by p and mod 2^f
+ */
+static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
+  if (d == 6) {
+    return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
+  }
+  return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
+}
+
+
+/*-*************************************
+* Context
+***************************************/
+typedef struct {
+  const BYTE *samples;
+  size_t *offsets;
+  const size_t *samplesSizes;
+  size_t nbSamples;
+  size_t nbTrainSamples;
+  size_t nbTestSamples;
+  size_t nbDmers;
+  U32 *freqs;
+  U16 *segmentFreqs;
+  unsigned d;
+} FASTCOVER_ctx_t;
+
+
+/*-*************************************
+*  Helper functions
+***************************************/
+/**
+ * Returns the sum of the sample sizes.
+ */
+static size_t FASTCOVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
+  size_t sum = 0;
+  unsigned i;
+  for (i = 0; i < nbSamples; ++i) {
+    sum += samplesSizes[i];
+  }
+  return sum;
+}
+
+
+/*-*************************************
+*  fast functions
+***************************************/
+/**
+ * A segment is a range in the source as well as the score of the segment.
+ */
+typedef struct {
+  U32 begin;
+  U32 end;
+  U32 score;
+} FASTCOVER_segment_t;
+
+
+/**
+ * Selects the best segment in an epoch.
+ * Segments of are scored according to the function:
+ *
+ * Let F(d) be the frequency of all dmers with hash value d.
+ * Let S_i be hash value of the dmer at position i of segment S which has length k.
+ *
+ *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
+ *
+ * Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2.
+ */
+static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
+                                                  U32 *freqs, U32 begin,U32 end,
+                                                  ZDICT_fastCover_params_t parameters) {
+  /* Constants */
+  const U32 k = parameters.k;
+  const U32 d = parameters.d;
+  const U32 dmersInK = k - d + 1;
+  /* Try each segment (activeSegment) and save the best (bestSegment) */
+  FASTCOVER_segment_t bestSegment = {0, 0, 0};
+  FASTCOVER_segment_t activeSegment;
+  /* Reset the activeDmers in the segment */
+  /* The activeSegment starts at the beginning of the epoch. */
+  activeSegment.begin = begin;
+  activeSegment.end = begin;
+  activeSegment.score = 0;
+  {
+    /* Slide the activeSegment through the whole epoch.
+     * Save the best segment in bestSegment.
+     */
+    while (activeSegment.end < end) {
+      /* Get hash value of current dmer */
+      const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
+      /* Add frequency of this index to score if this is the first occurence of index in active segment */
+      if (ctx->segmentFreqs[index] == 0) {
+        activeSegment.score += freqs[index];
+      }
+      ctx->segmentFreqs[index] += 1;
+      /* Increment end of segment */
+      activeSegment.end += 1;
+      /* If the window is now too large, drop the first position */
+      if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
+        /* Get hash value of the dmer to be eliminated from active segment */
+        const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d);
+        ctx->segmentFreqs[delIndex] -= 1;
+        /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
+        if (ctx->segmentFreqs[delIndex] == 0) {
+          activeSegment.score -= freqs[delIndex];
+        }
+        /* Increment start of segment */
+        activeSegment.begin += 1;
+      }
+      /* If this segment is the best so far save it */
+      if (activeSegment.score > bestSegment.score) {
+        bestSegment = activeSegment;
+      }
+    }
+    /* Zero out rest of segmentFreqs array */
+    while (activeSegment.begin < end) {
+      const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d);
+      ctx->segmentFreqs[delIndex] -= 1;
+      activeSegment.begin += 1;
+    }
+  }
+  {
+    /* Trim off the zero frequency head and tail from the segment. */
+    U32 newBegin = bestSegment.end;
+    U32 newEnd = bestSegment.begin;
+    U32 pos;
+    for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+      const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
+      U32 freq = freqs[index];
+      if (freq != 0) {
+        newBegin = MIN(newBegin, pos);
+        newEnd = pos + 1;
+      }
+    }
+    bestSegment.begin = newBegin;
+    bestSegment.end = newEnd;
+  }
+  {
+    /*  Zero the frequency of hash value of each dmer covered by the chosen segment. */
+    U32 pos;
+    for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+      const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
+      freqs[i] = 0;
+    }
+  }
+  return bestSegment;
+}
+
+/**
+ * Check the validity of the parameters.
+ * Returns non-zero if the parameters are valid and 0 otherwise.
+ */
+static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters,
+                                 size_t maxDictSize) {
+  /* k, d, and f are required parameters */
+  if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) {
+    return 0;
+  }
+  /* d has to be 6 or 8 */
+  if (parameters.d != 6 && parameters.d != 8) {
+    return 0;
+  }
+  /* 0 < f <= FASTCOVER_MAX_F */
+  if (parameters.f > FASTCOVER_MAX_F) {
+    return 0;
+  }
+  /* k <= maxDictSize */
+  if (parameters.k > maxDictSize) {
+    return 0;
+  }
+  /* d <= k */
+  if (parameters.d > parameters.k) {
+    return 0;
+  }
+  /* 0 < splitPoint <= 1 */
+  if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) {
+    return 0;
+  }
+  return 1;
+}
+
+
+/**
+ * Clean up a context initialized with `FASTCOVER_ctx_init()`.
+ */
+static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
+  if (!ctx) {
+    return;
+  }
+  if (ctx->segmentFreqs) {
+    free(ctx->segmentFreqs);
+    ctx->segmentFreqs = NULL;
+  }
+  if (ctx->freqs) {
+    free(ctx->freqs);
+    ctx->freqs = NULL;
+  }
+  if (ctx->offsets) {
+    free(ctx->offsets);
+    ctx->offsets = NULL;
+  }
+}
+
+/**
+ * Calculate for frequency of hash value of each dmer in ctx->samples
+ */
+static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){
+  size_t start; /* start of current dmer */
+  for (unsigned i = 0; i < ctx->nbTrainSamples; i++) {
+    size_t currSampleStart = ctx->offsets[i];
+    size_t currSampleEnd = ctx->offsets[i+1];
+    start = currSampleStart;
+    while (start + ctx->d <= currSampleEnd) {
+      const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d);
+      freqs[dmerIndex]++;
+      start++;
+    }
+  }
+}
+
+/**
+ * Prepare a context for dictionary building.
+ * The context is only dependent on the parameter `d` and can used multiple
+ * times.
+ * Returns 1 on success or zero on error.
+ * The context must be destroyed with `FASTCOVER_ctx_destroy()`.
+ */
+static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
+                          const size_t *samplesSizes, unsigned nbSamples,
+                          unsigned d, double splitPoint, unsigned f) {
+  const BYTE *const samples = (const BYTE *)samplesBuffer;
+  const size_t totalSamplesSize = FASTCOVER_sum(samplesSizes, nbSamples);
+  /* Split samples into testing and training sets */
+  const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
+  const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
+  const size_t trainingSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
+  const size_t testSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
+  /* Checks */
+  if (totalSamplesSize < MAX(d, sizeof(U64)) ||
+      totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
+    DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
+                 (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
+    return 0;
+  }
+  /* Check if there are at least 5 training samples */
+  if (nbTrainSamples < 5) {
+    DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
+    return 0;
+  }
+  /* Check if there's testing sample */
+  if (nbTestSamples < 1) {
+    DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
+    return 0;
+  }
+  /* Zero the context */
+  memset(ctx, 0, sizeof(*ctx));
+  DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
+               (U32)trainingSamplesSize);
+  DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
+               (U32)testSamplesSize);
+
+  ctx->samples = samples;
+  ctx->samplesSizes = samplesSizes;
+  ctx->nbSamples = nbSamples;
+  ctx->nbTrainSamples = nbTrainSamples;
+  ctx->nbTestSamples = nbTestSamples;
+  ctx->nbDmers = trainingSamplesSize - d + 1;
+  ctx->d = d;
+
+  /* The offsets of each file */
+  ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
+  if (!ctx->offsets) {
+    DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
+    FASTCOVER_ctx_destroy(ctx);
+    return 0;
+  }
+
+  /* Fill offsets from the samplesSizes */
+  {
+    U32 i;
+    ctx->offsets[0] = 0;
+    for (i = 1; i <= nbSamples; ++i) {
+      ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+    }
+  }
+
+  /* Initialize frequency array of size 2^f */
+  ctx->freqs = (U32 *)calloc((1 << f), sizeof(U32));
+  ctx->segmentFreqs = (U16 *)calloc((1 << f), sizeof(U16));
+  DISPLAYLEVEL(2, "Computing frequencies\n");
+  FASTCOVER_computeFrequency(ctx->freqs, f, ctx);
+
+  return 1;
+}
+
+
+/**
+ * Given the prepared context build the dictionary.
+ */
+static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
+                                    void *dictBuffer,
+                                    size_t dictBufferCapacity,
+                                    ZDICT_fastCover_params_t parameters){
+  BYTE *const dict = (BYTE *)dictBuffer;
+  size_t tail = dictBufferCapacity;
+  /* Divide the data up into epochs of equal size.
+   * We will select at least one segment from each epoch.
+   */
+  const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
+  const U32 epochSize = (U32)(ctx->nbDmers / epochs);
+  size_t epoch;
+  DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
+               epochSize);
+  /* Loop through the epochs until there are no more segments or the dictionary
+   * is full.
+   */
+  for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
+    const U32 epochBegin = (U32)(epoch * epochSize);
+    const U32 epochEnd = epochBegin + epochSize;
+    size_t segmentSize;
+    /* Select a segment */
+    FASTCOVER_segment_t segment = FASTCOVER_selectSegment(
+        ctx, freqs, epochBegin, epochEnd, parameters);
+
+    /* If the segment covers no dmers, then we are out of content */
+    if (segment.score == 0) {
+      break;
+    }
+
+    /* Trim the segment if necessary and if it is too small then we are done */
+    segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
+    if (segmentSize < parameters.d) {
+      break;
+    }
+
+    /* We fill the dictionary from the back to allow the best segments to be
+     * referenced with the smallest offsets.
+     */
+    tail -= segmentSize;
+    memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
+    DISPLAYUPDATE(
+        2, "\r%u%%       ",
+        (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
+  }
+  DISPLAYLEVEL(2, "\r%79s\r", "");
+  return tail;
+}
+
+
+/**
+ * FASTCOVER_best_t is used for two purposes:
+ * 1. Synchronizing threads.
+ * 2. Saving the best parameters and dictionary.
+ *
+ * All of the methods except FASTCOVER_best_init() are thread safe if zstd is
+ * compiled with multithreaded support.
+ */
+typedef struct fast_best_s {
+  ZSTD_pthread_mutex_t mutex;
+  ZSTD_pthread_cond_t cond;
+  size_t liveJobs;
+  void *dict;
+  size_t dictSize;
+  ZDICT_fastCover_params_t parameters;
+  size_t compressedSize;
+} FASTCOVER_best_t;
+
+/**
+ * Initialize the `FASTCOVER_best_t`.
+ */
+static void FASTCOVER_best_init(FASTCOVER_best_t *best) {
+  if (best==NULL) return; /* compatible with init on NULL */
+  (void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
+  (void)ZSTD_pthread_cond_init(&best->cond, NULL);
+  best->liveJobs = 0;
+  best->dict = NULL;
+  best->dictSize = 0;
+  best->compressedSize = (size_t)-1;
+  memset(&best->parameters, 0, sizeof(best->parameters));
+}
+
+/**
+ * Wait until liveJobs == 0.
+ */
+static void FASTCOVER_best_wait(FASTCOVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  ZSTD_pthread_mutex_lock(&best->mutex);
+  while (best->liveJobs != 0) {
+    ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
+  }
+  ZSTD_pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Call FASTCOVER_best_wait() and then destroy the FASTCOVER_best_t.
+ */
+static void FASTCOVER_best_destroy(FASTCOVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  FASTCOVER_best_wait(best);
+  if (best->dict) {
+    free(best->dict);
+  }
+  ZSTD_pthread_mutex_destroy(&best->mutex);
+  ZSTD_pthread_cond_destroy(&best->cond);
+}
+
+/**
+ * Called when a thread is about to be launched.
+ * Increments liveJobs.
+ */
+static void FASTCOVER_best_start(FASTCOVER_best_t *best) {
+  if (!best) {
+    return;
+  }
+  ZSTD_pthread_mutex_lock(&best->mutex);
+  ++best->liveJobs;
+  ZSTD_pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Called when a thread finishes executing, both on error or success.
+ * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
+ * If this dictionary is the best so far save it and its parameters.
+ */
+static void FASTCOVER_best_finish(FASTCOVER_best_t *best, size_t compressedSize,
+                              ZDICT_fastCover_params_t parameters, void *dict,
+                              size_t dictSize) {
+  if (!best) {
+    return;
+  }
+  {
+    size_t liveJobs;
+    ZSTD_pthread_mutex_lock(&best->mutex);
+    --best->liveJobs;
+    liveJobs = best->liveJobs;
+    /* If the new dictionary is better */
+    if (compressedSize < best->compressedSize) {
+      /* Allocate space if necessary */
+      if (!best->dict || best->dictSize < dictSize) {
+        if (best->dict) {
+          free(best->dict);
+        }
+        best->dict = malloc(dictSize);
+        if (!best->dict) {
+          best->compressedSize = ERROR(GENERIC);
+          best->dictSize = 0;
+          return;
+        }
+      }
+      /* Save the dictionary, parameters, and size */
+      memcpy(best->dict, dict, dictSize);
+      best->dictSize = dictSize;
+      best->parameters = parameters;
+      best->compressedSize = compressedSize;
+    }
+    ZSTD_pthread_mutex_unlock(&best->mutex);
+    if (liveJobs == 0) {
+      ZSTD_pthread_cond_broadcast(&best->cond);
+    }
+  }
+}
+
+/**
+ * Parameters for FASTCOVER_tryParameters().
+ */
+typedef struct FASTCOVER_tryParameters_data_s {
+  const FASTCOVER_ctx_t *ctx;
+  FASTCOVER_best_t *best;
+  size_t dictBufferCapacity;
+  ZDICT_fastCover_params_t parameters;
+} FASTCOVER_tryParameters_data_t;
+
+/**
+ * Tries a set of parameters and updates the FASTCOVER_best_t with the results.
+ * This function is thread safe if zstd is compiled with multithreaded support.
+ * It takes its parameters as an *OWNING* opaque pointer to support threading.
+ */
+static void FASTCOVER_tryParameters(void *opaque) {
+  /* Save parameters as local variables */
+  FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
+  const FASTCOVER_ctx_t *const ctx = data->ctx;
+  const ZDICT_fastCover_params_t parameters = data->parameters;
+  size_t dictBufferCapacity = data->dictBufferCapacity;
+  size_t totalCompressedSize = ERROR(GENERIC);
+  /* Allocate space for hash table, dict, and freqs */
+  BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+  U32 *freqs = (U32*) malloc((1 << parameters.f) * sizeof(U32));
+  if (!dict || !freqs) {
+    DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
+    goto _cleanup;
+  }
+  /* Copy the frequencies because we need to modify them */
+  memcpy(freqs, ctx->freqs, (1 << parameters.f) * sizeof(U32));
+  /* Build the dictionary */
+  {
+    const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict,
+                                              dictBufferCapacity, parameters);
+
+    dictBufferCapacity = ZDICT_finalizeDictionary(
+        dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+        ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
+        parameters.zParams);
+    if (ZDICT_isError(dictBufferCapacity)) {
+      DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+      goto _cleanup;
+    }
+  }
+  /* Check total compressed size */
+  {
+    /* Pointers */
+    ZSTD_CCtx *cctx;
+    ZSTD_CDict *cdict;
+    void *dst;
+    /* Local variables */
+    size_t dstCapacity;
+    size_t i;
+    /* Allocate dst with enough space to compress the maximum sized sample */
+    {
+      size_t maxSampleSize = 0;
+      i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
+      for (; i < ctx->nbSamples; ++i) {
+        maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
+      }
+      dstCapacity = ZSTD_compressBound(maxSampleSize);
+      dst = malloc(dstCapacity);
+    }
+    /* Create the cctx and cdict */
+    cctx = ZSTD_createCCtx();
+    cdict = ZSTD_createCDict(dict, dictBufferCapacity,
+                             parameters.zParams.compressionLevel);
+    if (!dst || !cctx || !cdict) {
+      goto _compressCleanup;
+    }
+    /* Compress each sample and sum their sizes (or error) */
+    totalCompressedSize = dictBufferCapacity;
+    i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
+    for (; i < ctx->nbSamples; ++i) {
+      const size_t size = ZSTD_compress_usingCDict(
+          cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
+          ctx->samplesSizes[i], cdict);
+      if (ZSTD_isError(size)) {
+        totalCompressedSize = ERROR(GENERIC);
+        goto _compressCleanup;
+      }
+      totalCompressedSize += size;
+    }
+  _compressCleanup:
+    ZSTD_freeCCtx(cctx);
+    ZSTD_freeCDict(cdict);
+    if (dst) {
+      free(dst);
+    }
+  }
+
+_cleanup:
+  FASTCOVER_best_finish(data->best, totalCompressedSize, parameters, dict,
+                    dictBufferCapacity);
+  free(data);
+  if (dict) {
+    free(dict);
+  }
+  if (freqs) {
+    free(freqs);
+  }
+}
+
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) {
+    BYTE* const dict = (BYTE*)dictBuffer;
+    FASTCOVER_ctx_t ctx;
+    parameters.splitPoint = 1.0;
+    /* Initialize global data */
+    g_displayLevel = parameters.zParams.notificationLevel;
+    /* Checks */
+    if (!FASTCOVER_checkParameters(parameters, dictBufferCapacity)) {
+      DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
+      return ERROR(GENERIC);
+    }
+    if (nbSamples == 0) {
+      DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
+      return ERROR(GENERIC);
+    }
+    if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+      DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+                   ZDICT_DICTSIZE_MIN);
+      return ERROR(dstSize_tooSmall);
+    }
+    /* Initialize context */
+    if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+                            parameters.d, parameters.splitPoint, parameters.f)) {
+      DISPLAYLEVEL(1, "Failed to initialize context\n");
+      return ERROR(GENERIC);
+    }
+    /* Build the dictionary */
+    DISPLAYLEVEL(2, "Building dictionary\n");
+    {
+      const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer,
+                                                dictBufferCapacity, parameters);
+
+      const size_t dictionarySize = ZDICT_finalizeDictionary(
+          dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+          samplesBuffer, samplesSizes, (unsigned)ctx.nbTrainSamples,
+          parameters.zParams);
+      if (!ZSTD_isError(dictionarySize)) {
+          DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
+                      (U32)dictionarySize);
+      }
+      FASTCOVER_ctx_destroy(&ctx);
+      return dictionarySize;
+    }
+}
+
+
+
+ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
+    void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+    const size_t *samplesSizes, unsigned nbSamples,
+    ZDICT_fastCover_params_t *parameters) {
+    /* constants */
+    const unsigned nbThreads = parameters->nbThreads;
+    const double splitPoint =
+        parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
+    const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
+    const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
+    const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
+    const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
+    const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
+    const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
+    const unsigned kIterations =
+        (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+    const unsigned f = parameters->f == 0 ? 23 : parameters->f;
+
+    /* Local variables */
+    const int displayLevel = parameters->zParams.notificationLevel;
+    unsigned iteration = 1;
+    unsigned d;
+    unsigned k;
+    FASTCOVER_best_t best;
+    POOL_ctx *pool = NULL;
+
+    /* Checks */
+    if (splitPoint <= 0 || splitPoint > 1) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
+      return ERROR(GENERIC);
+    }
+    if (kMinK < kMaxD || kMaxK < kMinK) {
+      LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
+      return ERROR(GENERIC);
+    }
+    if (nbSamples == 0) {
+      DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
+      return ERROR(GENERIC);
+    }
+    if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+      DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+                   ZDICT_DICTSIZE_MIN);
+      return ERROR(dstSize_tooSmall);
+    }
+    if (nbThreads > 1) {
+      pool = POOL_create(nbThreads, 1);
+      if (!pool) {
+        return ERROR(memory_allocation);
+      }
+    }
+    /* Initialization */
+    FASTCOVER_best_init(&best);
+    /* Turn down global display level to clean up display at level 2 and below */
+    g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
+    /* Loop through d first because each new value needs a new context */
+    LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+                      kIterations);
+    for (d = kMinD; d <= kMaxD; d += 2) {
+      /* Initialize the context for this value of d */
+      FASTCOVER_ctx_t ctx;
+      LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+      if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f)) {
+        LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+        FASTCOVER_best_destroy(&best);
+        POOL_free(pool);
+        return ERROR(GENERIC);
+      }
+      /* Loop through k reusing the same context */
+      for (k = kMinK; k <= kMaxK; k += kStepSize) {
+        /* Prepare the arguments */
+        FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
+            sizeof(FASTCOVER_tryParameters_data_t));
+        LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+        if (!data) {
+          LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+          FASTCOVER_best_destroy(&best);
+          FASTCOVER_ctx_destroy(&ctx);
+          POOL_free(pool);
+          return ERROR(GENERIC);
+        }
+        data->ctx = &ctx;
+        data->best = &best;
+        data->dictBufferCapacity = dictBufferCapacity;
+        data->parameters = *parameters;
+        data->parameters.k = k;
+        data->parameters.d = d;
+        data->parameters.f = f;
+        data->parameters.splitPoint = splitPoint;
+        data->parameters.steps = kSteps;
+        data->parameters.zParams.notificationLevel = g_displayLevel;
+        /* Check the parameters */
+        if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity)) {
+          DISPLAYLEVEL(1, "fastCover parameters incorrect\n");
+          free(data);
+          continue;
+        }
+        /* Call the function and pass ownership of data to it */
+        FASTCOVER_best_start(&best);
+        if (pool) {
+          POOL_add(pool, &FASTCOVER_tryParameters, data);
+        } else {
+          FASTCOVER_tryParameters(data);
+        }
+        /* Print status */
+        LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%%       ",
+                           (U32)((iteration * 100) / kIterations));
+        ++iteration;
+      }
+      FASTCOVER_best_wait(&best);
+      FASTCOVER_ctx_destroy(&ctx);
+    }
+    LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+    /* Fill the output buffer and parameters with output of the best parameters */
+    {
+      const size_t dictSize = best.dictSize;
+      if (ZSTD_isError(best.compressedSize)) {
+        const size_t compressedSize = best.compressedSize;
+        FASTCOVER_best_destroy(&best);
+        POOL_free(pool);
+        return compressedSize;
+      }
+      *parameters = best.parameters;
+      memcpy(dictBuffer, best.dict, dictSize);
+      FASTCOVER_best_destroy(&best);
+      POOL_free(pool);
+      return dictSize;
+    }
+
+}