v1.4.3: Merge pull request #1730 from facebook/dev

Fix Buffer Overflow in Legacy (v0.3) Raw Literals Decompression

.circleci/config.yml

@@ -1,32 +1,14 @@
 version: 2
 
-references:
-  # Install the dependencies required for tests.
-  # Add the step "- *install-dependencies" to the beginning of your job to run
-  # this command.
-  install-dependencies: &install-dependencies
-    run:
-      name: Install dependencies
-      # TODO: We can split these dependencies up by job to reduce installation
-      # time.
-      command: |
-        sudo dpkg --add-architecture i386
-        sudo apt-get -y -qq update
-        sudo apt-get -y install \
-            gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
-            libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
-            libc6-dev-ppc64-powerpc-cross
-
 jobs:
   # the first half of the jobs are in this test
   short-tests-0:
     # TODO: Create a small custom docker image with all the dependencies we need
     #       preinstalled to reduce installation time.
     docker:
-      - image: circleci/buildpack-deps:bionic
+      - image: fbopensource/zstd-circleci-primary:0.0.1
     steps:
       - checkout
-      - *install-dependencies
       - run:
           name: Test
           command: |
@@ -40,10 +22,9 @@ jobs:
   # the second half of the jobs are in this test
   short-tests-1:
     docker:
-      - image: circleci/buildpack-deps:bionic
+      - image: fbopensource/zstd-circleci-primary:0.0.1
     steps:
       - checkout
-      - *install-dependencies
       - run:
           name: Test
           command: |
@@ -60,15 +41,11 @@ jobs:
   # tagged release.
   publish-github-release:
     docker:
-      - image: cibuilds/github:0.12.0
+      - image: fbopensource/zstd-circleci-primary:0.0.1
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
     steps:
       - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            apk add -q gzip coreutils
       - run:
           name: Publish
           command: |
@@ -76,12 +53,57 @@ jobs:
             export ZSTD_VERSION=zstd-$VERSION
             git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
                         -o $ZSTD_VERSION.tar
-            gzip -9 $ZSTD_VERSION.tar
-            sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256sum
+            sha256sum $ZSTD_VERSION.tar > $ZSTD_VERSION.tar.sha256
+            zstd -19 $ZSTD_VERSION.tar
+            sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
+            gzip -k -9 $ZSTD_VERSION.tar
+            sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
             mkdir -p $CIRCLE_ARTIFACTS
-            cp $ZSTD_VERSION.tar.gz{,.sha256sum} $CIRCLE_ARTIFACTS
+            cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS
       - store_artifacts:
           path: /tmp/circleci-artifacts
+  # This step should only be run in a cron job
+  regression-test:
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    steps:
+      - checkout
+      # Restore the cached resources.
+      - restore_cache:
+          # We try our best to bust the cache when the data changes by hashing
+          # data.c. If that doesn't work, simply update the version number here
+          # and below. If we fail to bust the cache, the regression testing will
+          # still work, since it has its own stamp, but will need to redownload
+          # everything.
+          keys:
+            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+      - run:
+          name: Regression Test
+          command: |
+            make -C programs zstd
+            make -C tests/regression test
+            mkdir -p $CIRCLE_ARTIFACTS
+            ./tests/regression/test                     \
+                --cache  tests/regression/cache         \
+                --output $CIRCLE_ARTIFACTS/results.csv  \
+                --zstd   programs/zstd
+            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
+            echo "      If this fails, go to the Artifacts pane in CircleCI, "
+            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
+            echo "      are still good, copy it into the repo and commit it."
+            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
+            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+      # Only save the cache on success (default), since if the failure happened
+      # before we stamp the data cache, we will have a bad cache for this key.
+      - save_cache:
+          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+          paths:
+            - tests/regression/cache
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+
 
 workflows:
   version: 2
@@ -96,6 +118,13 @@ workflows:
           filters:
             tags:
               only: /.*/
+      # Create a branch called regression and set it to dev to force a
+      # regression test run
+      - regression-test:
+          filters:
+            branches:
+              only:
+                - regression
       # Only run on release tags.
       - publish-github-release:
           requires:
@@ -106,6 +135,20 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v\d+\.\d+\.\d+$/
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+                - dev
+    jobs:
+      # Run daily long regression tests
+      - regression-test
+
+
 
   # Longer tests
     #- make -C tests test-zstd-nolegacy && make clean

.circleci/images/primary/Dockerfile

@@ -0,0 +1,9 @@
+FROM circleci/buildpack-deps:bionic
+
+RUN sudo dpkg --add-architecture i386
+RUN sudo apt-get -y -qq update
+RUN sudo apt-get -y install \
+    gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
+    libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+    libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
+    libcurl4-openssl-dev

.cirrus.yml

@@ -0,0 +1,16 @@
+env:
+  CIRRUS_CLONE_DEPTH: 1
+  ARCH: amd64
+
+task:
+  freebsd_instance:
+    matrix:
+      image: freebsd-12-0-release-amd64
+      image: freebsd-11-2-release-amd64
+  install_script:
+    - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf
+    - pkg upgrade -y
+    - pkg install -y gmake coreutils
+  script: |
+    MOREFLAGS="-Werror" gmake -j all
+    gmake shortest

.gitattributes

@@ -19,6 +19,3 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
-
-# .travis.yml merging
-.travis.yml merge=ours

.gitignore

@@ -14,7 +14,7 @@
 *.dylib
 
 # Executables
-zstd
+/zstd
 zstdmt
 *.exe
 *.out
@@ -22,7 +22,7 @@ zstdmt
 
 # Test artefacts
 tmp*
-dictionary*
+dictionary.
 NUL
 
 # Build artefacts
@@ -41,3 +41,4 @@ _zstdbench/
 .DS_Store
 googletest/
 *.d
+*.vscode

.travis.yml

@@ -2,58 +2,6 @@
 
 language: c
 dist: trusty
-sudo: required
-addons:
-  apt:
-    update: true
-
-matrix:
-  include:
-    # Ubuntu 14.04
-    - env: Cmd='make test'
-
-    - env: Cmd='make gcc6install && CC=gcc-6 CFLAGS=-Werror make -j all
-             && make clean && CC=gcc-6 make -j uasan-test-zstd </dev/null'   # test when stdin is not a tty
-    - env: Cmd='make gcc6install libc6install
-             && make clean && CC=gcc-6 make -j uasan-test-zstd32'
-    - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd'
-    - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
-
-    - env: Cmd='make staticAnalyze'
-
-    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
-    - env: Cmd='make gcc6install libc6install
-             && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
-
-    - env: Cmd='make -j uasanregressiontest
-             && make clean && make -j msanregressiontest'
-
-    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
-             && make clean && make -C tests test-fuzzer-stackmode'
-
-    - env: Cmd='make arminstall && make armfuzz'
-    - env: Cmd='make arminstall && make aarch64fuzz'
-    - env: Cmd='make ppcinstall && make ppcfuzz'
-    - env: Cmd='make ppcinstall && make ppc64fuzz'
-
-    - env: Cmd='make lz4install && make -C tests test-lz4
-             && make clean && make -C tests test-pool
-             && make clean && bash tests/libzstd_partial_builds.sh'
-
-    # tag-specific test
-    - if: tag =~ ^v[0-9]\.[0-9]
-      env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'
-
-    # additional tests for master branch and cron job
-    - env: Cmd="make test"
-      os: osx
-    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
-    - env: Cmd='make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install'
-    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
-
 
 git:
   depth: 1
@@ -64,10 +12,212 @@ branches:
   - master
   - travisTest
 
-script:
-  - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-  - export FUZZERTEST=-T2mn;
-    export ZSTREAM_TESTTIME=-T2mn;
-    export DECODECORPUS_TESTTIME=-T1mn;
-    sh -c "$Cmd" || travis_terminate 1;
+addons:
+  apt:
+    update: true
+
+env:
+  global:
+    - FUZZERTEST=-T2mn
+      ZSTREAM_TESTTIME=-T2mn
+      DECODECORPUS_TESTTIME=-T1mn
+
+
+matrix:
+  fast_finish: true
+  include:
+    - name: Trusty (Test All)
+      script:
+        - make test
+
+    - name: gcc-6 + gcc-7 compilation
+      script:
+        - make gcc6install gcc7install
+        - CC=gcc-6 CFLAGS=-Werror make -j all
+        - make clean
+        - CC=gcc-7 CFLAGS=-Werror make -j all
+
+    - name: gcc-8 + ASan + UBSan + Test Zstd
+      script:
+        - make gcc8install
+        - CC=gcc-8 CFLAGS="-Werror" make -j all
+        - make clean
+        - CC=gcc-8 make -j uasan-test-zstd </dev/null   # test when stdin is not a tty
+
+    - name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
+      script:
+        - make gcc6install libc6install
+        - CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
+        - make clean
+        - CC=gcc-6 make -j uasan-test-zstd32   # note : can complain about pointer overflow
+
+    - name: Trusty (clang-3.8 + MSan + Test Zstd)
+      script:
+        - make clang38install
+        - CC=clang-3.8 make clean msan-test-zstd
+
+    - name: Trusty (Minimal Decompressor Macros)
+      script:
+        - make clean
+        - CFLAGS=-Werror make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        - make clean
+        - make -j check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
+        - make clean
+        - CFLAGS=-Werror make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        - make clean
+        - make -j check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
+        - make clean
+        - CFLAGS=-Werror make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+        - make clean
+        - make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
+
+    - name: Trusty (CMake)
+      script:
+        - make cmakebuild
+
+    - name: Trusty (Static Analyze)
+      script:
+        - make staticAnalyze
+
+    - name: Trusty (gcc-8 + ASan + UBSan + Fuzz Test)
+      script:
+        - make gcc8install
+        - CC=gcc-8 make clean uasan-fuzztest
+
+    - name: Trusty (gcc-6 + ASan + UBSan + Fuzz Test 32bit)
+      script:
+        - make gcc6install libc6install
+        - CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest   # can complain about pointer overflow
+
+    - name: Trusty (clang-3.8 + MSan + Fuzz Test)
+      script:
+        - make clang38install
+        - CC=clang-3.8 make clean msan-fuzztest
+
+    - name: Trusty (ASan + UBSan + MSan + Regression Test)
+      script:
+        - make -j uasanregressiontest
+        - make clean
+        - make -j msanregressiontest
+
+    - name: Trusty (Valgrind + Fuzz Test Stack Mode)
+      script:
+        - make valgrindinstall
+        - make -C tests clean valgrindTest
+        - make clean
+        - make -C tests test-fuzzer-stackmode
+
+    - name: Trusty (ARM + Fuzz Test)
+      script:
+        - make arminstall
+        - make armfuzz
+
+    - name: Xenial (AARCH64 + Fuzz Test)
+      dist: xenial
+      script:
+        - make arminstall
+        - make aarch64fuzz
+
+    - name: Trusty (PPC + Fuzz Test)
+      script:
+        - make ppcinstall
+        - make ppcfuzz
+
+    - name: Trusty (Versions Compatibility Test)
+      script:
+        - make -C tests versionsTest
+
+    # check release number
+    - name: Tag-Specific Test
+      if: tag =~ ^v[0-9]\.[0-9]
+      script:
+        - make -C tests checkTag
+        - tests/checkTag "$TRAVIS_BRANCH"
+
+    # tests for master branch and cron job only
+    - name: OS-X
+      if: branch = master
+      os: osx
+      script:
+        - make test
+
+    - name: thread sanitizer
+      if: branch = master
+      script:
+        - make clang38install
+        - CC=clang-3.8 make tsan-test-zstream
+        - CC=clang-3.8 make tsan-fuzztest
+
+    - name: C++ and gnu90 compatibility
+      if: branch = master
+      script:
+        - make cxxtest
+        - make clean
+        - make gnu90build
+        - make clean
+        - make travis-install    # just ensures `make install` works
+
+    - name: c99 compatibility
+      if: branch = master
+      script:
+        - make c99build
+        - make -C tests test-zstd
+
+    - name: PPC64
+      if: branch = master
+      script:
+        - make ppcinstall
+        - make ppc64fuzz
+
+    - name: zlib wrapper test
+      if: branch = master
+      script:
+        - make gpp6install valgrindinstall
+        - make -C zlibWrapper test
+        - make -C zlibWrapper valgrindTest
+
+    - name: LZ4, thread pool, and partial libs tests
+      if: branch = master
+      script:
+        - make lz4install
+        - make -C tests test-lz4
+        - make clean
+        - make -C tests test-pool
+        - make clean
+        - bash tests/libzstd_partial_builds.sh
+
+    # meson dedicated test
+    - name: Xenial (Meson + clang)
+      # env: ALLOW_FAILURES=true
+      dist: xenial
+      language: cpp
+      compiler: clang
+      install:
+        - sudo apt-get install -qq liblz4-dev valgrind tree
+        - |
+          travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
+          unzip ~/ninja.zip -d ~/.local/bin
+        - |
+          travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
+          python3 ~/get-pip.py --user &&
+          pip3 install --user meson
+      script:
+        - |
+          meson setup \
+            --buildtype=debugoptimized \
+            -Db_lundef=false \
+            -Dauto_features=enabled \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Dbin_contrib=true \
+            -Ddefault_library=both \
+             build/meson builddir
+        - pushd builddir
+        - ninja
+        - meson test --verbose --no-rebuild
+        - DESTDIR=./staging ninja install
+        - tree ./staging
+      after_failure:
+        - cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
+  allow_failures:
+    - env: ALLOW_FAILURES=true

CHANGELOG

@@ -1,3 +1,104 @@
+v1.4.3
+bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
+bug: Fix Buffer Overflow in v0.3 Decompression by @felixhandte (#1722)
+build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
+misc: Add NULL pointer check in util.c by @leeyoung624 (#1706)
+
+v1.4.2
+bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
+bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
+misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
+misc: Restructure source files by @ephiepark (#1679)
+
+v1.4.1
+bug: Fix data corruption in niche use cases by @terrelln (#1659)
+bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
+bug: Fix out of bounds read by @terrelln (#1590)
+perf: Improve decode speed by ~7% @mgrice (#1668)
+perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
+perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
+perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
+perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
+api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
+cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
+cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
+cli: Restrict read permissions on destination files by @chungy (#1644)
+cli: zstdgrep: handle -f flag by @felixhandte (#1618)
+cli: zstdcat: follow symlinks by @vejnar (#1604)
+doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
+doc: Fix typo by @yk-tanigawa (#1633)
+doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
+build: CMake: support building with LZ4 @leeyoung624 (#1626)
+build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
+build: CMake: respect existing uninstall target by @j301scott (#1619)
+build: Make: skip multithread tests when built without support by @michaelforney (#1620)
+build: Make: Fix examples/ test target by @sjnam (#1603)
+build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
+build: Meson: fix build by @lzutao (#1602)
+build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
+build: Visual Studio: fix linking by @absotively (#1639)
+build: Fix MinGW-W64 build by @myzhang1029 (#1600)
+misc: Expand decodecorpus coverage by @ephiepark (#1664)
+
+v1.4.0
+perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
+api: Move the advanced API, including all functions in the staging section, to the stable section
+api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
+api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+api: Don't export ZSTDMT functions from the shared library by default
+api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
+api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
+api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
+api: Move ZSTD_getDictID_*() functions to the stable section
+api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
+api: Allow compression parameters to be set when a dictionary is used
+api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
+api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
+api: Setting ZSTD_d_maxWindowLog to 0 means use the default
+cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
+cli: Add --[no-]compress-literals flag to enable or disable literal compression
+doc: Update the examples to use the advanced API
+doc: Explain how to transition from old streaming functions to the advanced API in the header
+build: Improve the Windows release packages
+build: Improve CMake build by @hjmjohnson
+build: Build fixes for FreeBSD by @lwhsu
+build: Remove redundant warnings by @thatsafunnyname
+build: Fix tests on OpenBSD by @bket
+build: Extend fuzzer build system to work with the new clang engine
+build: CMake now creates the libzstd.so.1 symlink
+build: Improve Menson build by @lzutao
+misc: Fix symbolic link detection on FreeBSD
+misc: Use physical core count for -T0 on FreeBSD by @cemeyer
+misc: Fix zstd --list on truncated files by @kostmo
+misc: Improve logging in debug mode by @felixhandte
+misc: Add CirrusCI tests by @lwhsu
+misc: Optimize dictionary memory usage in corner cases
+misc: Improve the dictionary builder on small or homogeneous data
+misc: Fix spelling across the repo by @jsoref
+
+v1.3.8
+perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
+perf: slightly better compression ratio at high compression modes
+api : finalized advanced API, last stage before "stable" status
+api : new --rsyncable mode, by @terrelln
+api : support decompression of empty frames into NULL (used to be an error) (#1385)
+build: new set of macros to build a minimal size decoder, by @felixhandte
+build: fix compilation on MIPS32, reported by @clbr (#1441)
+build: fix compilation with multiple -arch flags, by @ryandesign
+build: highly upgraded meson build, by @lzutao
+build: improved buck support, by @obelisk
+build: fix cmake script : can create debug build, by @pitrou
+build: Makefile : grep works on both colored consoles and systems without color support
+build: fixed zstd-pgo, by @bmwiedemann
+cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
+cli : --no-progress flag, preserving final summary (#1371), by @terrelln
+cli : ensure destination file is not source file (#1422)
+cli : clearer error messages, especially when input file not present
+doc : clarified zstd_compression_format.md, by @ulikunitz
+misc: fixed zstdgrep, returns 1 on failure, by @lzutao
+misc: NEWS renamed as CHANGELOG, in accordance with fboss
+
 v1.3.7
 perf: slightly better decompression speed on clang (depending on hardware target)
 fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
@@ -217,7 +318,7 @@ v1.0.0
 Change Licensing, all project is now BSD, Copyright Facebook
 Small decompression speed improvement
 API : Streaming API supports legacy format
-API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
 CLI supports legacy formats v0.4+
 Fixed : compression fails on certain huge files, reported by Jesse McGrew
 Enhanced documentation, by Przemyslaw Skibinski

Makefile

@@ -64,7 +64,8 @@ zlibwrapper: lib
 
 ## test: run long-duration tests
 .PHONY: test
-test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
+DEBUGLEVEL ?= 1
+test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
 test:
 	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
@@ -129,7 +130,12 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
-EGREP = egrep --color=never
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
 
 # Print a two column output of targets and their description. To add a target description, put a
 # comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:
@@ -150,7 +156,7 @@ list:
 	    done \
 	} | column -t -s $$'\t'
 
-.PHONY: install clangtest armtest usan asan uasan
+.PHONY: install armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
@@ -182,7 +188,7 @@ gcc7build: clean
 .PHONY: clangbuild
 clangbuild: clean
 	clang -v
-	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
 
 m32build: clean
 	gcc -v
@@ -226,10 +232,6 @@ gcc6test: clean
 	gcc-6 -v
 	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
 
-clangtest: clean
-	clang -v
-	$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
-
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests

README.md

@@ -9,7 +9,12 @@ and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
 
-Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]
+**Development branch status:**
+
+[![Build Status][travisDevBadge]][travisLink]
+[![Build status][AppveyorDevBadge]][AppveyorLink]
+[![Build status][CircleDevBadge]][CircleLink]
+[![Build status][CirrusDevBadge]][CirrusLink]
 
 [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.org/facebook/zstd
@@ -17,14 +22,16 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink]   [![B
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd
+[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
+[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
 
-### Benchmarks
+## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
-with a Core i7-6700K CPU @ 4.0GHz,
+on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
+with a Core i9-9900K CPU @ 5.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 7.3.0,
+compiled with [gcc] 8.2.1,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -33,16 +40,16 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.3.4 -1**       | 2.877 |   470 MB/s |  1380 MB/s |
-| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   400 MB/s |
-| brotli 1.0.2 -0         | 2.701 |   410 MB/s |   430 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
-| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
-| lz4 1.8.1               | 2.101 |   750 MB/s |  3700 MB/s |
-| snappy 1.1.4            | 2.091 |   530 MB/s |  1800 MB/s |
-| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
+| **zstd 1.4.0 -1**       | 2.884 |   530 MB/s |  1360 MB/s |
+| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   440 MB/s |
+| brotli 1.0.7 -0         | 2.701 |   430 MB/s |   470 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   600 MB/s |   800 MB/s |
+| lzo1x 2.09 -1           | 2.106 |   680 MB/s |   950 MB/s |
+| lz4 1.8.3               | 2.101 |   800 MB/s |  4220 MB/s |
+| snappy 1.1.4            | 2.073 |   580 MB/s |  2020 MB/s |
+| lzf 3.6 -1              | 2.077 |   440 MB/s |   930 MB/s |
 
-[zlib]:http://www.zlib.net/
+[zlib]: http://www.zlib.net/
 [LZ4]: http://www.lz4.org/
 
 Zstd can also offer stronger compression ratios at the cost of compression speed.
@@ -65,7 +72,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
 For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
 
 
-### The case for Small Data compression
+## The case for Small Data compression
 
 Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
 
@@ -89,24 +96,24 @@ Training works if there is some correlation in a family of small data samples. T
 Hence, deploying one dictionary per type of data will provide the greatest benefits.
 Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
 
-#### Dictionary compression How To:
+### Dictionary compression How To:
 
-1) Create the dictionary
+1. Create the dictionary
 
-`zstd --train FullPathToTrainingSet/* -o dictionaryName`
+   `zstd --train FullPathToTrainingSet/* -o dictionaryName`
 
-2) Compress with dictionary
+2. Compress with dictionary
 
-`zstd -D dictionaryName FILE`
+   `zstd -D dictionaryName FILE`
 
-3) Decompress with dictionary
+3. Decompress with dictionary
 
-`zstd -D dictionaryName --decompress FILE.zst`
+   `zstd -D dictionaryName --decompress FILE.zst`
 
 
-### Build instructions
+## Build instructions
 
-#### Makefile
+### Makefile
 
 If your system is compatible with standard `make` (or `gmake`),
 invoking `make` in root directory will generate `zstd` cli in root directory.
@@ -115,7 +122,7 @@ Other available options include:
 - `make install` : create and install zstd cli, library and man pages
 - `make check` : create and run `zstd`, tests its behavior on local platform
 
-#### cmake
+### cmake
 
 A `cmake` project generator is provided within `build/cmake`.
 It can generate Makefiles or other build scripts
@@ -123,11 +130,17 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
 By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 
-#### Meson
+### Meson
 
-A Meson project is provided within `contrib/meson`.
+A Meson project is provided within [`build/meson`](build/meson). Follow
+build instructions in that directory.
 
-#### Visual Studio (Windows)
+You can also take a look at [`.travis.yml`](.travis.yml) file for an
+example about how Meson is used to build this project.
+
+Note that default build type is **release**.
+
+### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities:
 - Projects for Visual Studio 2005, 2008 and 2010.
@@ -135,17 +148,21 @@ Going into `build` directory, you will find additional possibilities:
 - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
   which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
 
+### Buck
 
-### Status
+You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
+The output binary will be in `buck-out/gen/programs/`.
+
+## Status
 
 Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
 Zstandard is considered safe for production environments.
 
-### License
+## License
 
 Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
 
-### Contributing
+## Contributing
 
 The "dev" branch is the one where all contributions are merged before reaching "master".
 If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.

appveyor.yml

@@ -3,6 +3,8 @@
   branches:
     only:
     - master
+    - appveyorTest
+    - /visual*/
   environment:
     matrix:
     - COMPILER: "gcc"

build/.gitignore

@@ -29,3 +29,5 @@ compile_commands.json
 CTestTestfile.cmake
 build
 lib
+!cmake/lib
+!meson/lib

build/VS2008/fullbench/fullbench.vcproj

@@ -364,6 +364,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
 				>
@@ -380,6 +388,14 @@
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\huf_decompress.c"
 				>
@@ -389,13 +405,21 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\bench.c"
+				RelativePath="..\..\..\programs\benchfn.c"
 				>
 			</File>
 			<File
 				RelativePath="..\..\..\tests\fullbench.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\programs\util.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\timefn.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -478,6 +502,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/fuzzer/fuzzer.vcproj

@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
-				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -328,6 +328,14 @@
 			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
+			<File
+				RelativePath="..\..\..\programs\util.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\timefn.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\programs\datagen.c"
 				>
@@ -404,10 +412,26 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -514,6 +538,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/zstd/zstd.vcproj

@@ -333,7 +333,19 @@
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
 			<File
-				RelativePath="..\..\..\programs\bench.c"
+				RelativePath="..\..\..\programs\util.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\timefn.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\benchfn.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\programs\benchzstd.c"
 				>
 			</File>
 			<File
@@ -412,10 +424,26 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\legacy\zstd_v01.c"
 				>
@@ -590,6 +618,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2008/zstdlib/zstdlib.vcproj

@@ -396,6 +396,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.c"
 				>
@@ -420,6 +428,14 @@
 				RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\legacy\zstd_v01.c"
 				>
@@ -534,6 +550,14 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_fast.h"
 				>

build/VS2010/datagen/datagen.vcxproj

@@ -155,6 +155,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\tests\datagencli.c" />
   </ItemGroup>
@@ -164,4 +165,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -166,14 +166,16 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
+    <ClCompile Include="..\..\..\programs\util.c" />
+    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\bench.c" />
+    <ClCompile Include="..\..\..\programs\benchfn.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
-    <ClInclude Include="..\..\..\programs\bench.h" />
+    <ClInclude Include="..\..\..\programs\benchfn.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <ItemGroup>
@@ -184,4 +186,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

build/VS2010/fullbench/fullbench.vcxproj

@@ -167,6 +167,8 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
@@ -175,8 +177,12 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include="..\..\..\programs\util.c" />
+    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\bench.c" />
+    <ClCompile Include="..\..\..\programs\benchfn.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
@@ -189,16 +195,19 @@
     <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
+    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
-    <ClInclude Include="..\..\..\programs\bench.h" />
+    <ClInclude Include="..\..\..\programs\benchfn.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -90,7 +90,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -105,7 +105,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
     </ClCompile>
@@ -122,7 +122,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <TreatWarningAsError>false</TreatWarningAsError>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -142,7 +142,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -167,6 +167,8 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -175,10 +177,14 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
+    <ClCompile Include="..\..\..\programs\util.c" />
+    <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\tests\fuzzer.c" />
   </ItemGroup>
@@ -192,12 +198,15 @@
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
+    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -31,6 +31,8 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -39,6 +41,8 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
@@ -76,12 +80,15 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
+    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="libzstd-dll.rc" />
@@ -206,7 +213,6 @@
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -231,7 +237,6 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <OmitFramePointers>true</OmitFramePointers>
-      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/VS2010/libzstd/libzstd.vcxproj

@@ -31,6 +31,8 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -39,6 +41,8 @@
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
@@ -76,42 +80,40 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
     <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
+    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>
     <RootNamespace>libzstd</RootNamespace>
+    <TargetName>libzstd_static</TargetName>
+    <CharacterSet>MultiByte</CharacterSet>
+    <ConfigurationType>StaticLibrary</ConfigurationType>
     <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
     <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
-    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
-    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-    <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
-    <CharacterSet>MultiByte</CharacterSet>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -131,35 +133,36 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <TargetName>libzstd_static</TargetName>
     <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
+    </ClCompile>
+  </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <PrecompiledHeader>
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -179,7 +182,7 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -199,11 +202,10 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -221,14 +223,13 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>false</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <OmitFramePointers>true</OmitFramePointers>
-      <AssemblerOutput>All</AssemblerOutput>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/VS2010/zstd/zstd.vcxproj

@@ -32,6 +32,8 @@
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -39,6 +41,8 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -50,7 +54,10 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
-    <ClCompile Include="..\..\..\programs\bench.c" />
+    <ClCompile Include="..\..\..\programs\util.c" />
+    <ClCompile Include="..\..\..\programs\timefn.c" />
+    <ClCompile Include="..\..\..\programs\benchfn.c" />
+    <ClCompile Include="..\..\..\programs\benchzstd.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
@@ -70,11 +77,14 @@
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
+    <ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
@@ -83,7 +93,7 @@
     <ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
-    <ClInclude Include="..\..\..\programs\bench.h" />
+    <ClInclude Include="..\..\..\programs\benchzstd.h" />
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\dibio.h" />
     <ClInclude Include="..\..\..\programs\fileio.h" />

build/cmake/CMakeLists.txt

@@ -7,104 +7,147 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(zstd)
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
-SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR)
+  
+# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies. 
+# Set and use the newest cmake policies that are validated to work 
+set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") 
+set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
+if("${CMAKE_MAJOR_VERSION}" LESS 3)
+  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
+elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND 
+        "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
+    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 
+else() 
+    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") 
+endif()
+cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
 
-# Ensure Release build even if not invoked via Makefile
-SET(CMAKE_BUILD_TYPE "Release")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
+set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+# Parse version
+include(GetZstdLibraryVersion)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
 
-LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
-INCLUDE(GNUInstallDirs)
+if( CMAKE_MAJOR_VERSION LESS 3 )
+  ## Provide cmake 3+ behavior for older versions of cmake
+  project(zstd)
+  set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
+  set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
+  set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
+  set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
+  enable_language(C)   # Main library is in C
+  enable_language(CXX) # Testing contributed code also utilizes CXX
+else()
+  project(zstd
+    VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
+    LANGUAGES C   # Main library is in C
+              CXX # Testing contributed code also utilizes CXX
+    )
+endif()
+message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
+set(zstd_HOMEPAGE_URL "http://www.zstd.net")
+set(zstd_DESCRIPTION  "Zstandard is a real-time compression algorithm, providing high compression ratios.")
+
+# Set a default build type if none was specified
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+  message(STATUS "Setting build type to 'Release' as none was specified.")
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+  # Set the possible values of build type for cmake-gui
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
+
+include(GNUInstallDirs)
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags
 #-----------------------------------------------------------------------------
-INCLUDE(AddZstdCompilationFlags)
+include(AddZstdCompilationFlags)
 ADD_ZSTD_COMPILATION_FLAGS()
 
 # Always hide XXHash symbols
-ADD_DEFINITIONS(-DXXH_NAMESPACE=ZSTD_)
+add_definitions(-DXXH_NAMESPACE=ZSTD_)
 
 #-----------------------------------------------------------------------------
 # Installation variables
 #-----------------------------------------------------------------------------
-MESSAGE(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
-MESSAGE(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
+message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
+message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
 
 #-----------------------------------------------------------------------------
 # Options
 #-----------------------------------------------------------------------------
 
 # Legacy support
-OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
 
-IF (ZSTD_LEGACY_SUPPORT)
-    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
-ELSE (ZSTD_LEGACY_SUPPORT)
-    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
-    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
-ENDIF (ZSTD_LEGACY_SUPPORT)
+if (ZSTD_LEGACY_SUPPORT)
+    message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=5)
+else ()
+    message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
+endif ()
 
 # Multi-threading support
-OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
+option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
 
-IF (ZSTD_MULTITHREAD_SUPPORT)
-    MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
-ELSE (ZSTD_MULTITHREAD_SUPPORT)
-    MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
-ENDIF (ZSTD_MULTITHREAD_SUPPORT)
+if (ZSTD_MULTITHREAD_SUPPORT)
+    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
+else ()
+    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
+endif ()
 
-OPTION(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
-OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
-OPTION(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
+option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
+option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
+option(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
 if (MSVC)
-    OPTION(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
+    option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
 endif ()
 
 #-----------------------------------------------------------------------------
 # External dependencies
 #-----------------------------------------------------------------------------
-IF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
-    SET(THREADS_PREFER_PTHREAD_FLAG ON)
-    FIND_PACKAGE(Threads REQUIRED)
-    IF(CMAKE_USE_PTHREADS_INIT)
-        SET(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
-    ELSE()
-        MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-    ENDIF()
-ENDIF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
+if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+    find_package(Threads REQUIRED)
+    if(CMAKE_USE_PTHREADS_INIT)
+        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+    else()
+        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+    endif()
+endif ()
 
 #-----------------------------------------------------------------------------
 # Add source directories
 #-----------------------------------------------------------------------------
-ADD_SUBDIRECTORY(lib)
+add_subdirectory(lib)
 
-IF (ZSTD_BUILD_PROGRAMS)
-    IF (NOT ZSTD_BUILD_STATIC)
-        MESSAGE(SEND_ERROR "You need to build static library to build zstd CLI")
-    ENDIF (NOT ZSTD_BUILD_STATIC)
+if (ZSTD_BUILD_PROGRAMS)
+    if (NOT ZSTD_BUILD_STATIC)
+        message(SEND_ERROR "You need to build static library to build zstd CLI")
+    endif ()
 
-    ADD_SUBDIRECTORY(programs)
-ENDIF (ZSTD_BUILD_PROGRAMS)
+    add_subdirectory(programs)
+endif ()
 
-IF (ZSTD_BUILD_TESTS)
-    IF (NOT ZSTD_BUILD_STATIC)
-        MESSAGE(SEND_ERROR "You need to build static library to build tests")
-    ENDIF (NOT ZSTD_BUILD_STATIC)
+if (ZSTD_BUILD_TESTS)
+    if (NOT ZSTD_BUILD_STATIC)
+        message(SEND_ERROR "You need to build static library to build tests")
+    endif ()
 
-    ADD_SUBDIRECTORY(tests)
-ENDIF (ZSTD_BUILD_TESTS)
+    add_subdirectory(tests)
+endif ()
 
-IF (ZSTD_BUILD_CONTRIB)
-    ADD_SUBDIRECTORY(contrib)
-ENDIF (ZSTD_BUILD_CONTRIB)
+if (ZSTD_BUILD_CONTRIB)
+    add_subdirectory(contrib)
+endif ()
 
 #-----------------------------------------------------------------------------
 # Add clean-all target
 #-----------------------------------------------------------------------------
-ADD_CUSTOM_TARGET(clean-all
+add_custom_target(clean-all
    COMMAND ${CMAKE_BUILD_TOOL} clean
    COMMAND rm -rf ${CMAKE_BINARY_DIR}/
 )

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -20,7 +20,7 @@ function(EnableCompilerFlag _flag _C _CXX)
     endif ()
 endfunction()
 
-MACRO(ADD_ZSTD_COMPILATION_FLAGS)
+macro(ADD_ZSTD_COMPILATION_FLAGS)
     if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         #Set c++11 by default
         EnableCompilerFlag("-std=c++11" false true)
@@ -46,22 +46,26 @@ MACRO(ADD_ZSTD_COMPILATION_FLAGS)
     endif ()
 
     # Remove duplicates compilation flags
-    FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+    foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
              CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
              CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
              CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-        separate_arguments(${flag_var})
-        list(REMOVE_DUPLICATES ${flag_var})
-        string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
-    ENDFOREACH (flag_var)
+        if( ${flag_var} )
+            separate_arguments(${flag_var})
+            list(REMOVE_DUPLICATES ${flag_var})
+            string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
+        endif()
+    endforeach ()
 
     if (MSVC AND ZSTD_USE_STATIC_RUNTIME)
-        FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+        foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
                  CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
                  CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
                  CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-            STRING(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
-        ENDFOREACH (flag_var)
+            if ( ${flag_var} )
+                string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+            endif()
+        endforeach ()
     endif ()
 
-ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)
+endmacro()

build/cmake/CMakeModules/FindLibLZ4.cmake

@@ -0,0 +1,49 @@
+# Find LibLZ4
+#
+# Find LibLZ4 headers and library
+#
+#   Result Variables
+#
+#   LIBLZ4_FOUND             - True if lz4 is found
+#   LIBLZ4_INCLUDE_DIRS      - lz4 headers directories
+#   LIBLZ4_LIBRARIES         - lz4 libraries
+#   LIBLZ4_VERSION_MAJOR     - The major version of lz4
+#   LIBLZ4_VERSION_MINOR     - The minor version of lz4
+#   LIBLZ4_VERSION_RELEASE   - The release version of lz4
+#   LIBLZ4_VERSION_STRING    - version number string (e.g. 1.8.3)
+#
+#   Hints
+#
+#   Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library
+
+set(_LIBLZ4_ROOT_HINTS
+    ENV LZ4_ROOT_DIR)
+
+find_path(  LIBLZ4_INCLUDE_DIR lz4.h
+            HINTS ${_LIBLZ4_ROOT_HINTS})
+find_library(   LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
+                HINTS ${_LIBLZ4_ROOT_HINTS})
+
+if(LIBLZ4_INCLUDE_DIR)
+    file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")
+
+    string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
+    string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
+    string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")
+
+    set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
+    unset(LIBLZ4_HEADER_CONTENT)
+endif()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4    REQUIRED_VARS   LIBLZ4_INCLUDE_DIR
+                                                            LIBLZ4_LIBRARY
+                                            VERSION_VAR     LIBLZ4_VERSION_STRING
+                                            FAIL_MESSAGE    "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")
+
+if (LIBLZ4_FOUND)
+    set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
+    set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
+endif ()
+
+mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )

build/cmake/CMakeModules/GetZstdLibraryVersion.cmake

@@ -1,9 +1,10 @@
-function(GetZstdLibraryVersion _header _major _minor _release)
+function(GetZstdLibraryVersion _header _major _minor _patch)
     # Read file content
-    FILE(READ ${_header} CONTENT)
+    file(READ ${_header} CONTENT)
 
     string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}")
-    SET(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
-    SET(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
-    SET(${_release} ${CMAKE_MATCH_3} PARENT_SCOPE)
+    set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
+    set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
+    set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE)
 endfunction()
+

build/cmake/README.md

@@ -0,0 +1,104 @@
+# Cmake contributions
+
+Contributions to the cmake build configurations are welcome. Please
+use case sensitivity that matches modern (ie. cmake version 2.6 and above)
+conventions of using lower-case for commands, and upper-case for
+variables.
+
+# How to build
+
+As cmake doesn't support command like `cmake clean`, it's recommanded to perform a "out of source build".
+To do this, you can create a new directory and build in it:
+```sh
+cd build/cmake
+mkdir builddir
+cd builddir
+cmake ..
+make
+```
+Then you can clean all cmake caches by simpily delete the new directory:
+```sh
+rm -rf build/cmake/builddir
+```
+
+And of course, you can directly build in build/cmake:
+```sh
+cd build/cmake
+cmake
+make
+```
+
+To show cmake build options, you can:
+```sh
+cd build/cmake/builddir
+cmake -LH ..
+```
+
+Bool options can be set to ON/OFF with -D\[option\]=\[ON/OFF\]. You can configure cmake options like this:
+```sh
+cd build/cmake/builddir
+cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
+make
+```
+
+## referring
+[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
+
+# CMake Style Recommendations
+
+## Indent all code correctly, i.e. the body of
+
+ * if/else/endif
+ * foreach/endforeach
+ * while/endwhile
+ * macro/endmacro
+ * function/endfunction
+
+Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
+spaces for indenting as is used in the rest of the file. Do not use tabs.
+
+## Upper/lower casing
+
+Most important: use consistent upper- or lowercasing within one file !
+
+In general, the all-lowercase style is preferred.
+
+So, this is recommended:
+
+```
+add_executable(foo foo.c)
+```
+
+These forms are discouraged
+
+```
+ADD_EXECUTABLE(bar bar.c)
+Add_Executable(hello hello.c)
+aDd_ExEcUtAbLe(blub blub.c)
+```
+
+## End commands
+To make the code easier to read, use empty commands for endforeach(), endif(),
+endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
+
+For example, do this:
+
+```
+if(FOOVAR)
+   some_command(...)
+else()
+   another_command(...)
+endif()
+```
+
+and not this:
+
+```
+if(BARVAR)
+   some_other_command(...)
+endif(BARVAR)
+```
+
+## Other resources for best practices
+
+`https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules`

build/cmake/contrib/CMakeLists.txt

@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(contrib)
+project(contrib)
 
-ADD_SUBDIRECTORY(pzstd)
-ADD_SUBDIRECTORY(gen_html)
+add_subdirectory(pzstd)
+add_subdirectory(gen_html)

build/cmake/contrib/gen_html/CMakeLists.txt

@@ -7,24 +7,24 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(gen_html)
-INCLUDE(GetZstdLibraryVersion)
+project(gen_html)
+include(GetZstdLibraryVersion)
 
-SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-SET(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
-SET(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
-INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
+set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
+set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
+include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
 
-ADD_EXECUTABLE(gen_html ${GENHTML_DIR}/gen_html.cpp)
+add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp)
 
 GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
-SET(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
-ADD_CUSTOM_TARGET(zstd_manual.html ALL
+set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
+add_custom_target(zstd_manual.html ALL
                   ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
                   DEPENDS gen_html COMMENT "Update zstd manual")
 
-INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
+install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -7,26 +7,26 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(pzstd)
+project(pzstd)
 
-SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
-INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
+set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
+include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
 
-ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
-SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
-SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
+add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
+set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 
-SET(THREADS_PREFER_PTHREAD_FLAG ON)
-FIND_PACKAGE(Threads REQUIRED)
-IF (CMAKE_USE_PTHREADS_INIT)
-    TARGET_LINK_LIBRARIES(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
-ELSE()
-    MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-ENDIF()
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+if (CMAKE_USE_PTHREADS_INIT)
+    target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
+else()
+    message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+endif()
 
-INSTALL(TARGETS pzstd RUNTIME DESTINATION "bin")
+install(TARGETS pzstd RUNTIME DESTINATION "bin")

build/cmake/lib/CMakeLists.txt

@@ -7,82 +7,51 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(libzstd)
+project(libzstd)
 
-SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
-OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
-OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
+option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
 
-IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
-    MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd")
-ENDIF()
+if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
+    message(SEND_ERROR "You need to build at least one flavor of libzstd")
+endif()
 
 # Define library directory, where sources and header files are located
-SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
+include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
-# Parse version
-INCLUDE(GetZstdLibraryVersion)
-GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
-MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
+file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
+file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
+file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
+file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
+file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
 
-SET(Sources
-        ${LIBRARY_DIR}/common/entropy_common.c
-        ${LIBRARY_DIR}/common/fse_decompress.c
-        ${LIBRARY_DIR}/common/threading.c
-        ${LIBRARY_DIR}/common/pool.c
-        ${LIBRARY_DIR}/common/zstd_common.c
-        ${LIBRARY_DIR}/common/error_private.c
-        ${LIBRARY_DIR}/common/xxhash.c
-        ${LIBRARY_DIR}/compress/hist.c
-        ${LIBRARY_DIR}/compress/fse_compress.c
-        ${LIBRARY_DIR}/compress/huf_compress.c
-        ${LIBRARY_DIR}/compress/zstd_compress.c
-        ${LIBRARY_DIR}/compress/zstdmt_compress.c
-        ${LIBRARY_DIR}/compress/zstd_fast.c
-        ${LIBRARY_DIR}/compress/zstd_double_fast.c
-        ${LIBRARY_DIR}/compress/zstd_lazy.c
-        ${LIBRARY_DIR}/compress/zstd_opt.c
-        ${LIBRARY_DIR}/compress/zstd_ldm.c
-        ${LIBRARY_DIR}/decompress/huf_decompress.c
-        ${LIBRARY_DIR}/decompress/zstd_decompress.c
-        ${LIBRARY_DIR}/dictBuilder/cover.c
-        ${LIBRARY_DIR}/dictBuilder/fastcover.c
-        ${LIBRARY_DIR}/dictBuilder/divsufsort.c
-        ${LIBRARY_DIR}/dictBuilder/zdict.c
-        ${LIBRARY_DIR}/deprecated/zbuff_common.c
-        ${LIBRARY_DIR}/deprecated/zbuff_compress.c
-        ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
+set(Sources
+        ${CommonSources}
+        ${CompressSources}
+        ${DecompressSources}
+        ${DictBuilderSources}
+        ${DeprecatedSources})
 
-SET(Headers
+file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
+file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
+file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
+file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
+file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
+
+set(Headers
         ${LIBRARY_DIR}/zstd.h
-        ${LIBRARY_DIR}/common/debug.h
-        ${LIBRARY_DIR}/common/pool.h
-        ${LIBRARY_DIR}/common/threading.h
-        ${LIBRARY_DIR}/common/bitstream.h
-        ${LIBRARY_DIR}/common/error_private.h
-        ${LIBRARY_DIR}/common/zstd_errors.h
-        ${LIBRARY_DIR}/common/fse.h
-        ${LIBRARY_DIR}/common/huf.h
-        ${LIBRARY_DIR}/common/mem.h
-        ${LIBRARY_DIR}/common/zstd_internal.h
-        ${LIBRARY_DIR}/compress/hist.h
-        ${LIBRARY_DIR}/compress/zstd_compress_internal.h
-        ${LIBRARY_DIR}/compress/zstd_fast.h
-        ${LIBRARY_DIR}/compress/zstd_double_fast.h
-        ${LIBRARY_DIR}/compress/zstd_lazy.h
-        ${LIBRARY_DIR}/compress/zstd_opt.h
-        ${LIBRARY_DIR}/compress/zstd_ldm.h
-        ${LIBRARY_DIR}/compress/zstdmt_compress.h
-        ${LIBRARY_DIR}/dictBuilder/zdict.h
-        ${LIBRARY_DIR}/dictBuilder/cover.h
-        ${LIBRARY_DIR}/deprecated/zbuff.h)
+        ${CommonHeaders}
+        ${CompressHeaders}
+        ${DecompressHeaders}
+        ${DictBuilderHeaders}
+        ${DeprecatedHeaders})
 
-IF (ZSTD_LEGACY_SUPPORT)
-    SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
-    INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
+if (ZSTD_LEGACY_SUPPORT)
+    set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
+    include_directories(${LIBRARY_LEGACY_DIR})
 
-    SET(Sources ${Sources}
+    set(Sources ${Sources}
             ${LIBRARY_LEGACY_DIR}/zstd_v01.c
             ${LIBRARY_LEGACY_DIR}/zstd_v02.c
             ${LIBRARY_LEGACY_DIR}/zstd_v03.c
@@ -91,7 +60,7 @@ IF (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v06.c
             ${LIBRARY_LEGACY_DIR}/zstd_v07.c)
 
-    SET(Headers ${Headers}
+    set(Headers ${Headers}
             ${LIBRARY_LEGACY_DIR}/zstd_legacy.h
             ${LIBRARY_LEGACY_DIR}/zstd_v01.h
             ${LIBRARY_LEGACY_DIR}/zstd_v02.h
@@ -100,83 +69,84 @@ IF (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v05.h
             ${LIBRARY_LEGACY_DIR}/zstd_v06.h
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
-ENDIF (ZSTD_LEGACY_SUPPORT)
+endif ()
 
-IF (MSVC)
-    SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
-    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
-ENDIF (MSVC)
+if (MSVC)
+    set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
+    set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
+endif ()
 
 # Split project to static and shared libraries build
-IF (ZSTD_BUILD_SHARED)
-    ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
-    IF (ZSTD_MULTITHREAD_SUPPORT)
-        SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
-        IF (UNIX)
-            TARGET_LINK_LIBRARIES(libzstd_shared ${THREADS_LIBS})
-        ENDIF ()
-    ENDIF()
-ENDIF (ZSTD_BUILD_SHARED)
-IF (ZSTD_BUILD_STATIC)
-    ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
-    IF (ZSTD_MULTITHREAD_SUPPORT)
-        SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
-        IF (UNIX)
-            TARGET_LINK_LIBRARIES(libzstd_static ${THREADS_LIBS})
-        ENDIF ()
-    ENDIF ()
-ENDIF (ZSTD_BUILD_STATIC)
+if (ZSTD_BUILD_SHARED)
+    add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
+    if (ZSTD_MULTITHREAD_SUPPORT)
+        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        if (UNIX)
+            target_link_libraries(libzstd_shared ${THREADS_LIBS})
+        endif ()
+    endif()
+endif ()
+if (ZSTD_BUILD_STATIC)
+    add_library(libzstd_static STATIC ${Sources} ${Headers})
+    if (ZSTD_MULTITHREAD_SUPPORT)
+        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+        if (UNIX)
+            target_link_libraries(libzstd_static ${THREADS_LIBS})
+        endif ()
+    endif ()
+endif ()
 
 # Add specific compile definitions for MSVC project
-IF (MSVC)
-    IF (ZSTD_BUILD_SHARED)
-        SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
-    ENDIF (ZSTD_BUILD_SHARED)
-    IF (ZSTD_BUILD_STATIC)
-        SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
-    ENDIF (ZSTD_BUILD_STATIC)
-ENDIF (MSVC)
+if (MSVC)
+    if (ZSTD_BUILD_SHARED)
+        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
+    endif ()
+    if (ZSTD_BUILD_STATIC)
+        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
+    endif ()
+endif ()
 
 # With MSVC static library needs to be renamed to avoid conflict with import library
-IF (MSVC)
-    SET(STATIC_LIBRARY_BASE_NAME zstd_static)
-ELSE ()
-    SET(STATIC_LIBRARY_BASE_NAME zstd)
-ENDIF (MSVC)
+if (MSVC)
+    set(STATIC_LIBRARY_BASE_NAME zstd_static)
+else ()
+    set(STATIC_LIBRARY_BASE_NAME zstd)
+endif ()
 
 # Define static and shared library names
-IF (ZSTD_BUILD_SHARED)
-    SET_TARGET_PROPERTIES(
+if (ZSTD_BUILD_SHARED)
+    set_target_properties(
             libzstd_shared
             PROPERTIES
             OUTPUT_NAME zstd
-            SOVERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE})
-ENDIF (ZSTD_BUILD_SHARED)
+            VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
+            SOVERSION ${zstd_VERSION_MAJOR})
+endif ()
 
-IF (ZSTD_BUILD_STATIC)
-    SET_TARGET_PROPERTIES(
+if (ZSTD_BUILD_STATIC)
+    set_target_properties(
             libzstd_static
             PROPERTIES
             OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
-ENDIF (ZSTD_BUILD_STATIC)
+endif ()
 
-IF (UNIX)
+if (UNIX)
     # pkg-config
-    SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
-    SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
-    SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
-    SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
-    ADD_CUSTOM_TARGET(libzstd.pc ALL
+    set(PREFIX "${CMAKE_INSTALL_PREFIX}")
+    set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
+    set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
+    add_custom_target(libzstd.pc ALL
             ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
             -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
             -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
             COMMENT "Creating pkg-config file")
 
-    INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${LIBDIR}/pkgconfig")
-ENDIF (UNIX)
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${LIBDIR}/pkgconfig")
+endif ()
 
 # install target
-INSTALL(FILES
+install(FILES
     ${LIBRARY_DIR}/zstd.h
     ${LIBRARY_DIR}/deprecated/zbuff.h
     ${LIBRARY_DIR}/dictBuilder/zdict.h
@@ -184,20 +154,22 @@ INSTALL(FILES
     ${LIBRARY_DIR}/common/zstd_errors.h
     DESTINATION "include")
 
-IF (ZSTD_BUILD_SHARED)
-    INSTALL(TARGETS libzstd_shared RUNTIME DESTINATION "bin"
+if (ZSTD_BUILD_SHARED)
+    install(TARGETS libzstd_shared RUNTIME DESTINATION "bin"
                                    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
                                    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-ENDIF()
-IF (ZSTD_BUILD_STATIC)
-    INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-ENDIF (ZSTD_BUILD_STATIC)
+endif()
+if (ZSTD_BUILD_STATIC)
+    install(TARGETS libzstd_static ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+endif ()
 
 # uninstall target
-CONFIGURE_FILE(
-        "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
-        "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
-        IMMEDIATE @ONLY)
+if (NOT TARGET uninstall)
+    configure_file(
+            "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
+            "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
+            IMMEDIATE @ONLY)
 
-ADD_CUSTOM_TARGET(uninstall
-        COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
+    add_custom_target(uninstall
+            COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
+endif ()

build/cmake/lib/cmake_uninstall.cmake.in

@@ -1,7 +1,7 @@
  
 if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
   message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
-endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
+endif()
 
 file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
 string(REGEX REPLACE "\n" ";" files "${files}")
@@ -15,8 +15,8 @@ foreach(file ${files})
       )
     if(NOT "${rm_retval}" STREQUAL 0)
       message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
-    endif(NOT "${rm_retval}" STREQUAL 0)
-  else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
+    endif()
+  else()
     message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
-  endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
-endforeach(file)
+  endif()
+endforeach()

build/cmake/lib/pkgconfig.cmake

@@ -1 +1 @@
-CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)
+configure_file("${IN}" "${OUT}" @ONLY)

build/cmake/programs/CMakeLists.txt

@@ -7,104 +7,121 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-PROJECT(programs)
+project(programs)
 
-SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
+set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-IF (ZSTD_LEGACY_SUPPORT)
-    SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
-    INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
-ENDIF (ZSTD_LEGACY_SUPPORT)
+if (ZSTD_LEGACY_SUPPORT)
+    set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
+    include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
+endif ()
 
-IF (MSVC)
-    SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
-    SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
-ENDIF (MSVC)
+if (MSVC)
+    set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
+    set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
+endif ()
 
-ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
-TARGET_LINK_LIBRARIES(zstd libzstd_static)
-IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
-    TARGET_LINK_LIBRARIES(zstd rt)
-ENDIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
-INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
+add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+target_link_libraries(zstd libzstd_static)
+if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
+    target_link_libraries(zstd rt)
+endif ()
+install(TARGETS zstd RUNTIME DESTINATION "bin")
 
-IF (UNIX)
-    ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
-    ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
-    INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
+if (UNIX)
+    add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
+    add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
+    install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "bin")
+    install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "bin")
 
-    ADD_CUSTOM_TARGET(zstd.1 ALL
+    add_custom_target(zstd.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
         COMMENT "Copying manpage zstd.1")
-    ADD_CUSTOM_TARGET(zstdgrep.1 ALL
+    add_custom_target(zstdgrep.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
         COMMENT "Copying manpage zstdgrep.1")
-    ADD_CUSTOM_TARGET(zstdless.1 ALL
+    add_custom_target(zstdless.1 ALL
         ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
         COMMENT "Copying manpage zstdless.1")
-    ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
-    ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
+    add_custom_target(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
+    add_custom_target(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
 
     # Define MAN_INSTALL_DIR if necessary
-    IF (MAN_INSTALL_DIR)
-    ELSE (MAN_INSTALL_DIR)
-      SET(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1)
-    ENDIF (MAN_INSTALL_DIR)
+    if (MAN_INSTALL_DIR)
+    else ()
+      set(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1)
+    endif ()
 
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
 
-    ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
-    TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
-    SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
-ENDIF (UNIX)
+    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
+    target_link_libraries(zstd-frugal libzstd_static)
+    set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
+endif ()
 
 # Add multi-threading support definitions
 
-IF (ZSTD_MULTITHREAD_SUPPORT)
-    SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+if (ZSTD_MULTITHREAD_SUPPORT)
+    set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
 
-    IF (UNIX)
-        TARGET_LINK_LIBRARIES(zstd ${THREADS_LIBS})
+    if (UNIX)
+        target_link_libraries(zstd ${THREADS_LIBS})
 
-        ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
-        INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
-    ENDIF (UNIX)
-ENDIF (ZSTD_MULTITHREAD_SUPPORT)
+        add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
+        install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
+    endif ()
+endif ()
 
-OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
-OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
+option(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
+option(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
+option(ZSTD_LZ4_SUPPORT "LZ4 SUPPORT" OFF)
 
-IF (ZSTD_ZLIB_SUPPORT)
-    FIND_PACKAGE(ZLIB REQUIRED)
+# Add gzip support
+if (ZSTD_ZLIB_SUPPORT)
+    find_package(ZLIB REQUIRED)
 
-    IF (ZLIB_FOUND)
-        INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
-        TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
-        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
-    ELSE ()
-        MESSAGE(SEND_ERROR "zlib library is missing")
-    ENDIF ()
-ENDIF ()
+    if (ZLIB_FOUND)
+        include_directories(${ZLIB_INCLUDE_DIRS})
+        target_link_libraries(zstd ${ZLIB_LIBRARIES})
+        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
+    else ()
+        message(SEND_ERROR "zlib library is missing")
+    endif ()
+endif ()
 
-IF (ZSTD_LZMA_SUPPORT)
-    FIND_PACKAGE(LibLZMA REQUIRED)
+# Add lzma support
+if (ZSTD_LZMA_SUPPORT)
+    find_package(LibLZMA REQUIRED)
 
-    IF (LIBLZMA_FOUND)
-        INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
-        TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
-        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
-    ELSE ()
-        MESSAGE(SEND_ERROR "lzma library is missing")
-    ENDIF ()
-ENDIF ()
+    if (LIBLZMA_FOUND)
+        include_directories(${LIBLZMA_INCLUDE_DIRS})
+        target_link_libraries(zstd ${LIBLZMA_LIBRARIES})
+        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
+    else ()
+        message(SEND_ERROR "lzma library is missing")
+    endif ()
+endif ()
+
+# Add lz4 support
+if (ZSTD_LZ4_SUPPORT)
+    find_package(LibLZ4 REQUIRED)
+
+    if (LIBLZ4_FOUND)
+        include_directories(${LIBLZ4_INCLUDE_DIRS})
+        target_link_libraries(zstd ${LIBLZ4_LIBRARIES})
+        set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZ4COMPRESS;ZSTD_LZ4DECOMPRESS")
+    else ()
+        message(SEND_ERROR "lz4 library is missing")
+    endif ()
+endif ()

build/cmake/tests/CMakeLists.txt

@@ -30,26 +30,26 @@
 #  - zstd homepage : http://www.zstd.net/
 # ################################################################
 
-PROJECT(tests)
+project(tests)
 
-SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
-SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
-SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
-SET(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
-INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
+set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
+set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
+include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
-TARGET_LINK_LIBRARIES(datagen libzstd_static)
+add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+target_link_libraries(datagen libzstd_static)
 
-ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
-TARGET_LINK_LIBRARIES(fullbench libzstd_static)
+add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
+target_link_libraries(fullbench libzstd_static)
 
-ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
-TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
+add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
+target_link_libraries(fuzzer libzstd_static)
 
-IF (UNIX)
-    ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
-    TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
-ENDIF (UNIX)
+if (UNIX)
+    add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
+    target_link_libraries(paramgrill libzstd_static m) #m is math library
+endif ()

build/meson/GetZstdLibraryVersion.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+import re
+
+
+def find_version_tuple(filepath):
+  version_file_data = None
+  with open(filepath) as fd:
+    version_file_data = fd.read()
+
+  patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
+#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
+#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
+"""
+  regex = re.compile(patterns, re.MULTILINE)
+  version_match = regex.search(version_file_data)
+  if version_match:
+    return version_match.groups()
+  raise Exception("Unable to find version string")
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
+  parser.add_argument('file', help='path to lib/zstd.h')
+  args = parser.parse_args()
+  version_tuple = find_version_tuple(args.file)
+  print('.'.join(version_tuple))
+
+
+if __name__ == '__main__':
+  main()

build/meson/InstallSymlink.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present  lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+# This file should be synced with https://github.com/lzutao/meson-symlink
+
+import os
+import pathlib  # since Python 3.4
+
+
+def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
+  if not install_dir.exists():
+    install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
+  if not install_dir.is_dir():
+    raise NotADirectoryError(install_dir)
+
+  new_dst = install_dir.joinpath(dst)
+  if new_dst.is_symlink() and os.readlink(new_dst) == src:
+    print('File exists: {!r} -> {!r}'.format(new_dst, src))
+    return
+  print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
+  new_dst.symlink_to(src, target_is_directory=dst_is_dir)
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Install a symlink',
+      usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
+            'example:\n'
+            '        {0} dash sh /bin'.format(pathlib.Path(__file__).name))
+  parser.add_argument('source', help='target to link')
+  parser.add_argument('dest', help='link name')
+  parser.add_argument('install_dir', help='installation directory')
+  parser.add_argument('-d', '--isdir',
+      action='store_true',
+      help='dest is a directory')
+  parser.add_argument('-m', '--mode',
+      help='directory mode on creating if not exist',
+      default='0o755')
+  args = parser.parse_args()
+
+  dir_mode = int(args.mode, 8)
+
+  meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
+  install_dir = pathlib.Path(meson_destdir, args.install_dir)
+  install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
+
+
+if __name__ == '__main__':
+  main()

build/meson/README.md

@@ -0,0 +1,38 @@
+Meson build system for zstandard
+================================
+
+Meson is a build system designed to optimize programmer productivity.
+It aims to do this by providing simple, out-of-the-box support for
+modern software development tools and practices, such as unit tests,
+coverage reports, Valgrind, CCache and the like.
+
+This Meson build system is provided with no guarantee and maintained
+by Dima Krasner \<dima@dimakrasner.com\>.
+
+It outputs one `libzstd`, either shared or static, depending on
+`default_library` option.
+
+## How to build
+
+`cd` to this meson directory (`build/meson`)
+
+```sh
+meson setup -Dbin_programs=true -Dbin_contrib=true builddir
+cd builddir
+ninja             # to build
+ninja install     # to install
+```
+
+You might want to install it in staging directory:
+
+```sh
+DESTDIR=./staging ninja install
+```
+
+To configure build options, use:
+
+```sh
+meson configure
+```
+
+See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).

build/meson/contrib/gen_html/meson.build

@@ -0,0 +1,30 @@
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../../..'
+
+gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
+  join_paths(zstd_rootdir, 'lib'),
+  join_paths(zstd_rootdir, 'lib/common'),
+  join_paths(zstd_rootdir, 'contrib/gen_html'))
+
+gen_html = executable('gen_html',
+  join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
+  include_directories: gen_html_includes,
+  native: true,
+  install: false)
+
+# Update zstd manual
+zstd_manual_html = custom_target('zstd_manual.html',
+  output : 'zstd_manual.html',
+  command : [gen_html,
+    zstd_version,
+    join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
+    '@OUTPUT@'],
+  install : false)

build/meson/contrib/meson.build

@@ -0,0 +1,12 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+subdir('pzstd')
+subdir('gen_html')

build/meson/contrib/pzstd/meson.build

@@ -0,0 +1,24 @@
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../../..'
+
+pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
+  join_paths(zstd_rootdir, 'contrib/pzstd'))
+pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
+pzstd = executable('pzstd',
+  pzstd_sources,
+  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
+  include_directories: pzstd_includes,
+  dependencies: [ libzstd_dep, thread_dep ],
+  install: true)

build/meson/lib/meson.build

@@ -0,0 +1,131 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
+  join_paths(zstd_rootdir, 'lib/common'),
+  join_paths(zstd_rootdir, 'lib/compress'),
+  join_paths(zstd_rootdir, 'lib/decompress'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder'),
+  join_paths(zstd_rootdir, 'lib/deprecated'))]
+
+libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/common/threading.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c'),
+  join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
+  join_paths(zstd_rootdir, 'lib/compress/hist.c'),
+  join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
+
+# Explicit define legacy support
+add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
+  language: 'c')
+
+if legacy_level == 0
+  message('Legacy support: DISABLED')
+else
+  # See ZSTD_LEGACY_SUPPORT of lib/README.md
+  message('Enable legacy support back to version 0.@0@'.format(legacy_level))
+
+  libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
+  foreach i : [1, 2, 3, 4, 5, 6, 7]
+    if legacy_level <= i
+      libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
+    endif
+  endforeach
+endif
+
+libzstd_deps = []
+if use_multi_thread
+  message('Enable multi-threading support')
+  add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
+  libzstd_deps = [ thread_dep ]
+endif
+
+libzstd_c_args = []
+if cc_id == compiler_msvc
+  if default_library_type != 'static'
+    libzstd_sources += [windows_mod.compile_resources(
+      join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
+    libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
+      '-DZSTD_HEAPMODE=0',
+      '-D_CONSOLE',
+      '-D_CRT_SECURE_NO_WARNINGS']
+  else
+    libzstd_c_args += ['-DZSTD_HEAPMODE=0',
+      '-D_CRT_SECURE_NO_WARNINGS']
+  endif
+endif
+
+mingw_ansi_stdio_flags = []
+if host_machine_os == os_windows and cc_id == compiler_gcc
+  mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
+endif
+libzstd_c_args += mingw_ansi_stdio_flags
+
+libzstd_debug_cflags = []
+if use_debug
+  libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
+  if cc_id == compiler_gcc or cc_id == compiler_clang
+    libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
+      '-Wdeclaration-after-statement', '-Wstrict-prototypes',
+      '-Wundef', '-Wpointer-arith', '-Wvla',
+      '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
+      '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
+  endif
+endif
+libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
+
+libzstd = library('zstd',
+  libzstd_sources,
+  include_directories: libzstd_includes,
+  c_args: libzstd_c_args,
+  dependencies: libzstd_deps,
+  install: true,
+  version: zstd_libversion)
+
+libzstd_dep = declare_dependency(link_with: libzstd,
+  include_directories: libzstd_includes)
+
+pkgconfig.generate(libzstd,
+  name: 'libzstd',
+  filebase: 'libzstd',
+  description: 'fast lossless compression algorithm library',
+  version: zstd_libversion,
+  url: 'http://www.zstd.net/')
+
+install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff.h'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))

build/meson/meson.build

@@ -0,0 +1,146 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+project('zstd',
+  ['c', 'cpp'],
+  license: ['BSD', 'GPLv2'],
+  default_options : [
+    'c_std=gnu99',
+    'cpp_std=c++11',
+    'buildtype=release'
+  ],
+  version: 'DUMMY',
+  meson_version: '>=0.47.0')
+
+cc = meson.get_compiler('c')
+cxx = meson.get_compiler('cpp')
+pkgconfig = import('pkgconfig')
+python3 = import('python').find_installation()
+windows_mod = import('windows')
+
+host_machine_os = host_machine.system()
+os_windows = 'windows'
+os_linux = 'linux'
+os_darwin = 'darwin'
+os_freebsd = 'freebsd'
+os_sun = 'sunos'
+
+cc_id = cc.get_id()
+compiler_gcc = 'gcc'
+compiler_clang = 'clang'
+compiler_msvc = 'msvc'
+
+zstd_version = meson.project_version()
+
+zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
+GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py')
+r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file)
+if r.returncode() == 0
+  zstd_version = r.stdout().strip()
+  message('Project version is now: @0@'.format(zstd_version))
+else
+  error('Cannot find project version in @0@'.format(zstd_h_file))
+endif
+
+zstd_libversion = zstd_version
+
+# =============================================================================
+# Installation directories
+# =============================================================================
+
+zstd_prefix = get_option('prefix')
+zstd_bindir = get_option('bindir')
+zstd_datadir = get_option('datadir')
+zstd_mandir = get_option('mandir')
+zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
+
+# =============================================================================
+# Project options
+# =============================================================================
+
+# Built-in options
+use_debug = get_option('debug')
+buildtype = get_option('buildtype')
+
+# Custom options
+debug_level = get_option('debug_level')
+legacy_level = get_option('legacy_level')
+use_backtrace = get_option('backtrace')
+use_static_runtime = get_option('static_runtime')
+
+bin_programs = get_option('bin_programs')
+bin_contrib = get_option('bin_contrib')
+bin_tests = get_option('bin_tests')
+
+feature_multi_thread = get_option('multi_thread')
+feature_zlib = get_option('zlib')
+feature_lzma = get_option('lzma')
+feature_lz4 = get_option('lz4')
+
+# =============================================================================
+# Dependencies
+# =============================================================================
+
+libm_dep = cc.find_library('m', required: bin_tests)
+thread_dep = dependency('threads', required: feature_multi_thread)
+use_multi_thread = thread_dep.found()
+# Arguments in dependency should be equivalent to those passed to pkg-config
+zlib_dep = dependency('zlib', required: feature_zlib)
+use_zlib = zlib_dep.found()
+lzma_dep = dependency('liblzma', required: feature_lzma)
+use_lzma = lzma_dep.found()
+lz4_dep = dependency('liblz4', required: feature_lz4)
+use_lz4 = lz4_dep.found()
+
+# =============================================================================
+# Compiler flags
+# =============================================================================
+
+add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
+
+if [compiler_gcc, compiler_clang].contains(cc_id)
+  common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
+  if cc_id == compiler_clang
+    # Should use Meson's own --werror build option
+    #common_warning_flags += '-Werror'
+    common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
+  endif
+  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
+  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
+  add_project_arguments(cc_compile_flags, language : 'c')
+  add_project_arguments(cxx_compile_flags, language : 'cpp')
+elif cc_id == compiler_msvc
+  msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
+  if use_multi_thread
+    msvc_compile_flags += '/MP'
+  endif
+  if enable_static_runtime
+    msvc_compile_flags += '/MT'
+  endif
+  add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
+endif
+
+# =============================================================================
+# Subdirs
+# =============================================================================
+
+subdir('lib')
+
+if bin_programs
+  subdir('programs')
+endif
+
+if bin_tests
+  subdir('tests')
+endif
+
+if bin_contrib
+  subdir('contrib')
+endif

build/meson/meson_options.txt

@@ -0,0 +1,36 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
+
+option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
+  description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
+option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
+  description: 'Enable run-time debug. See lib/common/debug.h')
+option('backtrace', type: 'boolean', value: false,
+  description: 'Display a stack backtrace when execution generates a runtime exception')
+option('static_runtime', type: 'boolean', value: false,
+  description: 'Link to static run-time libraries on MSVC')
+
+option('bin_programs', type: 'boolean', value: true,
+  description: 'Enable programs build')
+option('bin_tests', type: 'boolean', value: false,
+  description: 'Enable tests build')
+option('bin_contrib', type: 'boolean', value: false,
+  description: 'Enable contrib build')
+
+option('multi_thread', type: 'feature', value: 'enabled',
+  description: 'Enable multi-threading when pthread is detected')
+option('zlib', type: 'feature', value: 'auto',
+  description: 'Enable zlib support')
+option('lzma', type: 'feature', value: 'auto',
+  description: 'Enable lzma support')
+option('lz4', type: 'feature', value: 'auto',
+  description: 'Enable lz4 support')

build/meson/programs/meson.build

@@ -0,0 +1,104 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/fileio.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/dibio.c')]
+
+zstd_c_args = libzstd_debug_cflags
+if use_multi_thread
+  zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
+endif
+
+zstd_deps = [ libzstd_dep ]
+if use_zlib
+  zstd_deps += [ zlib_dep ]
+  zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
+endif
+
+if use_lzma
+  zstd_deps += [ lzma_dep ]
+  zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
+endif
+
+if use_lz4
+  zstd_deps += [ lz4_dep ]
+  zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
+endif
+
+export_dynamic_on_windows = false
+# explicit backtrace enable/disable for Linux & Darwin
+if not use_backtrace
+  zstd_c_args += '-DBACKTRACE_ENABLE=0'
+elif use_debug and host_machine_os == os_windows  # MinGW target
+  zstd_c_args += '-DBACKTRACE_ENABLE=1'
+  export_dynamic_on_windows = true
+endif
+
+if cc_id == compiler_msvc
+  if default_library_type != 'static'
+    zstd_programs_sources += [windows_mod.compile_resources(
+      join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
+  endif
+endif
+
+zstd = executable('zstd',
+  zstd_programs_sources,
+  c_args: zstd_c_args,
+  dependencies: zstd_deps,
+  export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
+  install: true)
+
+zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/fileio.c')]
+
+# Minimal target, with only zstd compression and decompression.
+# No bench. No legacy.
+executable('zstd-frugal',
+  zstd_frugal_sources,
+  dependencies: libzstd_dep,
+  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
+  install: true)
+
+install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
+  join_paths(zstd_rootdir, 'programs/zstdless'),
+  install_dir: zstd_bindir)
+
+# =============================================================================
+# Programs and manpages installing
+# =============================================================================
+
+install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
+  join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
+  join_paths(zstd_rootdir, 'programs/zstdless.1'))
+
+InstallSymlink_py = '../InstallSymlink.py'
+zstd_man1_dir = join_paths(zstd_mandir, 'man1')
+bin_EXT = host_machine_os == os_windows ? '.exe' : ''
+man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
+
+foreach f : ['zstdcat', 'unzstd']
+  meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, f + bin_EXT, zstd_bindir)
+  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
+endforeach
+
+if use_multi_thread
+  meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, 'zstdmt' + bin_EXT, zstd_bindir)
+  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
+endif

build/meson/tests/meson.build

@@ -0,0 +1,230 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
+  os_freebsd, 'netbsd', 'dragonfly', os_sun]
+
+# =============================================================================
+# Test flags
+# =============================================================================
+
+FUZZER_FLAGS = ['--no-big-tests']
+FUZZERTEST = '-T200s'
+ZSTREAM_TESTTIME = '-T90s'
+DECODECORPUS_TESTTIME = '-T30'
+ZSTDRTTEST = ['--test-large-data']
+
+# =============================================================================
+# Executables
+# =============================================================================
+
+test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
+
+datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'tests/datagencli.c')]
+datagen = executable('datagen',
+  datagen_sources,
+  c_args: [ '-DNDEBUG' ],
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'tests/fullbench.c')]
+fullbench = executable('fullbench',
+  fullbench_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/fuzzer.c')]
+fuzzer = executable('fuzzer',
+  fuzzer_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+zbufftest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/zbufftest.c')]
+zbufftest = executable('zbufftest',
+  zbufftest_sources,
+  c_args: ['-Wno-deprecated-declarations'],
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/seqgen.c'),
+  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
+zstreamtest = executable('zstreamtest',
+  zstreamtest_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/paramgrill.c')]
+paramgrill = executable('paramgrill',
+  paramgrill_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, libm_dep ],
+  install: false)
+
+roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
+roundTripCrash = executable('roundTripCrash',
+  roundTripCrash_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
+longmatch = executable('longmatch',
+  longmatch_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
+invalidDictionaries = executable('invalidDictionaries',
+  invalidDictionaries_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+if 0 < legacy_level and legacy_level <= 4
+  legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
+  legacy = executable('legacy',
+    legacy_sources,
+    # Use -Dlegacy_level build option to control it
+    #c_args: '-DZSTD_LEGACY_SUPPORT=4',
+    dependencies: [ libzstd_dep ],
+    install: false)
+endif
+
+decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
+decodecorpus = executable('decodecorpus',
+  decodecorpus_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, libm_dep ],
+  install: false)
+
+symbols_sources = [join_paths(zstd_rootdir, 'tests/symbols.c')]
+symbols = executable('symbols',
+  symbols_sources,
+  include_directories: test_includes,
+  c_args: host_machine_os == os_windows ? '-DZSTD_DLL_IMPORT=1' : [],
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/timefn.c'),
+  join_paths(zstd_rootdir, 'tests/poolTests.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/threading.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c')]
+poolTests = executable('poolTests',
+  poolTests_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, thread_dep ],
+  install: false)
+
+checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
+checkTag = executable('checkTag',
+  checkTag_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+# =============================================================================
+# Tests (Use "meson test --list" to list all tests)
+# =============================================================================
+
+if tests_supported_oses.contains(host_machine_os)
+  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
+  valgrindTest_py = files('valgrindTest.py')
+  test('valgrindTest',
+    valgrindTest_py,
+    args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
+    depends: [zstd, datagen, fuzzer, fullbench],
+    timeout: 600) # Timeout should work on HDD drive
+endif
+
+if host_machine_os != os_windows
+  playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
+  test('test-zstd',
+    playTests_sh,
+    args: ZSTDRTTEST,
+    env: ['ZSTD=' + zstd.full_path()],
+    depends: [datagen],
+    workdir: meson.current_build_dir(),
+    timeout: 2800) # Timeout should work on HDD drive
+endif
+
+test('test-fullbench-1',
+  fullbench,
+  args: ['-i1'],
+  depends: [datagen],
+  timeout: 60)
+test('test-fullbench-2',
+  fullbench,
+  args: ['-i1', '-P0'],
+  depends: [datagen],
+  timeout: 60)
+
+if use_zlib
+  test('test-fuzzer',
+    fuzzer,
+    args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
+    timeout: 480)
+endif
+
+test('test-zbuff',
+  zbufftest,
+  args: [ZSTREAM_TESTTIME],
+  timeout: 120)
+test('test-zstream-1',
+  zstreamtest,
+  args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 240)
+test('test-zstream-2',
+  zstreamtest,
+  args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
+test('test-zstream-3',
+  zstreamtest,
+  args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
+test('test-longmatch', longmatch, timeout: 36)
+test('test-invalidDictionaries', invalidDictionaries) # should be fast
+test('test-symbols', symbols) # should be fast
+if 0 < legacy_level and legacy_level <= 4
+  test('test-legacy', legacy) # should be fast
+endif
+test('test-decodecorpus',
+  decodecorpus,
+  args: ['-t', DECODECORPUS_TESTTIME],
+  timeout: 60)
+test('test-poolTests', poolTests) # should be fast

build/meson/tests/valgrindTest.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+import os
+import subprocess
+import tempfile
+
+
+def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
+  VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
+
+  print('\n ---- valgrind tests : memory analyzer ----')
+
+  subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
+
+  if subprocess.call([*VALGRIND_ARGS, zstd],
+                     stdout=subprocess.DEVNULL) == 0:
+    raise subprocess.CalledProcessError('zstd without argument should have failed')
+
+  with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
+       subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
+                        stdin=p1.stdout,
+                        stdout=subprocess.DEVNULL) as p2:
+    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
+    p2.communicate()
+    if p2.returncode != 0:
+      raise subprocess.CalledProcessError()
+
+  with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
+       subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
+                        stdin=p1.stdout,
+                        stdout=subprocess.DEVNULL) as p2:
+    p1.stdout.close()
+    p2.communicate()
+    if p2.returncode != 0:
+      raise subprocess.CalledProcessError()
+
+  with tempfile.NamedTemporaryFile() as tmp_fd:
+    with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
+         subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
+                          stdin=p1.stdout) as p2:
+      p1.stdout.close()
+      p2.communicate()
+      if p2.returncode != 0:
+        raise subprocess.CalledProcessError()
+
+    subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
+                          stdout=subprocess.DEVNULL)
+
+    with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
+         subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
+                          stdin=p1.stdout,
+                          stdout=subprocess.DEVNULL) as p2:
+      p1.stdout.close()
+      p2.communicate()
+      if p2.returncode != 0:
+        raise subprocess.CalledProcessError()
+
+  subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
+  subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
+  parser.add_argument('valgrind', help='valgrind path')
+  parser.add_argument('zstd', help='zstd path')
+  parser.add_argument('datagen', help='datagen path')
+  parser.add_argument('fuzzer', help='fuzzer path')
+  parser.add_argument('fullbench', help='fullbench path')
+
+  args = parser.parse_args()
+
+  valgrind = args.valgrind
+  zstd = args.zstd
+  datagen = args.datagen
+  fuzzer = args.fuzzer
+  fullbench = args.fullbench
+
+  valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
+
+
+if __name__ == '__main__':
+  main()

contrib/adaptive-compression/Makefile

@@ -13,7 +13,7 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 CFLAGS   ?= -O3
 CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wstrict-prototypes -Wundef                                     \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
             -Wredundant-decls
 CFLAGS   += $(DEBUGFLAGS)
@@ -22,10 +22,10 @@ FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
 
 all: adapt datagen
 
-adapt: $(ZSTD_FILES) adapt.c
+adapt: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
 	$(CC) $(FLAGS) $^ -o $@
 
-adapt-debug: $(ZSTD_FILES) adapt.c
+adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
 	$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
 
 datagen : $(PRGDIR)/datagen.c datagencli.c

contrib/adaptive-compression/adapt.c

@@ -13,6 +13,7 @@
 #include <string.h>     /* memset */
 #include "zstd_internal.h"
 #include "util.h"
+#include "timefn.h"     /* UTIL_time_t, UTIL_getTime, UTIL_getSpanTimeMicro */
 
 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
 #define PRINT(...) fprintf(stdout, __VA_ARGS__)
@@ -579,7 +580,7 @@ static void* compressionThread(void* arg)
                 params.cParams.windowLog = 23;
                 {
                     size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);
-                    size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);
+                    size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_forceMaxWindow, 1);
                     if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
                         DISPLAY("Error: something went wrong while starting compression\n");
                         signalErrorToThreads(ctx);

contrib/adaptive-compression/datagencli.c

@@ -120,7 +120,7 @@ int main(int argc, const char** argv)
     DISPLAYLEVEL(4, "Compressible data Generator \n");
     if (probaU32!=COMPRESSIBILITY_DEFAULT)
         DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
-    DISPLAYLEVEL(3, "Seed = %u \n", seed);
+    DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
 
     RDG_genStdout(size, (double)probaU32/100, litProba, seed);
     DISPLAYLEVEL(1, "\n");

contrib/docker/README.md

@@ -5,7 +5,7 @@ The `Dockerfile` script requires a version of `docker` >= 17.05
 
 ## Installing docker
 
-The officiel docker install docs use a ppa with a modern version available:
+The official docker install docs use a ppa with a modern version available:
 https://docs.docker.com/install/linux/docker-ce/ubuntu/
 
 ## How to run

contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c

@@ -127,7 +127,7 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
 
 
 /** compressWithDict() :
- *  Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
+ *  Compress samples from sample buffer given dictionary stored on dictionary buffer and compression level
  *  @return compression ratio
  */
 double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
@@ -194,7 +194,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev
     totalCompressedSize += compressedSize;
   }
 
-  /* Sum orignal sizes */
+  /* Sum original sizes */
   for (i = 0; i<srcInfo->nbSamples; i++) {
     totalOriginalSize += srcInfo->samplesSizes[i];
   }

contrib/experimental_dict_builders/fastCover/fastCover.c

@@ -125,7 +125,7 @@ typedef struct {
  *
  *     Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
  *
- * Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2.
+ * Once the dmer with hash value d is in the dictionary we set F(d) = F(d)/2.
  */
 static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
                                                   U32 *freqs, U32 begin,U32 end,
@@ -149,7 +149,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
     while (activeSegment.end < end) {
       /* Get hash value of current dmer */
       const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
-      /* Add frequency of this index to score if this is the first occurence of index in active segment */
+      /* Add frequency of this index to score if this is the first occurrence of index in active segment */
       if (ctx->segmentFreqs[index] == 0) {
         activeSegment.score += freqs[index];
       }

contrib/largeNbDicts/Makefile

@@ -18,7 +18,7 @@ CFLAGS  ?= -O3
 CFLAGS  += -std=gnu99
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
             -Wredundant-decls
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
@@ -28,22 +28,31 @@ default: largeNbDicts
 
 all : largeNbDicts
 
-largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
+largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 .PHONY: $(LIBZSTD)
 $(LIBZSTD):
-	$(MAKE) -C $(LIBDIR) libzstd.a
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
 
-bench.o  : $(PROGDIR)/bench.c
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
 
 datagen.o: $(PROGDIR)/datagen.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
 
+util.o: $(PROGDIR)/util.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
 xxhash.o : $(LIBDIR)/common/xxhash.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
 
+
 clean:
 	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
 	$(RM) largeNbDicts

contrib/largeNbDicts/largeNbDicts.c

@@ -21,10 +21,11 @@
 #include <stddef.h>   /* size_t */
 #include <stdlib.h>   /* malloc, free, abort */
 #include <stdio.h>    /* fprintf */
+#include <limits.h>   /* UINT_MAX */
 #include <assert.h>   /* assert */
 
 #include "util.h"
-#include "bench.h"
+#include "benchfn.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 #include "zdict.h"
@@ -49,6 +50,7 @@
 
 
 /*---  Macros  ---*/
+
 #define CONTROL(c)   { if (!(c)) abort(); }
 #undef MIN
 #define MIN(a,b)     ((a) < (b) ? (a) : (b))
@@ -126,7 +128,7 @@ static buffer_t createBuffer_fromFile(const char* fileName)
 static buffer_t
 createDictionaryBuffer(const char* dictionaryName,
                        const void* srcBuffer,
-                       const size_t* srcBlockSizes, unsigned nbBlocks,
+                       const size_t* srcBlockSizes, size_t nbBlocks,
                        size_t requestedDictSize)
 {
     if (dictionaryName) {
@@ -140,9 +142,10 @@ createDictionaryBuffer(const char* dictionaryName,
         void* const dictBuffer = malloc(requestedDictSize);
         CONTROL(dictBuffer != NULL);
 
+        assert(nbBlocks <= UINT_MAX);
         size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
                                                       srcBuffer,
-                                                      srcBlockSizes, nbBlocks);
+                                                      srcBlockSizes, (unsigned)nbBlocks);
         CONTROL(!ZSTD_isError(dictSize));
 
         buffer_t result;
@@ -421,7 +424,7 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
 }
 
 
-/* mess with adresses, so that linear scanning dictionaries != linear address scanning */
+/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
 void shuffleDictionaries(ddict_collection_t dicts)
 {
     size_t const nbDicts = dicts.nbDDict;
@@ -537,19 +540,26 @@ static int benchMem(slice_collection_t dstBlocks,
     BMK_timedFnState_t* const benchState =
             BMK_createTimedFnState(total_time_ms, ms_per_round);
     decompressInstructions di = createDecompressInstructions(dictionaries);
+    BMK_benchParams_t const bp = {
+        .benchFn = decompress,
+        .benchPayload = &di,
+        .initFn = NULL,
+        .initPayload = NULL,
+        .errorFn = ZSTD_isError,
+        .blockCount = dstBlocks.nbSlices,
+        .srcBuffers = (const void* const*) srcBlocks.slicePtrs,
+        .srcSizes = srcBlocks.capacities,
+        .dstBuffers = dstBlocks.slicePtrs,
+        .dstCapacities = dstBlocks.capacities,
+        .blockResults = NULL
+    };
 
     for (;;) {
-        BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState,
-                                decompress, &di,
-                                NULL, NULL,
-                                dstBlocks.nbSlices,
-                                (const void* const *)srcBlocks.slicePtrs, srcBlocks.capacities,
-                                dstBlocks.slicePtrs, dstBlocks.capacities,
-                                NULL);
+        BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, bp);
         CONTROL(BMK_isSuccessful_runOutcome(outcome));
 
         BMK_runTime_t const result = BMK_extract_runTime(outcome);
-        U64 const dTime_ns = result.nanoSecPerRun;
+        double const dTime_ns = result.nanoSecPerRun;
         double const dTime_sec = (double)dTime_ns / 1000000000;
         size_t const srcSize = result.sumOfReturn;
         double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB);
@@ -594,6 +604,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
     if (blockSize)
         DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
     DISPLAYLEVEL(3, "\n");
+    size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
 
 
     size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
@@ -625,8 +636,8 @@ int bench(const char** fileNameTable, unsigned nbFiles,
 
     /* dictionary determination */
     buffer_t const dictBuffer = createDictionaryBuffer(dictionary,
-                                srcBuffer.ptr,
-                                srcSlices.capacities, nbBlocks,
+                                srcs.buffer.ptr,
+                                srcs.slices.capacities, srcs.slices.nbSlices,
                                 DICTSIZE);
     CONTROL(dictBuffer.ptr != NULL);
 
@@ -637,7 +648,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
     CONTROL(cTotalSizeNoDict != 0);
     DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f  (%u bytes) \n",
                     clevel,
-                    (double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
+                    (double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
 
     size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
     CONTROL(cSizes != NULL);
@@ -646,7 +657,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
     CONTROL(cTotalSize != 0);
     DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f  (%u bytes) \n",
                     (unsigned)dictBuffer.size,
-                    (double)srcSize / cTotalSize, (unsigned)cTotalSize);
+                    (double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
 
     /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
     shrinkSizes(dstSlices, cSizes);

contrib/linux-kernel/0002-lib-Add-zstd-modules.patch

@@ -4,7 +4,7 @@ Date: Mon, 17 Jul 2017 17:08:19 -0700
 Subject: [PATCH v5 2/5] lib: Add zstd modules
 
 Add zstd compression and decompression kernel modules.
-zstd offers a wide varity of compression speed and quality trade-offs.
+zstd offers a wide variety of compression speed and quality trade-offs.
 It can compress at speeds approaching lz4, and quality approaching lzma.
 zstd decompressions at speeds more than twice as fast as zlib, and
 decompression speed remains roughly the same across all compression levels.
@@ -21,7 +21,7 @@ will be easier to keep the kernel zstd up to date.
 I benchmarked zstd compression as a special character device. I ran zstd
 and zlib compression at several levels, as well as performing no
 compression, which measure the time spent copying the data to kernel space.
-Data is passed to the compresser 4096 B at a time. The benchmark file is
+Data is passed to the compressor 4096 B at a time. The benchmark file is
 located in the upstream zstd source repository under
 `contrib/linux-kernel/zstd_compress_test.c` [2].
 
@@ -86,7 +86,7 @@ Tested in userland using the test-suite in the zstd repo under
 `contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel
 functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under
 `contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8]
-with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the
+with ASAN, UBSAN, and MSAN. Additionally, it was tested while testing the
 BtrFS and SquashFS patches coming next.
 
 [1] https://clang.llvm.org/docs/ClangFormat.html
@@ -4200,14 +4200,14 @@ index 0000000..ff18ae6
 +	BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
 +	U32 const fcsCode =
 +	    params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
-+	BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
++	BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
 +	size_t pos;
 +
 +	if (dstCapacity < ZSTD_frameHeaderSize_max)
 +		return ERROR(dstSize_tooSmall);
 +
 +	ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
-+	op[4] = frameHeaderDecriptionByte;
++	op[4] = frameHeaderDescriptionByte;
 +	pos = 5;
 +	if (!singleSegment)
 +		op[pos++] = windowLogByte;
@@ -8812,8 +8812,8 @@ index 0000000..ef3d174
 +		U32 position = 0;
 +		U32 symbol;
 +		for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
-+			int nbOccurences;
-+			for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
++			int nbOccurrences;
++			for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
 +				tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
 +				position = (position + step) & tableMask;
 +				while (position > highThreshold)
@@ -9944,7 +9944,7 @@ index 0000000..2143da2
 +	HUF_repeat_none,  /**< Cannot use the previous table */
 +	HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
 +			     4}X_repeat */
-+	HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
++	HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
 +} HUF_repeat;
 +/** HUF_compress4X_repeat() :
 +*   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.

contrib/linux-kernel/0006-squashfs-tools-Add-zstd-support.patch

@@ -11,7 +11,7 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
 ---
 v4 -> v5:
 - Fix patch documentation to reflect that Sean Purcell is the author
-- Don't strip trailing whitespace of unreleated code
+- Don't strip trailing whitespace of unrelated code
 - Make zstd_display_options() static
 
 v5 -> v6:
@@ -224,7 +224,7 @@ index 0000000..dcab75a
 + * set the default options, this is to ensure any user supplied
 + * -X options on the appending mksquashfs command line are over-ridden.
 + *
-+ * This function returns 0 on sucessful extraction of options, and -1 on error.
++ * This function returns 0 on successful extraction of options, and -1 on error.
 + */
 +static int zstd_extract_options(int block_size, void *buffer, int size)
 +{

contrib/linux-kernel/lib/zstd/compress.c

@@ -2436,14 +2436,14 @@ static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_paramete
 	BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
 	U32 const fcsCode =
 	    params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
-	BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
+	BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
 	size_t pos;
 
 	if (dstCapacity < ZSTD_frameHeaderSize_max)
 		return ERROR(dstSize_tooSmall);
 
 	ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
-	op[4] = frameHeaderDecriptionByte;
+	op[4] = frameHeaderDescriptionByte;
 	pos = 5;
 	if (!singleSegment)
 		op[pos++] = windowLogByte;

contrib/linux-kernel/lib/zstd/fse_compress.c

@@ -141,8 +141,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsi
 		U32 position = 0;
 		U32 symbol;
 		for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
-			int nbOccurences;
-			for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
+			int nbOccurrences;
+			for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
 				tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
 				position = (position + step) & tableMask;
 				while (position > highThreshold)

contrib/linux-kernel/lib/zstd/huf.h

@@ -134,7 +134,7 @@ typedef enum {
 	HUF_repeat_none,  /**< Cannot use the previous table */
 	HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
 			     4}X_repeat */
-	HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+	HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
 } HUF_repeat;
 /** HUF_compress4X_repeat() :
 *   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.

contrib/linux-kernel/test/include/linux/compiler.h

@@ -1,5 +1,5 @@
-#ifndef LINUX_COMIPLER_H_
-#define LINUX_COMIPLER_H_
+#ifndef LINUX_COMPILER_H_
+#define LINUX_COMPILER_H_
 
 #ifndef __always_inline
 #  define __always_inline inline
@@ -9,4 +9,4 @@
 #  define noinline __attribute__((__noinline__))
 #endif
 
-#endif // LINUX_COMIPLER_H_
+#endif // LINUX_COMPILER_H_

contrib/meson/README

@@ -1,3 +0,0 @@
-This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
-
-It outputs one libzstd, either shared or static, depending on default_library.

contrib/meson/meson.build

@@ -1,144 +0,0 @@
-project('zstd', 'c', license: 'BSD')
-
-libm = meson.get_compiler('c').find_library('m', required: true)
-
-lib_dir = join_paths('..', '..', 'lib')
-common_dir = join_paths(lib_dir, 'common')
-compress_dir = join_paths(lib_dir, 'compress')
-decompress_dir = join_paths(lib_dir, 'decompress')
-dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
-deprecated_dir = join_paths(lib_dir, 'deprecated')
-
-libzstd_srcs = [
-    join_paths(common_dir, 'entropy_common.c'),
-    join_paths(common_dir, 'fse_decompress.c'),
-    join_paths(common_dir, 'threading.c'),
-    join_paths(common_dir, 'pool.c'),
-    join_paths(common_dir, 'zstd_common.c'),
-    join_paths(common_dir, 'error_private.c'),
-    join_paths(common_dir, 'xxhash.c'),
-    join_paths(compress_dir, 'fse_compress.c'),
-    join_paths(compress_dir, 'hist.c'),
-    join_paths(compress_dir, 'huf_compress.c'),
-    join_paths(compress_dir, 'zstd_compress.c'),
-    join_paths(compress_dir, 'zstd_fast.c'),
-    join_paths(compress_dir, 'zstd_double_fast.c'),
-    join_paths(compress_dir, 'zstd_lazy.c'),
-    join_paths(compress_dir, 'zstd_opt.c'),
-    join_paths(compress_dir, 'zstd_ldm.c'),
-    join_paths(compress_dir, 'zstdmt_compress.c'),
-    join_paths(decompress_dir, 'huf_decompress.c'),
-    join_paths(decompress_dir, 'zstd_decompress.c'),
-    join_paths(dictbuilder_dir, 'cover.c'),
-    join_paths(dictbuilder_dir, 'divsufsort.c'),
-    join_paths(dictbuilder_dir, 'zdict.c'),
-    join_paths(deprecated_dir, 'zbuff_common.c'),
-    join_paths(deprecated_dir, 'zbuff_compress.c'),
-    join_paths(deprecated_dir, 'zbuff_decompress.c')
-]
-
-libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
-
-legacy = get_option('legacy_support')
-if legacy == '0'
-    legacy = 'false'
-endif
-if legacy != 'false'
-    if legacy == 'true'
-        legacy = '1'
-    endif
-    #See ZSTD_LEGACY_SUPPORT of programs/README.md
-    message('Enabling legacy support back to version 0.' + legacy)
-    legacy_int = legacy.to_int()
-    if legacy_int > 7
-        legacy_int = 7
-    endif
-    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy]
-
-    legacy_dir = join_paths(lib_dir, 'legacy')
-    libzstd_includes += [include_directories(legacy_dir)]
-    if legacy_int <= 1
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c')
-    endif
-    if legacy_int <= 2
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c')
-    endif
-    if legacy_int <= 3
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c')
-    endif
-    if legacy_int <= 4
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c')
-    endif
-    if legacy_int <= 5
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c')
-    endif
-    if legacy_int <= 6
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c')
-    endif
-    if legacy_int <= 7
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c')
-    endif
-else
-    libzstd_cflags = []
-endif
-
-if get_option('multithread')
-    message('Enabling multi-threading support')
-    add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
-    libzstd_deps = [dependency('threads')]
-else
-    libzstd_deps = []
-endif
-
-libzstd = library('zstd',
-                  libzstd_srcs,
-                  include_directories: libzstd_includes,
-                  c_args: libzstd_cflags,
-                  dependencies: libzstd_deps,
-                  install: true,
-                  soversion: '1',
-                 )
-
-programs_dir = join_paths('..', '..', 'programs')
-
-zstd = executable('zstd',
-                  join_paths(programs_dir, 'bench.c'),
-                  join_paths(programs_dir, 'datagen.c'),
-                  join_paths(programs_dir, 'dibio.c'),
-                  join_paths(programs_dir, 'fileio.c'),
-                  join_paths(programs_dir, 'zstdcli.c'),
-                  include_directories: libzstd_includes,
-                  c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
-                  link_with: libzstd,
-                  install: true)
-
-tests_dir = join_paths('..', '..', 'tests')
-datagen_c = join_paths(programs_dir, 'datagen.c')
-test_includes = libzstd_includes + [include_directories(programs_dir)]
-
-fullbench = executable('fullbench',
-                       datagen_c, join_paths(tests_dir, 'fullbench.c'),
-                       include_directories: test_includes,
-                       link_with: libzstd)
-test('fullbench', fullbench)
-
-fuzzer = executable('fuzzer',
-                    datagen_c, join_paths(tests_dir, 'fuzzer.c'),
-                    include_directories: test_includes,
-                    link_with: libzstd)
-test('fuzzer', fuzzer)
-
-if target_machine.system() != 'windows'
-    paramgrill = executable('paramgrill',
-                            datagen_c, join_paths(tests_dir, 'paramgrill.c'),
-                            join_paths(programs_dir, 'bench.c'),
-                            include_directories: test_includes,
-                            link_with: libzstd,
-                            dependencies: libm)
-    test('paramgrill', paramgrill)
-
-    datagen = executable('datagen',
-                         datagen_c, join_paths(tests_dir, 'datagencli.c'),
-                         include_directories: test_includes,
-                         link_with: libzstd)
-endif

contrib/meson/meson_options.txt

@@ -1,3 +0,0 @@
-option('multithread', type: 'boolean', value: false)
-option('legacy_support', type: 'string', value: '4',
-    description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.')

contrib/pzstd/Makefile

@@ -171,7 +171,7 @@ roundtripcheck: roundtrip check
 	$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
 
 # Build the main binary
-pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
 	$(LD_COMMAND)
 
 # Target that depends on all the tests
@@ -190,13 +190,15 @@ $(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
 	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
 
 # Rules to build the tests
-test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \
+                          $(PROGDIR)/util.o Options.o \
                           Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
 	$(LD_COMMAND)
 
 test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
 test/%Test$(EXT): LIBS += -lgtest -lgtest_main
-test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o  \
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \
+                  $(PROGDIR)/util.o Options.o Pzstd.o \
                   SkippableFrame.o $(ZSTDDIR)/libzstd.a
 	$(LD_COMMAND)
 

contrib/pzstd/Pzstd.cpp

@@ -55,7 +55,7 @@ static std::uint64_t handleOneInput(const Options &options,
                              SharedState& state) {
   auto inputSize = fileSizeOrZero(inputFile);
   // WorkQueue outlives ThreadPool so in the case of error we are certain
-  // we don't accidently try to call push() on it after it is destroyed
+  // we don't accidentally try to call push() on it after it is destroyed
   WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{options.numThreads + 1};
   std::uint64_t bytesRead;
   std::uint64_t bytesWritten;

contrib/pzstd/utils/Range.h

@@ -9,7 +9,7 @@
  
 /**
  * A subset of `folly/Range.h`.
- * All code copied verbatiam modulo formatting
+ * All code copied verbatim modulo formatting
  */
 #pragma once
 

contrib/pzstd/utils/ResourcePool.h

@@ -54,7 +54,7 @@ class ResourcePool {
 
   /**
    * @returns  A unique pointer to a resource.  The resource is null iff
-   *           there are no avaiable resources and `factory()` returns null.
+   *           there are no available resources and `factory()` returns null.
    */
   UniquePtr get() {
     std::lock_guard<std::mutex> lock(mutex_);

contrib/seekable_format/examples/.gitignore

@@ -1,4 +1,5 @@
 seekable_compression
 seekable_decompression
+seekable_decompression_mem
 parallel_processing
 parallel_compression

contrib/seekable_format/examples/Makefile

@@ -24,7 +24,8 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
 
 default: all
 
-all: seekable_compression seekable_decompression parallel_processing
+all: seekable_compression seekable_decompression seekable_decompression_mem \
+	parallel_processing
 
 $(ZSTDLIB):
 	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
@@ -35,6 +36,9 @@ seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
 seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 parallel_processing : parallel_processing.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
 
@@ -44,5 +48,6 @@ parallel_compression : parallel_compression.c $(SEEKABLE_OBJS)
 clean:
 	@rm -f core *.o tmp* result* *.zst \
 		seekable_compression seekable_decompression \
+		seekable_decompression_mem \
 		parallel_processing parallel_compression
 	@echo Cleaning completed

contrib/seekable_format/examples/parallel_processing.c

@@ -148,20 +148,20 @@ static void sumFile_orDie(const char* fname, int nbThreads)
     size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
     if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
 
-    size_t const numFrames = ZSTD_seekable_getNumFrames(seekable);
+    unsigned const numFrames = ZSTD_seekable_getNumFrames(seekable);
     struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job));
 
-    size_t i;
-    for (i = 0; i < numFrames; i++) {
-        jobs[i] = (struct sum_job){ fname, 0, i, 0 };
-        POOL_add(pool, sumFrame, &jobs[i]);
+    unsigned fnb;
+    for (fnb = 0; fnb < numFrames; fnb++) {
+        jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
+        POOL_add(pool, sumFrame, &jobs[fnb]);
     }
 
     unsigned long long total = 0;
 
-    for (i = 0; i < numFrames; i++) {
-        while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
-        total += jobs[i].sum;
+    for (fnb = 0; fnb < numFrames; fnb++) {
+        while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
+        total += jobs[fnb].sum;
     }
 
     printf("Sum: %llu\n", total);

contrib/seekable_format/examples/seekable_decompression_mem.c

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // fprintf, perror, feof
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>      // presumes zstd library is installed
+#include <zstd_errors.h>
+
+#include "zstd_seekable.h"
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+#define MAX_FILE_SIZE (8 * 1024 * 1024)
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(1);
+}
+
+static void* realloc_orDie(void* ptr, size_t size)
+{
+    ptr = realloc(ptr, size);
+    if (ptr) return ptr;
+    /* error */
+    perror("realloc");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+static void fseek_orDie(FILE* file, long int offset, int origin) {
+    if (!fseek(file, offset, origin)) {
+        if (!fflush(file)) return;
+    }
+    /* error */
+    perror("fseek");
+    exit(7);
+}
+
+
+static void decompressFile_orDie(const char* fname, off_t startOffset, off_t endOffset)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = stdout;
+    // Just for demo purposes, assume file is <= MAX_FILE_SIZE
+    void*  const buffIn = malloc_orDie(MAX_FILE_SIZE);
+    size_t const inSize = fread_orDie(buffIn, MAX_FILE_SIZE, fin);
+    size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    ZSTD_seekable* const seekable = ZSTD_seekable_create();
+    if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); }
+
+    size_t const initResult = ZSTD_seekable_initBuff(seekable, buffIn, inSize);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+    while (startOffset < endOffset) {
+        size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
+
+        if (ZSTD_isError(result)) {
+            fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
+                    ZSTD_getErrorName(result));
+            exit(12);
+        }
+        fwrite_orDie(buffOut, result, fout);
+        startOffset += result;
+    }
+
+    ZSTD_seekable_free(seekable);
+    fclose_orDie(fin);
+    fclose_orDie(fout);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=4) {
+        fprintf(stderr, "wrong arguments\n");
+        fprintf(stderr, "usage:\n");
+        fprintf(stderr, "%s FILE START END\n", exeName);
+        return 1;
+    }
+
+    {
+        const char* const inFilename = argv[1];
+        off_t const startOffset = atoll(argv[2]);
+        off_t const endOffset = atoll(argv[3]);
+        decompressFile_orDie(inFilename, startOffset, endOffset);
+    }
+
+    return 0;
+}

contrib/seekable_format/zstdseek_compress.c

@@ -8,6 +8,8 @@
  */
 
 #include <stdlib.h>     /* malloc, free */
+#include <limits.h>     /* UINT_MAX */
+#include <assert.h>
 
 #define XXH_STATIC_LINKING_ONLY
 #define XXH_NAMESPACE ZSTD_
@@ -139,7 +141,7 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
 size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
                                  int compressionLevel,
                                  int checksumFlag,
-                                 U32 maxFrameSize)
+                                 unsigned maxFrameSize)
 {
     zcs->framelog.size = 0;
     zcs->frameCSize = 0;
@@ -167,9 +169,9 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
 }
 
 size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
-                                     unsigned compressedSize,
-                                     unsigned decompressedSize,
-                                     unsigned checksum)
+                              unsigned compressedSize,
+                              unsigned decompressedSize,
+                              unsigned checksum)
 {
     if (fl->size == ZSTD_SEEKABLE_MAXFRAMES)
         return ERROR(frameIndex_tooLarge);
@@ -184,7 +186,8 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
         if (newEntries == NULL) return ERROR(memory_allocation);
 
         fl->entries = newEntries;
-        fl->capacity = newCapacity;
+        assert(newCapacity <= UINT_MAX);
+        fl->capacity = (U32)newCapacity;
     }
 
     fl->entries[fl->size] = (framelogEntry_t){
@@ -268,7 +271,7 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
 static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog* fl)
 {
     size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0);
-    size_t const seekTableLen = ZSTD_skippableHeaderSize +
+    size_t const seekTableLen = ZSTD_SKIPPABLEHEADERSIZE +
                                 sizePerFrame * fl->size +
                                 ZSTD_seekTableFooterSize;
 
@@ -307,32 +310,32 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
     size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl);
 
     CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
-    CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_skippableHeaderSize,
-                           4));
+    assert(seekTableLen <= (size_t)UINT_MAX);
+    CHECK_Z(ZSTD_stwrite32(fl, output, (U32)seekTableLen - ZSTD_SKIPPABLEHEADERSIZE, 4));
 
     while (fl->seekTableIndex < fl->size) {
+        unsigned long long const start = ZSTD_SKIPPABLEHEADERSIZE + sizePerFrame * fl->seekTableIndex;
+        assert(start + 8 <= UINT_MAX);
         CHECK_Z(ZSTD_stwrite32(fl, output,
                                fl->entries[fl->seekTableIndex].cSize,
-                               ZSTD_skippableHeaderSize +
-                                       sizePerFrame * fl->seekTableIndex + 0));
+                               (U32)start + 0));
 
         CHECK_Z(ZSTD_stwrite32(fl, output,
                                fl->entries[fl->seekTableIndex].dSize,
-                               ZSTD_skippableHeaderSize +
-                                       sizePerFrame * fl->seekTableIndex + 4));
+                               (U32)start + 4));
 
         if (fl->checksumFlag) {
             CHECK_Z(ZSTD_stwrite32(
                     fl, output, fl->entries[fl->seekTableIndex].checksum,
-                    ZSTD_skippableHeaderSize +
-                            sizePerFrame * fl->seekTableIndex + 8));
+                    (U32)start + 8));
         }
 
         fl->seekTableIndex++;
     }
 
+    assert(seekTableLen <= UINT_MAX);
     CHECK_Z(ZSTD_stwrite32(fl, output, fl->size,
-                           seekTableLen - ZSTD_seekTableFooterSize));
+                           (U32)seekTableLen - ZSTD_seekTableFooterSize));
 
     if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
     if (fl->seekTablePos < seekTableLen - 4) {
@@ -345,7 +348,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
     }
 
     CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_SEEKABLE_MAGICNUMBER,
-                           seekTableLen - 4));
+                           (U32)seekTableLen - 4));
 
     if (fl->seekTablePos != seekTableLen) return ERROR(GENERIC);
     return 0;

contrib/seekable_format/zstdseek_decompress.c

@@ -54,8 +54,9 @@
 #   define LONG_SEEK fseek
 #endif
 
-#include <stdlib.h> /* malloc, free */
-#include <stdio.h>  /* FILE* */
+#include <stdlib.h>  /* malloc, free */
+#include <stdio.h>   /* FILE* */
+#include <limits.h>  /* UNIT_MAX */
 #include <assert.h>
 
 #define XXH_STATIC_LINKING_ONLY
@@ -105,7 +106,7 @@ typedef struct {
 static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
 {
     buffWrapper_t* buff = (buffWrapper_t*) opaque;
-    if (buff->size + n > buff->pos) return -1;
+    if (buff->pos + n > buff->size) return -1;
     memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n);
     buff->pos += n;
     return 0;
@@ -123,7 +124,7 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
         newOffset = (unsigned long long)buff->pos + offset;
         break;
     case SEEK_END:
-        newOffset = (unsigned long long)buff->size - offset;
+        newOffset = (unsigned long long)buff->size + offset;
         break;
     default:
         assert(0);  /* not possible */
@@ -200,13 +201,14 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
  *  Performs a binary search to find the last frame with a decompressed offset
  *  <= pos
  *  @return : the frame's index */
-U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
+unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
 {
     U32 lo = 0;
-    U32 hi = zs->seekTable.tableLen;
+    U32 hi = (U32)zs->seekTable.tableLen;
+    assert(zs->seekTable.tableLen <= UINT_MAX);
 
     if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) {
-        return zs->seekTable.tableLen;
+        return (U32)zs->seekTable.tableLen;
     }
 
     while (lo + 1 < hi) {
@@ -220,31 +222,32 @@ U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long
     return lo;
 }
 
-U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
+unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
 {
-    return zs->seekTable.tableLen;
+    assert(zs->seekTable.tableLen <= UINT_MAX);
+    return (unsigned)zs->seekTable.tableLen;
 }
 
-unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
+unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
     return zs->seekTable.entries[frameIndex].cOffset;
 }
 
-unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
+unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
     return zs->seekTable.entries[frameIndex].dOffset;
 }
 
-size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
+size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
     return zs->seekTable.entries[frameIndex + 1].cOffset -
            zs->seekTable.entries[frameIndex].cOffset;
 }
 
-size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
+size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
 {
     if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
     return zs->seekTable.entries[frameIndex + 1].dOffset -
@@ -275,7 +278,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
     {   U32 const numFrames = MEM_readLE32(zs->inBuff);
         U32 const sizePerEntry = 8 + (checksumFlag?4:0);
         U32 const tableSize = sizePerEntry * numFrames;
-        U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
+        U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
 
         U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
         {
@@ -290,7 +293,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
         if (MEM_readLE32(zs->inBuff) != (ZSTD_MAGIC_SKIPPABLE_START | 0xE)) {
             return ERROR(prefix_unknown);
         }
-        if (MEM_readLE32(zs->inBuff+4) + ZSTD_skippableHeaderSize != frameSize) {
+        if (MEM_readLE32(zs->inBuff+4) + ZSTD_SKIPPABLEHEADERSIZE != frameSize) {
             return ERROR(prefix_unknown);
         }
 
@@ -444,7 +447,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
     return len;
 }
 
-size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, U32 frameIndex)
+size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex)
 {
     if (frameIndex >= zs->seekTable.tableLen) {
         return ERROR(frameIndex_tooLarge);

doc/README.md

@@ -12,8 +12,8 @@ __`zstd_compression_format.md`__ : This document defines the Zstandard compressi
 Compliant decoders must adhere to this document,
 and compliant encoders must generate data that follows it.
 
-Should you look for ressources to develop your own port of Zstandard algorithm,
-you may find the following ressources useful :
+Should you look for resources to develop your own port of Zstandard algorithm,
+you may find the following resources useful :
 
 __`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
 compliant with the Zstandard compression format.

doc/educational_decoder/Makefile

@@ -7,7 +7,7 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 CFLAGS   ?= -O3
 CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wstrict-prototypes -Wundef                                     \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
             -Wredundant-decls
 CFLAGS   += $(DEBUGFLAGS)

doc/educational_decoder/zstd_decompress.c

@@ -358,7 +358,7 @@ static u32 copy_literals(const size_t seq, istream_t *litstream,
                          ostream_t *const out);
 
 // Given an offset code from a sequence command (either an actual offset value
-// or an index for previous offset), computes the correct offset and udpates
+// or an index for previous offset), computes the correct offset and updates
 // the offset history
 static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist);
 

doc/zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.3.0 (25/09/18)
+0.3.2 (17/07/19)
 
 
 Introduction
@@ -390,9 +390,7 @@ A block can contain any number of bytes (even zero), up to
 -  Window_Size
 -  128 KB
 
-A `Compressed_Block` has the extra restriction that `Block_Size` is always
-strictly less than the decompressed size.
-If this condition cannot be respected,
+If this condition cannot be respected when generating a `Compressed_Block`,
 the block must be sent uncompressed instead (`Raw_Block`).
 
 
@@ -913,11 +911,37 @@ Note that blocks which are not `Compressed_Block` are skipped, they do not contr
 ###### Offset updates rules
 
 The newest offset takes the lead in offset history,
-shifting others back (up to its previous place if it was already present).
+shifting others back by one rank,
+up to the previous rank of the new offset _if it was present in history_.
+
+__Examples__ :
+
+In the common case, when new offset is not part of history :
+`Repeated_Offset3` = `Repeated_Offset2`
+`Repeated_Offset2` = `Repeated_Offset1`
+`Repeated_Offset1` = `NewOffset`
+
+When the new offset _is_ part of history, there may be specific adjustments.
+
+When `NewOffset` == `Repeated_Offset1`, offset history remains actually unmodified.
+
+When `NewOffset` == `Repeated_Offset2`,
+`Repeated_Offset1` and `Repeated_Offset2` ranks are swapped.
+`Repeated_Offset3` is unmodified.
+
+When `NewOffset` == `Repeated_Offset3`,
+there is actually no difference with the common case :
+all offsets are shifted by one rank,
+`NewOffset` (== `Repeated_Offset3`) becomes the new `Repeated_Offset1`.
+
+Also worth mentioning, the specific corner case when `offset_value` == 3,
+and the literal length of the current sequence is zero.
+In which case , `NewOffset` = `Repeated_Offset1` - 1_byte.
+Here also, from an offset history update perspective, it's just a common case :
+`Repeated_Offset3` = `Repeated_Offset2`
+`Repeated_Offset2` = `Repeated_Offset1`
+`Repeated_Offset1` = `NewOffset` ( == `Repeated_Offset1` - 1_byte )
 
-This means that when `Repeated_Offset1` (most recent) is used, history is unmodified.
-When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
-If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one.
 
 
 Skippable Frames
@@ -1629,6 +1653,8 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
+- 0.3.2 : remove additional block size restriction on compressed blocks
+- 0.3.1 : minor clarification regarding offset history update rules
 - 0.3.0 : minor edits to match RFC8478
 - 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz
 - 0.2.8 : clarifications for IETF RFC discuss

examples/.gitignore

@@ -1,6 +1,7 @@
 #build
 simple_compression
 simple_decompression
+multiple_simple_compression
 dictionary_compression
 dictionary_decompression
 streaming_compression

examples/Makefile

@@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a
 default: all
 
 all: simple_compression simple_decompression \
+	multiple_simple_compression\
 	dictionary_compression dictionary_decompression \
 	streaming_compression streaming_decompression \
 	multiple_streaming_compression streaming_memory_usage
@@ -24,33 +25,37 @@ all: simple_compression simple_decompression \
 $(LIB) :
 	$(MAKE) -C ../lib libzstd.a
 
-simple_compression : simple_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+simple_compression : simple_compression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-simple_decompression : simple_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+simple_decompression : simple_decompression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_compression : dictionary_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+multiple_simple_compression : multiple_simple_compression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_decompression : dictionary_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+dictionary_compression : dictionary_compression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_compression : streaming_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+dictionary_decompression : dictionary_decompression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-multiple_streaming_compression : multiple_streaming_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+streaming_compression : streaming_compression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_decompression : streaming_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+multiple_streaming_compression : multiple_streaming_compression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
+
+streaming_decompression : streaming_decompression.c common.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
 streaming_memory_usage : streaming_memory_usage.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.zst \
         simple_compression simple_decompression \
+        multiple_simple_compression \
         dictionary_compression dictionary_decompression \
         streaming_compression streaming_decompression \
         multiple_streaming_compression streaming_memory_usage
@@ -62,6 +67,7 @@ test: all
 	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
+	./multiple_simple_compression *.c
 	./streaming_decompression tmp.zst > /dev/null
 	@echo -- Streaming memory usage
 	./streaming_memory_usage
@@ -71,7 +77,6 @@ test: all
 	@echo -- Edge cases detection
 	! ./streaming_decompression tmp    # invalid input, must fail
 	! ./simple_decompression tmp       # invalid input, must fail
-	! ./simple_decompression tmp.zst   # unknown input size, must fail
 	touch tmpNull                      # create 0-size file
 	./simple_compression tmpNull
 	./simple_decompression tmpNull.zst # 0-size frame : must work

examples/README.md

@@ -11,8 +11,14 @@ Zstandard library : usage examples
   Result remains in memory.
   Introduces usage of : `ZSTD_decompress()`
 
+- [Multiple simple compression](multiple_simple_compression.c) :
+  Compress multiple files (in simple mode) in a single command line.
+  Demonstrates memory preservation technique that
+  minimizes malloc()/free() calls by re-using existing resources.
+  Introduces usage of : `ZSTD_compressCCtx()`
+
 - [Streaming memory usage](streaming_memory_usage.c) :
-  Provides amount of memory used by streaming context
+  Provides amount of memory used by streaming context.
   Introduces usage of : `ZSTD_sizeof_CStream()`
 
 - [Streaming compression](streaming_compression.c) :
@@ -20,7 +26,7 @@ Zstandard library : usage examples
   Introduces usage of : `ZSTD_compressStream()`
 
 - [Multiple Streaming compression](multiple_streaming_compression.c) :
-  Compress multiple files in a single command line.
+  Compress multiple files (in streaming mode) in a single command line.
   Introduces memory usage preservation technique,
   reducing impact of malloc()/free() and memset() by re-using existing resources.
 

examples/common.h

@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This header file has common utility functions used in examples.
+ */
+#ifndef COMMON_H
+#define COMMON_H
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, fopen, etc.
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>
+
+/*
+ * Define the returned error code from utility functions.
+ */
+typedef enum {
+    ERROR_fsize = 1,
+    ERROR_fopen = 2,
+    ERROR_fclose = 3,
+    ERROR_fread = 4,
+    ERROR_fwrite = 5,
+    ERROR_loadFile = 6,
+    ERROR_saveFile = 7,
+    ERROR_malloc = 8,
+    ERROR_largeFile = 9,
+} COMMON_ErrorCode;
+
+/*! CHECK
+ * Check that the condition holds. If it doesn't print a message and die.
+ */
+#define CHECK(cond, ...)                        \
+    do {                                        \
+        if (!(cond)) {                          \
+            fprintf(stderr,                     \
+                    "%s:%d CHECK(%s) failed: ", \
+                    __FILE__,                   \
+                    __LINE__,                   \
+                    #cond);                     \
+            fprintf(stderr, "" __VA_ARGS__);    \
+            fprintf(stderr, "\n");              \
+            exit(1);                            \
+        }                                       \
+    } while (0)
+
+/*! CHECK_ZSTD
+ * Check the zstd error code and die if an error occurred after printing a
+ * message.
+ */
+#define CHECK_ZSTD(fn, ...)                                      \
+    do {                                                         \
+        size_t const err = (fn);                                 \
+        CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
+    } while (0)
+
+/*! fsize_orDie() :
+ * Get the size of a given file path.
+ *
+ * @return The size of a given file path.
+ */
+static size_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) != 0) {
+        /* error */
+        perror(filename);
+        exit(ERROR_fsize);
+    }
+
+    off_t const fileSize = st.st_size;
+    size_t const size = (size_t)fileSize;
+    /* 1. fileSize should be non-negative,
+     * 2. if off_t -> size_t type conversion results in discrepancy,
+     *    the file size is too large for type size_t.
+     */
+    if ((fileSize < 0) || (fileSize != (off_t)size)) {
+        fprintf(stderr, "%s : filesize too large \n", filename);
+        exit(ERROR_largeFile);
+    }
+    return size;
+}
+
+/*! fopen_orDie() :
+ * Open a file using given file path and open option.
+ *
+ * @return If successful this function will return a FILE pointer to an
+ * opened file otherwise it sends an error to stderr and exits.
+ */
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(ERROR_fopen);
+}
+
+/*! fclose_orDie() :
+ * Close an opened file using given FILE pointer.
+ */
+static void fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) { return; };
+    /* error */
+    perror("fclose");
+    exit(ERROR_fclose);
+}
+
+/*! fread_orDie() :
+ *
+ * Read sizeToRead bytes from a given file, storing them at the
+ * location given by buffer.
+ *
+ * @return The number of bytes read.
+ */
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(ERROR_fread);
+}
+
+/*! fwrite_orDie() :
+ *
+ * Write sizeToWrite bytes to a file pointed to by file, obtaining
+ * them from a location given by buffer.
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write data to the given file pointer.
+ *
+ * @return The number of bytes written.
+ */
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(ERROR_fwrite);
+}
+
+/*! malloc_orDie() :
+ * Allocate memory.
+ *
+ * @return If successful this function returns a pointer to allo-
+ * cated memory.  If there is an error, this function will send that
+ * error to stderr and exit.
+ */
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(ERROR_malloc);
+}
+
+/*! loadFile_orDie() :
+ * load file into buffer (memory).
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot read data from the given file path.
+ *
+ * @return If successful this function will load file into buffer and
+ * return file size, otherwise it will printout an error to stderr and exit.
+ */
+static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
+{
+    size_t const fileSize = fsize_orDie(fileName);
+    CHECK(fileSize <= bufferSize, "File too large!");
+
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    size_t const readSize = fread(buffer, 1, fileSize, inFile);
+    if (readSize != (size_t)fileSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fread);
+    }
+    fclose(inFile);  /* can't fail, read only */
+    return fileSize;
+}
+
+/*! mallocAndLoadFile_orDie() :
+ * allocate memory buffer and then load file into it.
+ *
+ * Note: This function will send an error to stderr and exit if memory allocation
+ * fails or it cannot read data from the given file path.
+ *
+ * @return If successful this function will return buffer and bufferSize(=fileSize),
+ * otherwise it will printout an error to stderr and exit.
+ */
+static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
+    size_t const fileSize = fsize_orDie(fileName);
+    *bufferSize = fileSize;
+    void* const buffer = malloc_orDie(*bufferSize);
+    loadFile_orDie(fileName, buffer, *bufferSize);
+    return buffer;
+}
+
+/*! saveFile_orDie() :
+ *
+ * Save buffSize bytes to a given file path, obtaining them from a location pointed
+ * to by buff.
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write to a given file.
+ */
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+{
+    FILE* const oFile = fopen_orDie(fileName, "wb");
+    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
+    if (wSize != (size_t)buffSize) {
+        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fwrite);
+    }
+    if (fclose(oFile)) {
+        perror(fileName);
+        exit(ERROR_fclose);
+    }
+}
+
+#endif

examples/dictionary_compression.c

@@ -7,71 +7,11 @@
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
-
-
-#include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf
-#include <string.h>    // strerror
-#include <errno.h>     // errno
-#include <sys/stat.h>  // stat
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc");
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);
-    *size = buffSize;
-    return buffer;
-}
-
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
-{
-    FILE* const oFile = fopen_orDie(fileName, "wb");
-    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
-    if (wSize != (size_t)buffSize) {
-        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
-        exit(5);
-    }
-    if (fclose(oFile)) {
-        perror(fileName);
-        exit(6);
-    }
-}
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
@@ -79,12 +19,9 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
-    if (!cdict) {
-        fprintf(stderr, "ZSTD_createCDict error \n");
-        exit(7);
-    }
+    CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
     free(dictBuffer);
     return cdict;
 }
@@ -93,17 +30,20 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 
+    /* Compress using the dictionary.
+     * This function writes the dictionary id, and content size into the header.
+     * But, it doesn't use a checksum. You can control these options using the
+     * advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
+     * and ZSTD_compress2().
+     */
     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-    if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
     size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(7);
-    }
+    CHECK_ZSTD(cSize);
 
     saveFile_orDie(oname, cBuff, cSize);
 

examples/dictionary_decompression.c

@@ -9,57 +9,10 @@
  */
 
 
-
-#include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf
-#include <string.h>    // strerror
-#include <errno.h>     // errno
-#include <sys/stat.h>  // stat
-#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include <stdlib.h>    // free
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc");
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);
-    *size = buffSize;
-    return buffer;
-}
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
@@ -67,36 +20,51 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
-    if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
+    CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
     free(dictBuffer);
     return ddict;
 }
 
-
 static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
-    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
-        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
-        exit(5);
-    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
-        fprintf(stderr, "%s : original size unknown \n", fname);
-        exit(6);
-    }
-
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
     void* const rBuff = malloc_orDie((size_t)rSize);
 
+    /* Check that the dictionary ID matches.
+     * If a non-zstd dictionary is used, then both will be zero.
+     * By default zstd always writes the dictionary ID into the frame.
+     * Zstd will check if there is a dictionary ID mismatch as well.
+     */
+    unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
+    unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
+    CHECK(actualDictID == expectedDictID,
+          "DictID mismatch: expected %u got %u",
+          expectedDictID,
+          actualDictID);
+
+    /* Decompress using the dictionary.
+     * If you need to control the decompression parameters, then use the
+     * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
+     * ZSTD_decompressDCtx().
+     */
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
-    if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
     size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
-    if (dSize != rSize) {
-        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
-        exit(7);
-    }
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
 
     /* success */
     printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);

examples/multiple_simple_compression.c

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memcpy, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+typedef struct {
+    void* fBuffer;
+    void* cBuffer;
+    size_t fBufferSize;
+    size_t cBufferSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+/*
+ * allocate memory for buffers big enough to compress all files
+ * as well as memory for output file name (ofn)
+ */
+static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
+{
+    size_t maxFilenameLength=0;
+    size_t maxFileSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const filename = argv[argNb];
+      size_t const filenameLength = strlen(filename);
+      size_t const fileSize = fsize_orDie(filename);
+
+      if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
+      if (fileSize > maxFileSize) maxFileSize = fileSize;
+    }
+
+    resources ress;
+    ress.fBufferSize = maxFileSize;
+    ress.cBufferSize = ZSTD_compressBound(maxFileSize);
+
+    *ofnBufferLen = maxFilenameLength + 5;
+    *ofn = (char*)malloc_orDie(*ofnBufferLen);
+    ress.fBuffer = malloc_orDie(ress.fBufferSize);
+    ress.cBuffer = malloc_orDie(ress.cBufferSize);
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+    return ress;
+}
+
+static void freeResources(resources ress, char *outFilename)
+{
+    free(ress.fBuffer);
+    free(ress.cBuffer);
+    ZSTD_freeCCtx(ress.cctx);   /* never fails */
+    free(outFilename);
+}
+
+/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
+static void compressFile_orDie(resources ress, const char* fname, const char* oname)
+{
+    size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
+
+    /* Compress using the context.
+     * If you need more control over parameters, use the advanced API:
+     * ZSTD_CCtx_setParameter(), and ZSTD_compress2().
+     */
+    size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, ress.cBuffer, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    /* memory allocation for outFilename and resources */
+    char* outFilename;
+    size_t outFilenameBufferLen;
+    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
+
+    /* compress files with shared context, input and output buffers */
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const inFilename = argv[argNb];
+        size_t const inFilenameLen = strlen(inFilename);
+        CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
+        memcpy(outFilename, inFilename, inFilenameLen);
+        memcpy(outFilename+inFilenameLen, ".zst", 5);
+        compressFile_orDie(ress, inFilename, outFilename);
+    }
+
+    /* free memory */
+    freeResources(ress,outFilename);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

examples/multiple_streaming_compression.c

@@ -14,118 +14,84 @@
 *  All structures and buffers will be created only once,
 *  and shared across all compression operations */
 
-#include <stdlib.h>    // malloc, exit
-#include <stdio.h>     // fprintf, perror, feof
-#include <string.h>    // strerror
-#include <errno.h>     // errno
-#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
-
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 typedef struct {
     void* buffIn;
     void* buffOut;
     size_t buffInSize;
     size_t buffOutSize;
-    ZSTD_CStream* cstream;
-} resources ;
+    ZSTD_CCtx* cctx;
+} resources;
 
-static resources createResources_orDie()
+static resources createResources_orDie(int cLevel)
 {
     resources ress;
     ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
     ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
     ress.buffIn = malloc_orDie(ress.buffInSize);
     ress.buffOut= malloc_orDie(ress.buffOutSize);
-    ress.cstream = ZSTD_createCStream();
-    if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any compression parameters you want here.
+     * They will persist for every compression operation.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
     return ress;
 }
 
 static void freeResources(resources ress)
 {
-    ZSTD_freeCStream(ress.cstream);
+    ZSTD_freeCCtx(ress.cctx);
     free(ress.buffIn);
     free(ress.buffOut);
 }
 
-
-static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
+static void compressFile_orDie(resources ress, const char* fname, const char* outName)
 {
+    // Open the input and output files.
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
 
-    size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+    /* Reset the context to a clean state to start a new compression operation.
+     * The parameters are sticky, so we keep the compression level and extra
+     * parameters that we set in createResources_orDie().
+     */
+    CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
+
+    size_t const toRead = ress.buffInSize;
+    size_t read;
+    while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        /* This loop is the same as streaming_compression.c.
+         * See that file for detailed comments.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
 
-    size_t read, toRead = ress.buffInSize;
-    while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { ress.buffIn, read, 0 };
-        while (input.pos < input.size) {
+        int finished;
+        do {
             ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
-            toRead = ZSTD_compressStream(ress.cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
-            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
-            if (toRead > ress.buffInSize) toRead = ress.buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
+            size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
+            CHECK_ZSTD(remaining);
             fwrite_orDie(ress.buffOut, output.pos, fout);
-        }
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
     }
 
-    ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
-    size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output);   /* close frame */
-    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
-    fwrite_orDie(ress.buffOut, output.pos, fout);
-
     fclose_orDie(fout);
     fclose_orDie(fin);
 }
 
-
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];
@@ -137,7 +103,8 @@ int main(int argc, const char** argv)
         return 1;
     }
 
-    resources const ress = createResources_orDie();
+    int const cLevel = 7;
+    resources const ress = createResources_orDie(cLevel);
     void* ofnBuffer = NULL;
     size_t ofnbSize = 0;
 
@@ -154,7 +121,7 @@ int main(int argc, const char** argv)
         memset(ofnBuffer, 0, ofnSize);
         strcat(ofnBuffer, ifn);
         strcat(ofnBuffer, ".zst");
-        compressFile_orDie(ress, ifn, ofnBuffer, 7);
+        compressFile_orDie(ress, ifn, ofnBuffer);
     }
 
     freeResources(ress);

examples/simple_compression.c

@@ -8,91 +8,25 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-
-
-#include <stdlib.h>    // malloc, free, exit
-#include <stdio.h>     // fprintf, perror, fopen, etc.
-#include <string.h>    // strlen, strcat, memset, strerror
-#include <errno.h>     // errno
-#include <sys/stat.h>  // stat
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // strlen, strcat, memset
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror(NULL);
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const fileSize = fsize_orDie(fileName);
-    size_t const buffSize = (size_t)fileSize;
-    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
-        fprintf(stderr, "%s : filesize too large \n", fileName);
-        exit(4);
-    }
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(5);
-    }
-    fclose(inFile);  /* can't fail, read only */
-    *size = buffSize;
-    return buffer;
-}
-
-
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
-{
-    FILE* const oFile = fopen_orDie(fileName, "wb");
-    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
-    if (wSize != (size_t)buffSize) {
-        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
-        exit(6);
-    }
-    if (fclose(oFile)) {
-        perror(fileName);
-        exit(7);
-    }
-}
-
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void compress_orDie(const char* fname, const char* oname)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 
+    /* Compress.
+     * If you are doing many compressions, you may want to reuse the context.
+     * See the multiple_simple_compression.c example.
+     */
     size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
-    if (ZSTD_isError(cSize)) {
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(8);
-    }
+    CHECK_ZSTD(cSize);
 
     saveFile_orDie(oname, cBuff, cSize);
 
@@ -103,7 +37,6 @@ static void compress_orDie(const char* fname, const char* oname)
     free(cBuff);
 }
 
-
 static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);

examples/simple_decompression.c

@@ -8,80 +8,36 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-#include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf
-#include <string.h>    // strerror
-#include <errno.h>     // errno
-#include <sys/stat.h>  // stat
-#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include <stdlib.h>    // free
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
-    if (buff) return buff;
-    /* error */
-    fprintf(stderr, "malloc: %s \n", strerror(errno));
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);   /* can't fail (read only) */
-    *size = buffSize;
-    return buffer;
-}
-
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void decompress(const char* fname)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
-    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
-        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
-        exit(5);
-    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
-        fprintf(stderr,
-                "%s : original size unknown. Use streaming decompression instead.\n", fname);
-        exit(6);
-    }
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
 
     void* const rBuff = malloc_orDie((size_t)rSize);
 
+    /* Decompress.
+     * If you are doing many decompressions, you may want to reuse the context
+     * and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
+     * use ZSTD_DCtx_setParameter().
+     */
     size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
-
-    if (dSize != rSize) {
-        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
-        exit(7);
-    }
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
 
     /* success */
     printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
@@ -90,7 +46,6 @@ static void decompress(const char* fname)
     free(cBuff);
 }
 
-
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];

examples/streaming_compression.c

@@ -9,98 +9,76 @@
  */
 
 
-#include <stdlib.h>    // malloc, free, exit
-#include <stdio.h>     // fprintf, perror, feof, fopen, etc.
-#include <string.h>    // strlen, memset, strcat
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat, strlen
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 
 static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
 {
+    /* Open the input and output files. */
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
-    size_t const buffInSize = ZSTD_CStreamInSize();    /* can always read one full block */
+    /* Create the input and output buffers.
+     * They may be any size, but we recommend using these functions to size them.
+     * Performance will only suffer significantly for very tiny buffers.
+     */
+    size_t const buffInSize = ZSTD_CStreamInSize();
     void*  const buffIn  = malloc_orDie(buffInSize);
-    size_t const buffOutSize = ZSTD_CStreamOutSize();  /* can always flush a full block */
+    size_t const buffOutSize = ZSTD_CStreamOutSize();
     void*  const buffOut = malloc_orDie(buffOutSize);
 
-    ZSTD_CStream* const cstream = ZSTD_createCStream();
-    if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
-    size_t const initResult = ZSTD_initCStream(cstream, cLevel);
-    if (ZSTD_isError(initResult)) {
-        fprintf(stderr, "ZSTD_initCStream() error : %s \n",
-                    ZSTD_getErrorName(initResult));
-        exit(11);
-    }
+    /* Create the context. */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
 
-    size_t read, toRead = buffInSize;
-    while( (read = fread_orDie(buffIn, toRead, fin)) ) {
+    /* Set any parameters you want.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
+
+    /* This loop read from the input file, compresses that entire chunk,
+     * and writes all output produced to the output file.
+     */
+    size_t const toRead = buffInSize;
+    size_t read;
+    while ((read = fread_orDie(buffIn, toRead, fin))) {
+        /* Select the flush mode.
+         * If the read may not be finished (read == toRead) we use
+         * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
+         * Zstd optimizes the case where the first flush mode is ZSTD_e_end,
+         * since it knows it is compressing the entire source in one pass.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+        /* Set the input buffer to what we just read.
+         * We compress until the input buffer is empty, each time flushing the
+         * output.
+         */
         ZSTD_inBuffer input = { buffIn, read, 0 };
-        while (input.pos < input.size) {
+        int finished;
+        do {
+            /* Compress into the output buffer and write all of the output to
+             * the file so we can reuse the buffer next iteration.
+             */
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_compressStream(cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
-            if (ZSTD_isError(toRead)) {
-                fprintf(stderr, "ZSTD_compressStream() error : %s \n",
-                                ZSTD_getErrorName(toRead));
-                exit(12);
-            }
-            if (toRead > buffInSize) toRead = buffInSize;   /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
+            size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+            CHECK_ZSTD(remaining);
             fwrite_orDie(buffOut, output.pos, fout);
-        }
+            /* If we're on the last chunk we're finished when zstd returns 0,
+             * which means its consumed all the input AND finished the frame.
+             * Otherwise, we're finished when we've consumed all the input.
+             */
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
     }
 
-    ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);   /* close frame */
-    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
-    fwrite_orDie(buffOut, output.pos, fout);
-
-    ZSTD_freeCStream(cstream);
+    ZSTD_freeCCtx(cctx);
     fclose_orDie(fout);
     fclose_orDie(fin);
     free(buffIn);

examples/streaming_decompression.c

@@ -9,58 +9,10 @@
  */
 
 
-#include <stdlib.h>    // malloc, exit
-#include <stdio.h>     // fprintf, perror, feof
-#include <string.h>    // strerror
-#include <errno.h>     // errno
+#include <stdio.h>     // fprintf
+#include <stdlib.h>    // free
 #include <zstd.h>      // presumes zstd library is installed
-
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
-
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 static void decompressFile_orDie(const char* fname)
 {
@@ -71,26 +23,40 @@ static void decompressFile_orDie(const char* fname)
     size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
     void*  const buffOut = malloc_orDie(buffOutSize);
 
-    ZSTD_DStream* const dstream = ZSTD_createDStream();
-    if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
 
-    /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
-    *  The following example decompresses only the first frame.
-    *  It is compatible with other provided streaming examples */
-    size_t const initResult = ZSTD_initDStream(dstream);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
-    size_t read, toRead = initResult;
+    /* This loop assumes that the input file is one or more concatenated zstd
+     * streams. This example won't work if there is trailing non-zstd data at
+     * the end, but streaming decompression in general handles this case.
+     * ZSTD_decompressStream() returns 0 exactly when the frame is completed,
+     * and doesn't consume input after the frame.
+     */
+    size_t const toRead = buffInSize;
+    size_t read;
     while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
+        /* Given a valid frame, zstd won't consume the last byte of the frame
+         * until it has flushed all of the decompressed data of the frame.
+         * Therefore, instead of checking if the return code is 0, we can
+         * decompress just check if input.pos < input.size.
+         */
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_decompressStream(dstream, &output , &input);  /* toRead : size of next compressed block */
-            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            /* The return code is zero if the frame is complete, but there may
+             * be multiple frames concatenated together. Zstd will automatically
+             * reset the context when a frame is complete. Still, calling
+             * ZSTD_DCtx_reset() can be useful to reset the context to a clean
+             * state, for instance if the last decompression call returned an
+             * error.
+             */
+            size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+            CHECK_ZSTD(ret);
             fwrite_orDie(buffOut, output.pos, fout);
         }
     }
 
-    ZSTD_freeDStream(dstream);
+    ZSTD_freeDCtx(dctx);
     fclose_orDie(fin);
     fclose_orDie(fout);
     free(buffIn);

examples/streaming_memory_usage.c

@@ -16,9 +16,10 @@
 
 
 /*===   Dependencies   ===*/
-#include <stdio.h>   /* printf */
+#include <stdio.h>     // printf
 #define ZSTD_STATIC_LINKING_ONLY
-#include "zstd.h"
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
 
 
 /*===   functions   ===*/
@@ -61,90 +62,75 @@ int main(int argc, char const *argv[]) {
         char const dataToCompress[INPUT_SIZE] = "abcde";
         char compressedData[COMPRESSED_SIZE];
         char decompressedData[INPUT_SIZE];
-        ZSTD_CStream* const cstream = ZSTD_createCStream();
-        if (cstream==NULL) {
-            printf("Level %i : ZSTD_CStream Memory allocation failure \n", compressionLevel);
-            return 1;
-        }
+        /* the ZSTD_CCtx_params structure is a way to save parameters and use
+         * them across multiple contexts. We use them here so we can call the
+         * function ZSTD_estimateCStreamSize_usingCCtxParams().
+         */
+        ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
+        CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!");
 
-        /* forces compressor to use maximum memory size for given compression level,
-         * by not providing any information on input size */
-        ZSTD_parameters params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
-        if (wLog) { /* special mode : specific wLog */
-            printf("Using custom compression parameter : level 1 + wLog=%u \n", wLog);
-            params = ZSTD_getParams(1 /*compressionLevel*/,
-                                    1 << wLog /*estimatedSrcSize*/,
-                                    0 /*no dictionary*/);
-            size_t const error = ZSTD_initCStream_advanced(cstream, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initCStream_advanced error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        } else {
-            size_t const error = ZSTD_initCStream(cstream, compressionLevel);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initCStream error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        }
+        /* Set the compression level. */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) );
+        /* Set the window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) );
 
+        /* Force the compressor to allocate the maximum memory size for a given
+         * level by not providing the pledged source size, or calling
+         * ZSTD_compressStream2() with ZSTD_e_end.
+         */
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+        CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
         size_t compressedSize;
-        {   ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
+        {
+            ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
             ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
-            size_t const cError = ZSTD_compressStream(cstream, &outBuff, &inBuff);
-            if (ZSTD_isError(cError)) {
-                printf("ZSTD_compressStream error : %s \n", ZSTD_getErrorName(cError));
-                return 1;
-            }
-            size_t const fError = ZSTD_endStream(cstream, &outBuff);
-            if (ZSTD_isError(fError)) {
-                printf("ZSTD_endStream error : %s \n", ZSTD_getErrorName(fError));
-                return 1;
-            }
+            CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) );
+            size_t const remaining = ZSTD_endStream(cctx, &outBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not flushed!");
             compressedSize = outBuff.pos;
         }
 
-        ZSTD_DStream* dstream = ZSTD_createDStream();
-        if (dstream==NULL) {
-            printf("Level %i : ZSTD_DStream Memory allocation failure \n", compressionLevel);
-            return 1;
-        }
-        {   size_t const error = ZSTD_initDStream(dstream);
-            if (ZSTD_isError(error)) {
-                printf("ZSTD_initDStream error : %s \n", ZSTD_getErrorName(error));
-                return 1;
-            }
-        }
-        /* forces decompressor to use maximum memory size, as decompressed size is not known */
+        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+        CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
+        /* Set the maximum allowed window log.
+         * The value 0 means use the default window log, which is equivalent to
+         * not setting it.
+         */
+        CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) );
+        /* forces decompressor to use maximum memory size, since the
+         * decompressed size is not stored in the frame header.
+         */
         {   ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
             ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
-            size_t const dResult = ZSTD_decompressStream(dstream, &outBuff, &inBuff);
-            if (ZSTD_isError(dResult)) {
-                printf("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(dResult));
-                return 1;
-            }
-            if (dResult != 0) {
-                printf("ZSTD_decompressStream error : unfinished decompression \n");
-                return 1;
-            }
-            if (outBuff.pos != sizeof(dataToCompress)) {
-                printf("ZSTD_decompressStream error : incorrect decompression \n");
-                return 1;
-            }
+            size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
+            CHECK_ZSTD(remaining);
+            CHECK(remaining == 0, "Frame not complete!");
+            CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!");
         }
 
-        size_t const cstreamSize = ZSTD_sizeof_CStream(cstream);
-        size_t const cstreamEstimatedSize = wLog ?
-                ZSTD_estimateCStreamSize_usingCParams(params.cParams) :
-                ZSTD_estimateCStreamSize(compressionLevel);
-        size_t const dstreamSize = ZSTD_sizeof_DStream(dstream);
+        size_t const cstreamSize = ZSTD_sizeof_CStream(cctx);
+        size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams);
+        size_t const dstreamSize = ZSTD_sizeof_DStream(dctx);
+        size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize);
 
-        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB \n",
+        CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)",
+                (unsigned)cstreamSize, (unsigned)cstreamEstimatedSize);
+        CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)",
+                (unsigned)dstreamSize, (unsigned)dstreamEstimatedSize);
+
+        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n",
                 compressionLevel,
-                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), (unsigned)(dstreamSize>>10));
+                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10),
+                (unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10));
 
-        ZSTD_freeDStream(dstream);
-        ZSTD_freeCStream(cstream);
+        ZSTD_freeDCtx(dctx);
+        ZSTD_freeCCtx(cctx);
+        ZSTD_freeCCtxParams(cctxParams);
         if (wLog) break;  /* single test */
     }
     return 0;

lib/BUCK

@@ -1,6 +1,7 @@
 cxx_library(
     name='zstd',
     header_namespace='',
+    exported_headers=['zstd.h'],
     visibility=['PUBLIC'],
     deps=[
         ':common',
@@ -17,7 +18,7 @@ cxx_library(
     exported_headers=subdir_glob([
         ('compress', 'zstd*.h'),
     ]),
-    srcs=glob(['compress/zstd*.c']),
+    srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
     deps=[':common'],
 )
 
@@ -40,7 +41,7 @@ cxx_library(
     header_namespace='',
     visibility=['PUBLIC'],
     exported_headers=subdir_glob([
-        ('decprecated', '*.h'),
+        ('deprecated', '*.h'),
     ]),
     srcs=glob(['deprecated/*.c']),
     deps=[':common'],
@@ -118,6 +119,7 @@ cxx_library(
         'decompress/huf_decompress.c',
     ],
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':errors',
@@ -204,9 +206,20 @@ cxx_library(
     ],
 )
 
+cxx_library(
+    name='debug',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'debug.h'),
+    ]),
+    srcs=['common/debug.c'],
+)
+
 cxx_library(
     name='common',
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':cpu',

lib/Makefile

@@ -17,6 +17,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER := $(shell echo $(LIBVER_SCRIPT))
 VERSION?= $(LIBVER)
+CCVER := $(shell $(CC) --version)
 
 CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
 ifeq ($(OS),Windows_NT)   # MinGW assumed
@@ -25,13 +26,18 @@ endif
 CFLAGS  ?= -O3
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wmissing-prototypes
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
-GREP = grep --color=never
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
 
 ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
 ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
@@ -40,11 +46,22 @@ ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
 ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
 ZSTD_FILES := $(ZSTDCOMMON_FILES)
 
+ifeq ($(findstring GCC,$(CCVER)),GCC)
+decompress/zstd_decompress_block.o :	CFLAGS+=-fno-tree-vectorize
+endif
+
 ZSTD_LEGACY_SUPPORT ?= 5
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
 ZSTD_LIB_DEPRECATED ?= 1
+HUF_FORCE_DECOMPRESS_X1 ?= 0
+HUF_FORCE_DECOMPRESS_X2 ?= 0
+ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
+ZSTD_FORCE_DECOMPRESS_LONG ?= 0
+ZSTD_NO_INLINE ?= 0
+ZSTD_STRIP_ERROR_STRINGS ?= 0
+ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
 	ZSTD_LIB_DICTBUILDER = 0
@@ -72,6 +89,34 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
 	ZSTD_FILES += $(ZDICT_FILES)
 endif
 
+ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
+	CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+endif
+
+ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
+	CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
+	CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
+	CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
+endif
+
+ifneq ($(ZSTD_NO_INLINE), 0)
+	CFLAGS += -DZSTD_NO_INLINE
+endif
+
+ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
+	CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+endif
+
+ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
+	CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+endif
+
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
 	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
@@ -116,8 +161,7 @@ ifneq (,$(filter Windows%,$(OS)))
 LIBZSTD = dll\libzstd.dll
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	@$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -shared $^ -o $@
-	dlltool -D $@ -d dll\libzstd.def -l dll\libzstd.lib
+	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
 
 else
 

lib/README.md

@@ -7,13 +7,30 @@ in order to make it easier to select or exclude features.
 
 #### Building
 
-`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
+`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
 including commands variables, staged install, directory variables and standard targets.
 - `make` : generates both static and dynamic libraries
-- `make install` : install libraries in default system directories
+- `make install` : install libraries and headers in target system directories
 
-`libzstd` default scope includes compression, decompression, dictionary building,
-and decoding support for legacy formats >= v0.5.0.
+`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
+and support for decoding legacy formats >= v0.5.0.
+The scope can be reduced on demand (see paragraph _modular build_).
+
+
+#### Multithreading support
+
+Multithreading is disabled by default when building with `make`.
+Enabling multithreading requires 2 conditions :
+- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
+- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
+
+Both conditions are automatically applied when invoking `make lib-mt` target.
+
+When linking a POSIX program with a multithreaded version of `libzstd`,
+note that it's necessary to request the `-pthread` flag during link stage.
+
+Multithreading capabilities are exposed
+via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
 
 
 #### API
@@ -26,63 +43,74 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
 Optional advanced features are exposed via :
 
 - `lib/common/zstd_errors.h` : translates `size_t` function results
-                              into an `ZSTD_ErrorCode`, for accurate error handling.
+                               into a `ZSTD_ErrorCode`, for accurate error handling.
+
 - `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
-                          it unlocks access to advanced experimental API,
-                          exposed in second part of `zstd.h`.
-                          These APIs are not "stable", their definition may change in the future.
-                          As a consequence, it shall ___never be used with dynamic library___ !
+                          it unlocks access to the experimental API,
+                          exposed in the second part of `zstd.h`.
+                          All definitions in the experimental APIs are unstable,
+                          they may still change in the future, or even be removed.
+                          As a consequence, experimental definitions shall ___never be used with dynamic library___ !
                           Only static linking is allowed.
 
 
 #### Modular build
 
-It's possible to compile only a limited set of features.
+It's possible to compile only a limited set of features within `libzstd`.
+The file structure is designed to make this selection manually achievable for any build system :
 
 - Directory `lib/common` is always required, for all variants.
+
 - Compression source code lies in `lib/compress`
+
 - Decompression source code lies in `lib/decompress`
+
 - It's possible to include only `compress` or only `decompress`, they don't depend on each other.
+
 - `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
         The API is exposed in `lib/dictBuilder/zdict.h`.
         This module depends on both `lib/common` and `lib/compress` .
-- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
+
+- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
         This module depends on `lib/common` and `lib/decompress`.
         To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
         Specifying a number limits versions supported to that version onward.
         For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
-        `ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
-        Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
-        It can be changed at build by any other value.
-        Note that any number >= 8 translates into "do __not__ support legacy formats",
-        since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
-        `ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
-        Once enabled, this capability is transparently triggered within decompression functions.
-        It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
-        Each version also provides an additional dedicated set of advanced API.
+        Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
+        By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
+        Decoding supported legacy format is a transparent capability triggered within decompression functions.
+        It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
+        Each version does also provide its own set of advanced API.
         For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
-        Note : `lib/legacy` only supports _decoding_ legacy formats.
-- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
-        and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
-        also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
-        dictBuilder).
 
+- While invoking `make libzstd`, it's possible to define build macros
+        `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
+        and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
+        This will also disable compilation of all dependencies
+        (eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
 
-#### Multithreading support
+- There are some additional build macros that can be used to minify the decoder.
 
-Multithreading is disabled by default when building with `make`.
-Enabling multithreading requires 2 conditions :
-- set macro `ZSTD_MULTITHREAD`
-- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
+  Zstandard often has more than one implementation of a piece of functionality,
+  where each implementation optimizes for different scenarios. For example, the
+  Huffman decoder has complementary implementations that decode the stream one
+  symbol at a time or two symbols at a time. Zstd normally includes both (and
+  dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
+  or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
+  compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
+  and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
+  only one or the other of two decompression implementations. The smallest
+  binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
+  `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
 
-Both conditions are automatically triggered by invoking `make lib-mt` target.
-Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
-it's necessary to trigger `-pthread` flag during link stage.
+  For squeezing the last ounce of size out, you can also define
+  `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
+  which removes the error messages that are otherwise returned by
+  `ZSTD_getErrorName`.
 
-Multithreading capabilities are exposed
-via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
-This API is still considered experimental,
-but is expected to become "stable" at some point in the future.
+- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1`
+  will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
+  the shared library, which is now hidden by default.
 
 
 #### Windows : using MinGW+MSYS to create DLL
@@ -113,8 +141,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`.
 
 The other files are not source code. There are :
 
- - `LICENSE` : contains the BSD license text
- - `Makefile` : `make` script to build and install zstd library (static and dynamic)
  - `BUCK` : support for `buck` build system (https://buckbuild.com/)
- - `libzstd.pc.in` : for `pkg-config` (used in `make install`)
+ - `Makefile` : `make` script to build and install zstd library (static and dynamic)
  - `README.md` : this file
+ - `dll/` : resources directory for Windows compilation
+ - `libzstd.pc.in` : script for `pkg-config` (used in `make install`)

lib/common/bitstream.h

@@ -57,6 +57,8 @@ extern "C" {
 =========================================*/
 #if defined(__BMI__) && defined(__GNUC__)
 #  include <immintrin.h>   /* support for bextr (experimental) */
+#elif defined(__ICCARM__)
+#  include <intrinsics.h>
 #endif
 
 #define STREAM_ACCUMULATOR_MIN_32  25
@@ -163,6 +165,8 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
         return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
         return 31 - __builtin_clz (val);
+#   elif defined(__ICCARM__)    /* IAR Intrinsic */
+        return 31 - __CLZ(val);
 #   else   /* Software version */
         static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
                                                  11, 14, 16, 18, 22, 25,  3, 30,
@@ -389,7 +393,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
-MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
@@ -398,7 +402,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works only if nbBits >= 1 */
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);

lib/common/compiler.h

@@ -15,13 +15,15 @@
 *  Compiler specifics
 *********************************************************/
 /* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
 #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
 #else
 #  define INLINE_KEYWORD
 #endif
 
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define FORCE_INLINE_ATTR __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #  define FORCE_INLINE_ATTR __forceinline
@@ -29,9 +31,16 @@
 #  define FORCE_INLINE_ATTR
 #endif
 
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
 /**
  * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
- * parameters. They must be inlined for the compiler to elimininate the constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
  * branches.
  */
 #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
@@ -56,7 +65,7 @@
 #ifdef _MSC_VER
 #  define FORCE_NOINLINE static __declspec(noinline)
 #else
-#  ifdef __GNUC__
+#  if defined(__GNUC__) || defined(__ICCARM__)
 #    define FORCE_NOINLINE static __attribute__((__noinline__))
 #  else
 #    define FORCE_NOINLINE static
@@ -67,7 +76,7 @@
 #ifndef __has_attribute
   #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
 #endif
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
 #else
 #  define TARGET_ATTRIBUTE(target)
@@ -89,23 +98,21 @@
 #endif
 
 /* prefetch
- * can be disabled, by declaring NO_PREFETCH macro
- * All prefetch invocations use a single default locality 2,
- * generating instruction prefetcht1,
- * which, according to Intel, means "load data into L2 cache".
- * This is a good enough "middle ground" for the time being,
- * though in theory, it would be better to specialize locality depending on data being prefetched.
- * Tests could not determine any sensible difference based on locality value. */
+ * can be disabled, by declaring NO_PREFETCH build macro */
 #if defined(NO_PREFETCH)
-#  define PREFETCH(ptr)     (void)(ptr)  /* disabled */
+#  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
+#  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
 #else
 #  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
-#    define PREFETCH(ptr)   _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
 #  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
-#    define PREFETCH(ptr)   __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
 #  else
-#    define PREFETCH(ptr)   (void)(ptr)  /* disabled */
+#    define PREFETCH_L1(ptr) (void)(ptr)  /* disabled */
+#    define PREFETCH_L2(ptr) (void)(ptr)  /* disabled */
 #  endif
 #endif  /* NO_PREFETCH */
 
@@ -116,10 +123,17 @@
     size_t const _size = (size_t)(s);     \
     size_t _pos;                          \
     for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) {  \
-        PREFETCH(_ptr + _pos);            \
+        PREFETCH_L2(_ptr + _pos);         \
     }                                     \
 }
 
+/* vectorization */
+#if !defined(__clang__) && defined(__GNUC__)
+#  define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#else
+#  define DONT_VECTORIZE
+#endif
+
 /* disable warnings */
 #ifdef _MSC_VER    /* Visual Studio */
 #  include <intrin.h>                    /* For Visual 2005 */

lib/common/cpu.h

@@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
       __asm__(
           "pushl %%ebx\n\t"
           "cpuid\n\t"
-          "movl %%ebx, %%eax\n\r"
+          "movl %%ebx, %%eax\n\t"
           "popl %%ebx"
           : "=a"(f7b), "=c"(f7c)
           : "a"(7), "c"(0)

lib/common/debug.h

@@ -57,9 +57,9 @@ extern "C" {
 #endif
 
 
-/* static assert is triggered at compile time, leaving no runtime artefact,
- * but can only work with compile-time constants.
- * This variant can only be used inside a function. */
+/* static assert is triggered at compile time, leaving no runtime artefact.
+ * static assert only works with compile-time constants.
+ * Also, this variant can only be used inside a function. */
 #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
 
 
@@ -70,9 +70,19 @@ extern "C" {
 #  define DEBUGLEVEL 0
 #endif
 
+
+/* DEBUGFILE can be defined externally,
+ * typically through compiler command line.
+ * note : currently useless.
+ * Value must be stderr or stdout */
+#ifndef DEBUGFILE
+#  define DEBUGFILE stderr
+#endif
+
+
 /* recommended values for DEBUGLEVEL :
- * 0 : no debug, all run-time functions disabled
- * 1 : no display, enables assert() only
+ * 0 : release mode, no debug, all run-time checks disabled
+ * 1 : enables assert() only, no display
  * 2 : reserved, for currently active debug path
  * 3 : events once per object lifetime (CCtx, CDict, etc.)
  * 4 : events once per frame
@@ -81,7 +91,7 @@ extern "C" {
  * 7+: events at every position (*very* verbose)
  *
  * It's generally inconvenient to output traces > 5.
- * In which case, it's possible to selectively enable higher verbosity levels
+ * In which case, it's possible to selectively trigger high verbosity levels
  * by modifying g_debug_level.
  */
 
@@ -95,11 +105,12 @@ extern "C" {
 
 #if (DEBUGLEVEL>=2)
 #  include <stdio.h>
-extern int g_debuglevel; /* here, this variable is only declared,
-                           it actually lives in debug.c,
-                           and is shared by the whole process.
-                           It's typically used to enable very verbose levels
-                           on selective conditions (such as position in src) */
+extern int g_debuglevel; /* the variable is only declared,
+                            it actually lives in debug.c,
+                            and is shared by the whole process.
+                            It's not thread-safe.
+                            It's useful when enabling very verbose levels
+                            on selective conditions (such as position in src) */
 
 #  define RAWLOG(l, ...) {                                      \
                 if (l<=g_debuglevel) {                          \

lib/common/error_private.c

@@ -14,6 +14,10 @@
 
 const char* ERR_getErrorString(ERR_enum code)
 {
+#ifdef ZSTD_STRIP_ERROR_STRINGS
+    (void)code;
+    return "Error strings stripped";
+#else
     static const char* const notErrorCode = "Unspecified error code";
     switch( code )
     {
@@ -39,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
+    case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
         /* following error codes are not stable and may be removed or changed in a future version */
     case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
+#endif
 }