Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yann Collet
|
cbc5225d38 | ||
|
Yann Collet
|
817995bb7a | ||
|
Yann Collet
|
99b6d9f1a3 | ||
|
Yann Collet
|
b9edd2f9fb |
+3
-3
@@ -1,8 +1,8 @@
|
||||
[cxx]
|
||||
cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
|
||||
cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
|
||||
cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
|
||||
cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
|
||||
cxxflags = -std=c++11 -Wno-deprecated-declarations
|
||||
cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
|
||||
cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations
|
||||
gtest_dep = //contrib/pzstd:gtest
|
||||
|
||||
[httpserver]
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
version: 2
|
||||
|
||||
jobs:
|
||||
# the first half of the jobs are in this test
|
||||
short-tests-0:
|
||||
# TODO: Create a small custom docker image with all the dependencies we need
|
||||
# preinstalled to reduce installation time.
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
./tests/test-license.py
|
||||
cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
|
||||
make c99build ; make clean
|
||||
make c11build ; make clean
|
||||
make aarch64build ; make clean
|
||||
make -j regressiontest; make clean
|
||||
make shortest ; make clean
|
||||
make cxxtest ; make clean
|
||||
# the second half of the jobs are in this test
|
||||
short-tests-1:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
make gnu90build; make clean
|
||||
make gnu99build; make clean
|
||||
make ppc64build; make clean
|
||||
make ppcbuild ; make clean
|
||||
make armbuild ; make clean
|
||||
make -C tests test-legacy test-longmatch; make clean
|
||||
make -C lib libzstd-nomt; make clean
|
||||
# This step is only run on release tags.
|
||||
# It publishes the source tarball as artifacts and if the GITHUB_TOKEN
|
||||
# environment variable is set it will publish the source tarball to the
|
||||
# tagged release.
|
||||
publish-github-release:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
environment:
|
||||
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Publish
|
||||
command: |
|
||||
export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
|
||||
export ZSTD_VERSION=zstd-$VERSION
|
||||
git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
|
||||
-o $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar > $ZSTD_VERSION.tar.sha256
|
||||
zstd -19 $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
|
||||
gzip -k -9 $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
|
||||
mkdir -p $CIRCLE_ARTIFACTS
|
||||
cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS
|
||||
- store_artifacts:
|
||||
path: /tmp/circleci-artifacts
|
||||
# This step should only be run in a cron job
|
||||
regression-test:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
environment:
|
||||
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||
steps:
|
||||
- checkout
|
||||
# Restore the cached resources.
|
||||
- restore_cache:
|
||||
# We try our best to bust the cache when the data changes by hashing
|
||||
# data.c. If that doesn't work, simply update the version number here
|
||||
# and below. If we fail to bust the cache, the regression testing will
|
||||
# still work, since it has its own stamp, but will need to redownload
|
||||
# everything.
|
||||
keys:
|
||||
- regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
- run:
|
||||
name: Regression Test
|
||||
command: |
|
||||
make -C programs zstd
|
||||
make -C tests/regression test
|
||||
mkdir -p $CIRCLE_ARTIFACTS
|
||||
./tests/regression/test \
|
||||
--cache tests/regression/cache \
|
||||
--output $CIRCLE_ARTIFACTS/results.csv \
|
||||
--zstd programs/zstd
|
||||
echo "NOTE: The new results.csv is uploaded as an artifact to this job"
|
||||
echo " If this fails, go to the Artifacts pane in CircleCI, "
|
||||
echo " download /tmp/circleci-artifacts/results.csv, and if they "
|
||||
echo " are still good, copy it into the repo and commit it."
|
||||
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
|
||||
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
|
||||
# Only save the cache on success (default), since if the failure happened
|
||||
# before we stamp the data cache, we will have a bad cache for this key.
|
||||
- save_cache:
|
||||
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
paths:
|
||||
- tests/regression/cache
|
||||
- store_artifacts:
|
||||
path: /tmp/circleci-artifacts
|
||||
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
commit:
|
||||
jobs:
|
||||
# Run the tests in parallel
|
||||
- short-tests-0:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
- short-tests-1:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
# Create a branch called regression and set it to dev to force a
|
||||
# regression test run
|
||||
- regression-test:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- regression
|
||||
# Only run on release tags.
|
||||
- publish-github-release:
|
||||
requires:
|
||||
- short-tests-0
|
||||
- short-tests-1
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
nightly:
|
||||
triggers:
|
||||
- schedule:
|
||||
cron: "0 0 * * *"
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- dev
|
||||
jobs:
|
||||
# Run daily long regression tests
|
||||
- regression-test
|
||||
|
||||
|
||||
|
||||
# Longer tests
|
||||
#- make -C tests test-zstd-nolegacy && make clean
|
||||
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
||||
#- make zlibwrapper && make clean
|
||||
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
||||
#- make uasan && make clean
|
||||
#- make asan32 && make clean
|
||||
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
||||
# Valgrind tests
|
||||
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
||||
#- make -C tests valgrindTest && make clean
|
||||
# ARM, AArch64, PowerPC, PowerPC64 tests
|
||||
#- make ppctest && make clean
|
||||
#- make ppc64test && make clean
|
||||
#- make armtest && make clean
|
||||
#- make aarch64test && make clean
|
||||
@@ -1,9 +0,0 @@
|
||||
FROM circleci/buildpack-deps:bionic
|
||||
|
||||
RUN sudo dpkg --add-architecture i386
|
||||
RUN sudo apt-get -y -qq update
|
||||
RUN sudo apt-get -y install \
|
||||
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
|
||||
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
|
||||
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
|
||||
libcurl4-openssl-dev
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
task:
|
||||
name: FreeBSD (shortest)
|
||||
freebsd_instance:
|
||||
matrix:
|
||||
image_family: freebsd-12-1
|
||||
# The stable 11.3 image causes "Agent is not responding" so use a snapshot
|
||||
image_family: freebsd-11-3-snap
|
||||
install_script: pkg install -y gmake coreutils
|
||||
script: |
|
||||
MOREFLAGS="-Werror" gmake -j all
|
||||
gmake shortest
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Downloads data '...'
|
||||
2. Run '...' with flags '...'
|
||||
3. Scroll up on the log to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots and charts**
|
||||
If applicable, add screenshots and charts to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. Mac]
|
||||
- Version [e.g. 22]
|
||||
- Compiler [e.g gcc]
|
||||
- Flags [e.g O2]
|
||||
- Other relevant hardware specs [e.g. Dual-core]
|
||||
- Build system [e.g. Makefile]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -1,184 +0,0 @@
|
||||
name: generic-dev
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ dev, master, actionsTest ]
|
||||
|
||||
jobs:
|
||||
|
||||
# Dev PR jobs that still have to be migrated from travis
|
||||
#
|
||||
# icc (need self-hosted)
|
||||
# versionTag
|
||||
# valgrindTest (keeps failing for some reason. need investigation)
|
||||
# staticAnalyze (need trusty so need self-hosted)
|
||||
# pcc-fuzz: (need trusty so need self-hosted)
|
||||
# min-decomp-macros (flakey)
|
||||
#
|
||||
# setting up self-hosted is pretty straightforward, but
|
||||
# I need admins permissions to the repo for that it looks like
|
||||
# So I'm tabling that for now
|
||||
#
|
||||
# The master branch exclusive jobs will be in a separate
|
||||
# workflow file (the osx tests and meson build that is)
|
||||
|
||||
benchmarking:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make benchmarking
|
||||
run: make benchmarking
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make test
|
||||
run: make test
|
||||
|
||||
gcc-6-7-libzstd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + gcc-7 + libzstdmt compilation
|
||||
run: |
|
||||
make gcc6install gcc7install
|
||||
CC=gcc-6 CFLAGS=-Werror make -j all
|
||||
make clean
|
||||
CC=gcc-7 CFLAGS=-Werror make -j all
|
||||
make clean
|
||||
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||
make -C tests zbufftest-dll
|
||||
|
||||
gcc-8-asan-ubsan-testzstd:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Test Zstd
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 CFLAGS="-Werror" make -j all
|
||||
make clean
|
||||
CC=gcc-8 make -j uasan-test-zstd </dev/null
|
||||
|
||||
gcc-6-asan-ubsan-testzstd-32bit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
|
||||
run: |
|
||||
make gcc6install libc6install
|
||||
CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
|
||||
make clean
|
||||
CC=gcc-6 make -j uasan-test-zstd32
|
||||
|
||||
clang-38-msan-testzstd:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: clang-3.8 + MSan + Test Zstd
|
||||
run: |
|
||||
# make clang38install (doesn't work)
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||
|
||||
cmake-build-and-test-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: cmake build and test check
|
||||
run: make cmakebuild
|
||||
|
||||
gcc-8-asan-ubsan-fuzz:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Fuzz Test
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 make clean uasan-fuzztest
|
||||
|
||||
gcc-6-asan-ubsan-fuzz32:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit
|
||||
run: |
|
||||
make gcc6install libc6install
|
||||
CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest
|
||||
|
||||
clang-38-msan-fuzz:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: clang-3.8 + MSan + Fuzz Test
|
||||
run: |
|
||||
# make clang38install (doesn't work)
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make clean msan-fuzztest
|
||||
|
||||
asan-ubsan-msan-regression:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ASan + UBSan + MSan + Regression Test
|
||||
run: |
|
||||
make -j uasanregressiontest
|
||||
make clean
|
||||
make -j msanregressiontest
|
||||
|
||||
cpp-gnu90-c99-compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: C++, gnu90 and c99 compatibility
|
||||
run: |
|
||||
make cxxtest
|
||||
make clean
|
||||
make gnu90build
|
||||
make clean
|
||||
make c99build
|
||||
make clean
|
||||
make travis-install
|
||||
|
||||
mingw-cross-compilation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: mingw cross-compilation
|
||||
run: |
|
||||
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
|
||||
sudo apt-get install gcc-mingw-w64
|
||||
CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
||||
|
||||
armbuild:
|
||||
runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ARM Build Test
|
||||
run: |
|
||||
make arminstall
|
||||
make armbuild
|
||||
|
||||
armfuzz:
|
||||
runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Qemu ARM emulation + Fuzz Test
|
||||
run: |
|
||||
make arminstall
|
||||
make armfuzz
|
||||
|
||||
bourne-shell:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Bourne shell compatibility (shellcheck)
|
||||
run: |
|
||||
wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
|
||||
@@ -1,68 +0,0 @@
|
||||
name: generic-release
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# This will eventually only be for pushes to master
|
||||
# but for dogfooding purposes, I'm running it even
|
||||
# on dev pushes
|
||||
branches: [ dev, master, actionsTest ]
|
||||
|
||||
jobs:
|
||||
# missing jobs
|
||||
#
|
||||
# ppc64le + fuzz test
|
||||
# Qemu PPC64 + Fuzz test
|
||||
# Qemu aarch64 + Fuzz Test (on Xenial)
|
||||
# versions comp
|
||||
# meson test
|
||||
|
||||
osx:
|
||||
runs-on: macos-10.15
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: OS-X
|
||||
run: |
|
||||
make test
|
||||
# make -c lib all (need to fix. not working right now)
|
||||
|
||||
zbuff:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zbuff test
|
||||
run: |
|
||||
make -C tests test-zbuff
|
||||
|
||||
tsan:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: thread sanitizer
|
||||
run: |
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make tsan-test-zstream
|
||||
CC=clang-3.8 make tsan-fuzztest
|
||||
zlib-wrapper:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zlib wrapper test
|
||||
run: |
|
||||
make gpp6install valgrindinstall
|
||||
make -C zlibWrapper test
|
||||
make -C zlibWrapper valgrindTest
|
||||
|
||||
lz4-threadpool-partial-libs:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: LZ4, thread pool, and partial libs testslib wrapper test
|
||||
run: |
|
||||
make lz4install
|
||||
make -C tests test-lz4
|
||||
make clean
|
||||
make -C tests test-pool
|
||||
make clean
|
||||
bash tests/libzstd_partial_builds.sh
|
||||
@@ -1,30 +0,0 @@
|
||||
name: CIFuzz
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
Fuzzing:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
sanitizer: [address, undefined, memory]
|
||||
steps:
|
||||
- name: Build Fuzzers (${{ matrix.sanitizer }})
|
||||
id: build
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Run Fuzzers (${{ matrix.sanitizer }})
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
fuzz-seconds: 600
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Upload Crash
|
||||
uses: actions/upload-artifact@v1
|
||||
if: failure() && steps.build.outcome == 'success'
|
||||
with:
|
||||
name: ${{ matrix.sanitizer }}-artifacts
|
||||
path: ./out/artifacts
|
||||
+7
-17
@@ -14,7 +14,7 @@
|
||||
*.dylib
|
||||
|
||||
# Executables
|
||||
/zstd
|
||||
zstd
|
||||
zstdmt
|
||||
*.exe
|
||||
*.out
|
||||
@@ -22,18 +22,7 @@ zstdmt
|
||||
|
||||
# Test artefacts
|
||||
tmp*
|
||||
*.zst
|
||||
dictionary.
|
||||
dictionary
|
||||
NUL
|
||||
|
||||
# Build artefacts
|
||||
projects/
|
||||
bin/
|
||||
.buckd/
|
||||
buck-out/
|
||||
build-*
|
||||
*.gcda
|
||||
dictionary*
|
||||
|
||||
# Other files
|
||||
.directory
|
||||
@@ -45,7 +34,8 @@ _zstdbench/
|
||||
.DS_Store
|
||||
googletest/
|
||||
*.d
|
||||
*.vscode
|
||||
*.code-workspace
|
||||
compile_commands.json
|
||||
.clangd
|
||||
|
||||
# Directories
|
||||
bin/
|
||||
.buckd/
|
||||
buck-out/
|
||||
|
||||
+157
-262
@@ -1,280 +1,175 @@
|
||||
# Medium Tests: Run on all commits/PRs to dev branch
|
||||
|
||||
language: c
|
||||
|
||||
git:
|
||||
depth: 1
|
||||
|
||||
branches:
|
||||
only:
|
||||
- dev
|
||||
- master
|
||||
- travisTest
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
|
||||
env:
|
||||
global:
|
||||
- FUZZERTEST=-T2mn
|
||||
ZSTREAM_TESTTIME=-T2mn
|
||||
DECODECORPUS_TESTTIME=-T1mn
|
||||
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- name: icc
|
||||
compiler: icc
|
||||
env:
|
||||
- C_COMPILER=icc
|
||||
- CXX_COMPILER=icpc
|
||||
# OS X Mavericks
|
||||
- env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
|
||||
os: osx
|
||||
|
||||
|
||||
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
|
||||
- env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
|
||||
os: linux
|
||||
sudo: false
|
||||
language: cpp
|
||||
install:
|
||||
- source /opt/intel/inteloneapi/compiler/latest/env/vars.sh
|
||||
- export CXX="g++-4.8" CC="gcc-4.8"
|
||||
- export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- sourceline: 'deb https://apt.repos.intel.com/oneapi all main'
|
||||
key_url: 'https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- intel-oneapi-icc
|
||||
script: make check
|
||||
- gcc-4.8
|
||||
- g++-4.8
|
||||
|
||||
- name: arm64 # ~2.5 mn
|
||||
- env: Ubu=12.04cont Cmd="make usan"
|
||||
os: linux
|
||||
arch: arm64
|
||||
script:
|
||||
- make check
|
||||
sudo: false
|
||||
|
||||
- name: make benchmarking
|
||||
script:
|
||||
- make benchmarking
|
||||
- env: Ubu=12.04cont Cmd="make asan"
|
||||
os: linux
|
||||
sudo: false
|
||||
|
||||
- name: make test (complete)
|
||||
script:
|
||||
# DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
|
||||
- DEVNULLRIGHTS=test make test
|
||||
|
||||
- name: gcc-6 + gcc-7 + libzstdmt compilation # ~ 6mn
|
||||
script:
|
||||
- make gcc6install gcc7install
|
||||
- CC=gcc-6 CFLAGS=-Werror make -j all
|
||||
- make clean
|
||||
- CC=gcc-7 CFLAGS=-Werror make -j all
|
||||
- make clean
|
||||
- LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||
- make -C tests zbufftest-dll
|
||||
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
|
||||
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
|
||||
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
|
||||
- env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest"
|
||||
os: linux
|
||||
sudo: required
|
||||
|
||||
- name: gcc-8 + ASan + UBSan + Test Zstd # ~6.5mn
|
||||
script:
|
||||
- make gcc8install
|
||||
- CC=gcc-8 CFLAGS="-Werror" make -j all
|
||||
- make clean
|
||||
- CC=gcc-8 make -j uasan-test-zstd </dev/null # test when stdin is not a tty
|
||||
- env: Ubu=12.04 Cmd="make asan32"
|
||||
os: linux
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- libc6-dev-i386
|
||||
- gcc-multilib
|
||||
|
||||
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode # ~4mn
|
||||
script:
|
||||
- make gcc6install libc6install
|
||||
- CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
|
||||
- make clean
|
||||
- CC=gcc-6 make -j uasan-test-zstd32 # note : can complain about pointer overflow
|
||||
|
||||
- name: clang-3.8 + MSan + Test Zstd # ~3.5mn
|
||||
script:
|
||||
- make clang38install
|
||||
# External libraries must be turned off when using MSAN tests,
|
||||
# because they are not msan-instrumented,
|
||||
# so any data coming from these libraries is always considered "uninitialized"
|
||||
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||
|
||||
- name: Minimal Decompressor Macros # ~5mn
|
||||
script:
|
||||
- make clean
|
||||
- make -j all check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||
|
||||
- name: cmake build and test check # ~6mn
|
||||
script:
|
||||
- make cmakebuild
|
||||
|
||||
- name: static analyzer scanbuild # ~26mn
|
||||
dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives
|
||||
script:
|
||||
- make staticAnalyze
|
||||
|
||||
- name: gcc-8 + ASan + UBSan + Fuzz Test # ~19mn
|
||||
script:
|
||||
- make gcc8install
|
||||
- CC=gcc-8 make clean uasan-fuzztest
|
||||
|
||||
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit # ~15.5mn
|
||||
script:
|
||||
- make gcc6install libc6install
|
||||
- CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest # can complain about pointer overflow
|
||||
|
||||
- name: clang-3.8 + MSan + Fuzz Test # ~14.5mn
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make clean msan-fuzztest
|
||||
|
||||
- name: ASan + UBSan + MSan + Regression Test # ~ 4.5mn
|
||||
script:
|
||||
- make -j uasanregressiontest
|
||||
- make clean
|
||||
- make -j msanregressiontest
|
||||
|
||||
- name: C++, gnu90 and c99 compatibility # ~3mn
|
||||
script:
|
||||
- make cxxtest
|
||||
- make clean
|
||||
- make gnu90build
|
||||
- make clean
|
||||
- make c99build
|
||||
- make clean
|
||||
- make travis-install # just ensures `make install` works
|
||||
|
||||
- name: mingw cross-compilation
|
||||
script :
|
||||
- sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
|
||||
- CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
||||
|
||||
- name: Valgrind + Fuzz Test Stack Mode # ~ 7mn
|
||||
script:
|
||||
- make valgrindinstall
|
||||
- make -C tests clean valgrindTest
|
||||
- make clean
|
||||
- make -C tests test-fuzzer-stackmode
|
||||
|
||||
- name: Qemu ARM emulation + Fuzz Test # ~13.5mn
|
||||
script:
|
||||
- make arminstall
|
||||
- make armfuzz
|
||||
|
||||
# Introduced to check compat with old toolchains, to prevent e.g. #1872
|
||||
- name: ARM Build Test (on Trusty)
|
||||
dist: trusty
|
||||
script:
|
||||
- make arminstall
|
||||
- make armbuild
|
||||
|
||||
- name: Qemu PPC + Fuzz Test # ~13mn
|
||||
dist: trusty # it seems ppc cross-compilation fails on "current"
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppcfuzz
|
||||
|
||||
# check release number (release only)
|
||||
- name: Tag-Specific Test
|
||||
if: tag =~ ^v[0-9]\.[0-9]
|
||||
script:
|
||||
- make -C tests checkTag
|
||||
- tests/checkTag "$TRAVIS_BRANCH"
|
||||
|
||||
# tests for master branch and cron job only
|
||||
- name: OS-X # ~13mn
|
||||
if: branch = master
|
||||
os: osx
|
||||
script:
|
||||
- make test
|
||||
- make -C lib all
|
||||
|
||||
- name: zbuff test
|
||||
if: branch = master
|
||||
script:
|
||||
- make -C tests test-zbuff
|
||||
|
||||
- name: Versions Compatibility Test # 11.5mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make -C tests versionsTest
|
||||
|
||||
- name: thread sanitizer # ~29mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make tsan-test-zstream
|
||||
- CC=clang-3.8 make tsan-fuzztest
|
||||
|
||||
- name: PPC64LE + Fuzz test # ~13mn
|
||||
if: branch = master
|
||||
arch: ppc64le
|
||||
script:
|
||||
- cat /proc/cpuinfo
|
||||
- make test
|
||||
|
||||
- name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?)
|
||||
dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
|
||||
if: branch = master
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppc64fuzz
|
||||
|
||||
# note : we already have aarch64 tests on hardware
|
||||
- name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn
|
||||
if: branch = master
|
||||
dist: xenial
|
||||
script:
|
||||
- make arminstall
|
||||
- make aarch64fuzz
|
||||
|
||||
- name: zlib wrapper test # ~7.5mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make gpp6install valgrindinstall
|
||||
- make -C zlibWrapper test
|
||||
- make -C zlibWrapper valgrindTest
|
||||
|
||||
- name: LZ4, thread pool, and partial libs tests # ~2mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make lz4install
|
||||
- make -C tests test-lz4
|
||||
- make clean
|
||||
- make -C tests test-pool
|
||||
- make clean
|
||||
- bash tests/libzstd_partial_builds.sh
|
||||
|
||||
# meson dedicated test
|
||||
- name: Xenial (Meson + clang) # ~15mn
|
||||
if: branch = master
|
||||
dist: xenial
|
||||
language: cpp
|
||||
compiler: clang
|
||||
- env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..'
|
||||
os: linux
|
||||
sudo: required
|
||||
install:
|
||||
- sudo apt-get install -qq liblz4-dev valgrind tree
|
||||
- |
|
||||
travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
|
||||
unzip ~/ninja.zip -d ~/.local/bin
|
||||
- |
|
||||
travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
|
||||
python3 ~/get-pip.py --user &&
|
||||
pip3 install --user meson
|
||||
script:
|
||||
- |
|
||||
meson setup \
|
||||
--buildtype=debugoptimized \
|
||||
-Db_lundef=false \
|
||||
-Dauto_features=enabled \
|
||||
-Dbin_programs=true \
|
||||
-Dbin_tests=true \
|
||||
-Dbin_contrib=true \
|
||||
-Ddefault_library=both \
|
||||
build/meson builddir
|
||||
- pushd builddir
|
||||
- ninja
|
||||
- meson test --verbose --no-rebuild
|
||||
- DESTDIR=./staging ninja install
|
||||
- tree ./staging
|
||||
after_failure:
|
||||
- cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
|
||||
- export CXX="g++-6" CC="gcc-6"
|
||||
- export LDFLAGS="-fuse-ld=gold"
|
||||
- export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- gcc-6
|
||||
- g++-6
|
||||
|
||||
allow_failures:
|
||||
- env: ALLOW_FAILURES=true
|
||||
|
||||
# Ubuntu 14.04 LTS Server Edition 64 bit
|
||||
- env: Ubu=14.04 Cmd="make armtest"
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- qemu-system-arm
|
||||
- qemu-user-static
|
||||
- gcc-arm-linux-gnueabi
|
||||
- libc6-dev-armel-cross
|
||||
|
||||
- env: Ubu=14.04 Cmd="make aarch64test"
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- qemu-system-arm
|
||||
- qemu-user-static
|
||||
- gcc-aarch64-linux-gnu
|
||||
- libc6-dev-arm64-cross
|
||||
|
||||
- env: Ubu=14.04 Cmd='make ppctest'
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- qemu-system-ppc
|
||||
- qemu-user-static
|
||||
- gcc-powerpc-linux-gnu
|
||||
|
||||
- env: Ubu=14.04 Cmd='make ppc64test'
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- qemu-system-ppc
|
||||
- qemu-user-static
|
||||
- gcc-powerpc-linux-gnu
|
||||
|
||||
- env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- valgrind
|
||||
|
||||
|
||||
|
||||
# other feature branches => short tests
|
||||
- env: Ubu=12.04cont Cmd="make test && make clean && make travis-install"
|
||||
os: linux
|
||||
sudo: false
|
||||
|
||||
- env: Ubu=14.04 Cmd="make -C tests test32"
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- libc6-dev-i386
|
||||
- gcc-multilib
|
||||
|
||||
- env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
|
||||
&& make c99test && make clean && make gnu99test && make clean
|
||||
&& make clangtest && make clean && make -C contrib/pzstd googletest32
|
||||
&& make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
install:
|
||||
- export CXX="g++-4.8" CC="gcc-4.8"
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- libc6-dev-i386
|
||||
- g++-multilib
|
||||
- gcc-4.8
|
||||
- gcc-4.8-multilib
|
||||
- g++-4.8
|
||||
- g++-4.8-multilib
|
||||
|
||||
- env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
|
||||
os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- gcc-multilib
|
||||
- gcc-5
|
||||
- gcc-5-multilib
|
||||
- gcc-6
|
||||
- gcc-6-multilib
|
||||
|
||||
script:
|
||||
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
||||
# dev => normal tests; other feature branches => short tests (number > 11)
|
||||
- if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi
|
||||
# master => long tests, as this is the final step towards a Release
|
||||
- if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi
|
||||
|
||||
@@ -1,555 +0,0 @@
|
||||
v1.4.5
|
||||
fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
|
||||
perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
|
||||
perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
|
||||
perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
|
||||
perf: Small level 1 compression speed gains (depending on compiler)
|
||||
cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
|
||||
cli : New --filelist= : Provide a list of files to operate upon from a file
|
||||
cli : -b -d command can now benchmark decompression on multiple files
|
||||
cli : New --no-content-size command
|
||||
cli : New --show-default-cparams information command
|
||||
api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
|
||||
api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
|
||||
build: Generate a single-file libzstd library (#2065, by @cwoffenden)
|
||||
build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
|
||||
build: zstd now compiles cleanly under -pedantic (#2099)
|
||||
build: zstd now compiles with make-4.3
|
||||
build: Support mingw cross-compilation from Linux, by @Ericson2314
|
||||
build: Meson multi-thread build fix on windows
|
||||
build: Some misc icc fixes backed by new ci test on travis
|
||||
misc: bitflip analyzer tool, by @felixhandte
|
||||
misc: Extend largeNbDicts benchmark to compression
|
||||
misc: Edit-distance match finder in contrib/
|
||||
doc : Improved beginner CONTRIBUTING.md docs
|
||||
doc : New issue templates for zstd
|
||||
|
||||
v1.4.4
|
||||
perf: Improved decompression speed, by > 10%, by @terrelln
|
||||
perf: Better compression speed when re-using a context, by @felixhandte
|
||||
perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
|
||||
perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
|
||||
perf: minor generic speed optimization, by @davidbolvansky
|
||||
api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
|
||||
api: fixed decoding of magic-less frames, by @terrelln
|
||||
api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
|
||||
cli: Named pipes support, by @bimbashrestha
|
||||
cli: short tar's extension support, by @stokito
|
||||
cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
|
||||
cli: commands --stream-size=# and --size-hint=#, by @nmagerko
|
||||
cli: command --exclude-compressed, by @shashank0791
|
||||
cli: faster `-t` test mode
|
||||
cli: improved some error messages, by @vangyzen
|
||||
cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
|
||||
cli: fix rare deadlock condition within dictionary builder, by @terrelln
|
||||
build: single-file decoder with emscripten compilation script, by @cwoffenden
|
||||
build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
|
||||
build: fixed deprecation warning for certain gcc version, reported by @jasonma163
|
||||
build: fix compilation on old gcc versions, by @cemeyer
|
||||
build: improved installation directories for cmake script, by Dmitri Shubin
|
||||
pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
|
||||
misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
|
||||
misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
|
||||
|
||||
v1.4.3
|
||||
bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
|
||||
bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
|
||||
build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
|
||||
|
||||
v1.4.2
|
||||
bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
|
||||
bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
|
||||
misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
|
||||
misc: Restructure source files by @ephiepark (#1679)
|
||||
|
||||
v1.4.1
|
||||
bug: Fix data corruption in niche use cases by @terrelln (#1659)
|
||||
bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
|
||||
bug: Fix out of bounds read by @terrelln (#1590)
|
||||
perf: Improve decode speed by ~7% @mgrice (#1668)
|
||||
perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
|
||||
perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
|
||||
perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
|
||||
perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
|
||||
api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
|
||||
cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
|
||||
cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
|
||||
cli: Restrict read permissions on destination files by @chungy (#1644)
|
||||
cli: zstdgrep: handle -f flag by @felixhandte (#1618)
|
||||
cli: zstdcat: follow symlinks by @vejnar (#1604)
|
||||
doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
|
||||
doc: Fix typo by @yk-tanigawa (#1633)
|
||||
doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
|
||||
build: CMake: support building with LZ4 @leeyoung624 (#1626)
|
||||
build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
|
||||
build: CMake: respect existing uninstall target by @j301scott (#1619)
|
||||
build: Make: skip multithread tests when built without support by @michaelforney (#1620)
|
||||
build: Make: Fix examples/ test target by @sjnam (#1603)
|
||||
build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
|
||||
build: Meson: fix build by @lzutao (#1602)
|
||||
build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
|
||||
build: Visual Studio: fix linking by @absotively (#1639)
|
||||
build: Fix MinGW-W64 build by @myzhang1029 (#1600)
|
||||
misc: Expand decodecorpus coverage by @ephiepark (#1664)
|
||||
|
||||
v1.4.0
|
||||
perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
|
||||
api: Move the advanced API, including all functions in the staging section, to the stable section
|
||||
api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
|
||||
api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
|
||||
api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
|
||||
api: Don't export ZSTDMT functions from the shared library by default
|
||||
api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
|
||||
api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
|
||||
api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
|
||||
api: Move ZSTD_getDictID_*() functions to the stable section
|
||||
api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
|
||||
api: Allow compression parameters to be set when a dictionary is used
|
||||
api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
|
||||
api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
|
||||
api: Setting ZSTD_d_maxWindowLog to 0 means use the default
|
||||
cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
|
||||
cli: Add --[no-]compress-literals flag to enable or disable literal compression
|
||||
doc: Update the examples to use the advanced API
|
||||
doc: Explain how to transition from old streaming functions to the advanced API in the header
|
||||
build: Improve the Windows release packages
|
||||
build: Improve CMake build by @hjmjohnson
|
||||
build: Build fixes for FreeBSD by @lwhsu
|
||||
build: Remove redundant warnings by @thatsafunnyname
|
||||
build: Fix tests on OpenBSD by @bket
|
||||
build: Extend fuzzer build system to work with the new clang engine
|
||||
build: CMake now creates the libzstd.so.1 symlink
|
||||
build: Improve Menson build by @lzutao
|
||||
misc: Fix symbolic link detection on FreeBSD
|
||||
misc: Use physical core count for -T0 on FreeBSD by @cemeyer
|
||||
misc: Fix zstd --list on truncated files by @kostmo
|
||||
misc: Improve logging in debug mode by @felixhandte
|
||||
misc: Add CirrusCI tests by @lwhsu
|
||||
misc: Optimize dictionary memory usage in corner cases
|
||||
misc: Improve the dictionary builder on small or homogeneous data
|
||||
misc: Fix spelling across the repo by @jsoref
|
||||
|
||||
v1.3.8
|
||||
perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
|
||||
perf: slightly better compression ratio at high compression modes
|
||||
api : finalized advanced API, last stage before "stable" status
|
||||
api : new --rsyncable mode, by @terrelln
|
||||
api : support decompression of empty frames into NULL (used to be an error) (#1385)
|
||||
build: new set of macros to build a minimal size decoder, by @felixhandte
|
||||
build: fix compilation on MIPS32, reported by @clbr (#1441)
|
||||
build: fix compilation with multiple -arch flags, by @ryandesign
|
||||
build: highly upgraded meson build, by @lzutao
|
||||
build: improved buck support, by @obelisk
|
||||
build: fix cmake script : can create debug build, by @pitrou
|
||||
build: Makefile : grep works on both colored consoles and systems without color support
|
||||
build: fixed zstd-pgo, by @bmwiedemann
|
||||
cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
|
||||
cli : --no-progress flag, preserving final summary (#1371), by @terrelln
|
||||
cli : ensure destination file is not source file (#1422)
|
||||
cli : clearer error messages, especially when input file not present
|
||||
doc : clarified zstd_compression_format.md, by @ulikunitz
|
||||
misc: fixed zstdgrep, returns 1 on failure, by @lzutao
|
||||
misc: NEWS renamed as CHANGELOG, in accordance with fboss
|
||||
|
||||
v1.3.7
|
||||
perf: slightly better decompression speed on clang (depending on hardware target)
|
||||
fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
|
||||
build: no longer build backtrace by default in release mode; restrict further automatic mode
|
||||
build: control backtrace support through build macro BACKTRACE
|
||||
misc: added man pages for zstdless and zstdgrep, by @samrussell
|
||||
|
||||
v1.3.6
|
||||
perf: much faster dictionary builder, by @jenniferliu
|
||||
perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
|
||||
perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
|
||||
cli : fix : does no longer overwrite destination when source does not exist (#1082)
|
||||
cli : new command --adapt, for automatic compression level adaptation
|
||||
api : fix : block api can be streamed with > 4 GB, reported by @catid
|
||||
api : reduced ZSTD_DDict size by 2 KB
|
||||
api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
|
||||
build: support Haiku target, by @korli
|
||||
build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
|
||||
doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
|
||||
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
|
||||
|
||||
v1.3.5
|
||||
perf: much faster dictionary compression, by @felixhandte
|
||||
perf: small quality improvement for dictionary generation, by @terrelln
|
||||
perf: slightly improved high compression levels (notably level 19)
|
||||
mem : automatic memory release for long duration contexts
|
||||
cli : fix : overlapLog can be manually set
|
||||
cli : fix : decoding invalid lz4 frames
|
||||
api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
|
||||
api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
|
||||
build: select custom libzstd scope through control macros, by @GeorgeLu97
|
||||
build: OpenBSD patch, by @bket
|
||||
build: make and make all are compatible with -j
|
||||
doc : clarify zstd_compression_format.md, updated for IETF RFC process
|
||||
misc: pzstd compatible with reproducible compilation, by @lamby
|
||||
|
||||
v1.3.4
|
||||
perf: faster speed (especially decoding speed) on recent cpus (haswell+)
|
||||
perf: much better performance associating --long with multi-threading, by @terrelln
|
||||
perf: better compression at levels 13-15
|
||||
cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
|
||||
cli : smoother status report in multi-threading mode
|
||||
cli : added command --fast=#, for faster compression modes
|
||||
cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
|
||||
api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
|
||||
api : compression levels can be negative, for even more speed
|
||||
api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
|
||||
api : ZSTDMT can accept new compression parameters during compression
|
||||
api : implemented all advanced dictionary decompression prototypes
|
||||
build: improved meson recipe, by Shawn Landden (@shawnl)
|
||||
build: VS2017 scripts, by @HaydnTrigg
|
||||
misc: all /contrib projects fixed
|
||||
misc: added /contrib/docker script by @gyscos
|
||||
|
||||
v1.3.3
|
||||
perf: faster zstd_opt strategy (levels 16-19)
|
||||
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
|
||||
cli : fix : content size written in header by default
|
||||
cli : fix : improved LZ4 format support, by @felixhandte
|
||||
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
|
||||
api : fix : support large skippable frames, by @terrelln
|
||||
api : fix : streaming interface was adding a useless 3-bytes null block to small frames
|
||||
api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
|
||||
build: fix : compilation under rhel6 and centos6, reported by @pixelb
|
||||
build: added `check` target
|
||||
|
||||
v1.3.2
|
||||
new : long range mode, using --long command, by Stella Lau (@stellamplau)
|
||||
new : ability to generate and decode magicless frames (#591)
|
||||
changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
|
||||
fix : multi-threading compression works with custom allocators
|
||||
fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
|
||||
fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
|
||||
fix : 32-bits build can now decode large offsets (levels 21+)
|
||||
cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
|
||||
cli : improved --list output
|
||||
cli : new : can split input file for dictionary training, using command -B#
|
||||
cli : new : clean operation artefact on Ctrl-C interruption
|
||||
cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
|
||||
cli : fix : write file size in header in multiple-files mode
|
||||
api : added macro ZSTD_COMPRESSBOUND() for static allocation
|
||||
api : experimental : new advanced decompression API
|
||||
api : fix : sizeof_CCtx() used to over-estimate
|
||||
build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
|
||||
build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
|
||||
example : added streaming_memory_usage
|
||||
license : changed /examples license to BSD + GPLv2
|
||||
license : fix a few header files to reflect new license (#825)
|
||||
|
||||
v1.3.1
|
||||
New license : BSD + GPLv2
|
||||
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
|
||||
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
|
||||
cli : improved and fixed --list command, by @ib (#772)
|
||||
cli : command -vV to list supported formats, by @ib (#771)
|
||||
build : fixed binary variants, reported by @svenha (#788)
|
||||
build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
|
||||
API exp : breaking change : ZSTD_getframeHeader() provides more information
|
||||
API exp : breaking change : pinned down values of error codes
|
||||
doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
|
||||
new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
|
||||
new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
|
||||
updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
|
||||
|
||||
v1.3.0
|
||||
cli : new : `--list` command, by Paul Cruz
|
||||
cli : changed : xz/lzma support enabled by default
|
||||
cli : changed : `-t *` continue processing list after a decompression error
|
||||
API : added : ZSTD_versionString()
|
||||
API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
|
||||
API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
|
||||
API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
|
||||
API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
|
||||
API exp : clarified memory estimation / measurement functions.
|
||||
API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
|
||||
tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
|
||||
new : contrib/seekable_format, demo and API, by Sean Purcell
|
||||
changed : contrib/linux-kernel, updated version and license, by Nick Terrell
|
||||
|
||||
v1.2.0
|
||||
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
|
||||
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
|
||||
cli : new : zstdmt symlink hardwired to `zstd -T0`
|
||||
cli : new : command --threads=# (#671)
|
||||
cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
|
||||
cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
|
||||
cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
|
||||
cli : fix : does not output compressed data on console
|
||||
cli : fix : ignore symbolic links unless --force specified,
|
||||
API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
|
||||
API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
|
||||
API : improved: ZSTDMT_compressCCtx() reduced memory usage
|
||||
API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
|
||||
API : fix : src size stored in frame header is controlled at end of frame
|
||||
API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
|
||||
API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
|
||||
build: improved cmake script, by @Majlen
|
||||
build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
|
||||
tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
|
||||
new : contrib/linux-kernel version, by Nick Terrell
|
||||
|
||||
v1.1.4
|
||||
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
||||
cli : new : advanced benchmark command --priority=rt
|
||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||
cli : fix : --rm remains silent when input is stdin
|
||||
cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
|
||||
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
|
||||
memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
|
||||
arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
|
||||
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
|
||||
API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
|
||||
build : new: meson build system in contrib/meson, by Dima Krasner
|
||||
build : improved cmake script, by @Majlen
|
||||
build : added -Wformat-security flag, as recommended by Padraig Brady
|
||||
doc : new : educational decoder, by Sean Purcell
|
||||
|
||||
v1.1.3
|
||||
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
|
||||
cli : new : experimental target `make zstdmt`, with multi-threading support
|
||||
cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
|
||||
cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
|
||||
cli : fix zstdless on Mac OS-X, by Andrew Janke
|
||||
cli : fix #232 "compress non-files"
|
||||
dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
|
||||
API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
|
||||
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
|
||||
API : new : ZDICT_finalizeDictionary()
|
||||
API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
|
||||
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
|
||||
build : support for Solaris target, by Przemyslaw Skibinski
|
||||
doc : clarified specification, by Sean Purcell
|
||||
|
||||
v1.1.2
|
||||
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
|
||||
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
|
||||
API : zbuff : changed : prototypes now generate deprecation warnings
|
||||
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
|
||||
lib : changed : only public ZSTD_ symbols are now exposed
|
||||
lib : changed : reduced usage of stack memory
|
||||
lib : fixed : several corner case bugs, by Nick Terrell
|
||||
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
|
||||
cli : new : preserve file attributes
|
||||
cli : new : added zstdless and zstdgrep tools
|
||||
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
|
||||
cli : fixed : zstdcat
|
||||
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
|
||||
install : better compatibility with FreeBSD, by Dimitry Andric
|
||||
source tree : changed : zbuff source files moved to lib/deprecated
|
||||
|
||||
v1.1.1
|
||||
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
|
||||
New : doc/zstd_manual.html, by Przemyslaw Skibinski
|
||||
Improved : slightly better compression ratio at --ultra levels (>= 20)
|
||||
Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
|
||||
Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
|
||||
Added : example/multiple_streaming_compression.c
|
||||
Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
|
||||
Updated man page
|
||||
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
|
||||
|
||||
v1.1.0
|
||||
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
||||
added : NetBSD install target (#338)
|
||||
Improved : speed for batches of small files
|
||||
Improved : speed of zlib wrapper, by Przemyslaw Skibinski
|
||||
Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
|
||||
Fixed : CLI -d output to stdout by default when input is stdin (#322)
|
||||
Fixed : CLI correctly detects console on Mac OS-X
|
||||
Fixed : CLI supports recursive mode `-r` on Mac OS-X
|
||||
Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
|
||||
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
|
||||
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
|
||||
Fixed : zstd-pgo, reported by octoploid (#329)
|
||||
|
||||
v1.0.0
|
||||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.1
|
||||
New streaming API
|
||||
Changed : --ultra now enables levels beyond 19
|
||||
Changed : -i# now selects benchmark time in second
|
||||
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
|
||||
Fixed : speed regression on specific patterns (#272)
|
||||
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
|
||||
Fixed : ICC compilation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.0
|
||||
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
|
||||
New : Build on FreeBSD and DragonFly, thanks to JrMarino
|
||||
Changed : modified API : ZSTD_compressEnd()
|
||||
Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
|
||||
Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
|
||||
Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
|
||||
Fixed : checksum correctly checked in single-pass mode
|
||||
Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
|
||||
Modified : minor compression level adaptations
|
||||
Updated : compression format specification to v0.2.0
|
||||
changed : zstd.h moved to /lib directory
|
||||
|
||||
v0.7.5
|
||||
Transition version, supporting decoding of v0.8.x
|
||||
|
||||
v0.7.4
|
||||
Added : homebrew for Mac, by Daniel Cade
|
||||
Added : more examples
|
||||
Fixed : segfault when using small dictionaries, reported by Felix Handte
|
||||
Modified : default compression level for CLI is now 3
|
||||
Updated : specification, to v0.1.1
|
||||
|
||||
v0.7.3
|
||||
New : compression format specification
|
||||
New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
|
||||
New : `ZSTD_getDecompressedSize()`
|
||||
New : OpenBSD target, by Juan Francisco Cantero Hurtado
|
||||
New : `examples` directory
|
||||
fixed : dictBuilder using HC levels, reported by Bartosz Taudul
|
||||
fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
|
||||
fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
|
||||
modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
|
||||
modified : legacy functions no longer need magic number
|
||||
|
||||
v0.7.2
|
||||
fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
|
||||
fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
|
||||
fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
|
||||
|
||||
v0.7.1
|
||||
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
|
||||
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
|
||||
fixed : corruption issue, reported by cj
|
||||
modified : checksum enabled by default in command line mode
|
||||
|
||||
v0.7.0
|
||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||
New : Command `--rm`, to remove source file after successful de/compression
|
||||
New : Visual build scripts, by Christophe Chevalier
|
||||
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
||||
New : Frame checksum support
|
||||
New : Support pass-through mode (when using `-df`)
|
||||
API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
|
||||
API : create dictionary files from custom content, by Giuseppe Ottaviano
|
||||
API : support for custom malloc/free functions
|
||||
New : controllable Dictionary ID
|
||||
New : Support for skippable frames
|
||||
|
||||
v0.6.1
|
||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||
New : Ability to compile compressor / decompressor separately
|
||||
Changed : new lib directory structure
|
||||
Fixed : Legacy codec v0.5 compatible with dictionary decompression
|
||||
Fixed : Decoder corruption error (#173)
|
||||
Fixed : null-string roundtrip (#176)
|
||||
New : benchmark mode can select directory as input
|
||||
Experimental : midipix support, VMS support
|
||||
|
||||
v0.6.0
|
||||
Stronger high compression modes, thanks to Przemyslaw Skibinski
|
||||
API : ZSTD_getFrameParams() provides size of decompressed content
|
||||
New : highest compression modes require `--ultra` command to fully unleash their capacity
|
||||
Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
|
||||
|
||||
v0.5.1
|
||||
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
|
||||
Changed : Dictionary builder integrated into libzstd and zstd cli
|
||||
Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
|
||||
Fix : high compression modes for big-endian platforms
|
||||
New : zstd cli : `-t` | `--test` command
|
||||
|
||||
v0.5.0
|
||||
New : dictionary builder utility
|
||||
Changed : streaming & dictionary API
|
||||
Improved : better compression of small data
|
||||
|
||||
v0.4.7
|
||||
Improved : small compression speed improvement in HC mode
|
||||
Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
|
||||
fix : bt search bug
|
||||
|
||||
v0.4.6
|
||||
fix : fast compression mode on Windows
|
||||
New : cmake configuration file, thanks to Artyom Dymchenko
|
||||
Improved : high compression mode on repetitive data
|
||||
New : block-level API
|
||||
New : ZSTD_duplicateCCtx()
|
||||
|
||||
v0.4.5
|
||||
new : -m/--multiple : compress/decompress multiple files
|
||||
|
||||
v0.4.4
|
||||
Fixed : high compression modes for Windows 32 bits
|
||||
new : external dictionary API extended to buffered mode and accessible through command line
|
||||
new : windows DLL project, thanks to Christophe Chevalier
|
||||
|
||||
v0.4.3 :
|
||||
new : external dictionary API
|
||||
new : zstd-frugal
|
||||
|
||||
v0.4.2 :
|
||||
Generic minor improvements for small blocks
|
||||
Fixed : big-endian compatibility, by Peter Harris (#85)
|
||||
|
||||
v0.4.1
|
||||
Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
|
||||
removed `zstd.c`
|
||||
|
||||
v0.4.0
|
||||
Command line utility compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
|
||||
v0.3.6
|
||||
small blocks params
|
||||
|
||||
v0.3.5
|
||||
minor generic compression improvements
|
||||
|
||||
v0.3.4
|
||||
Faster fast cLevels
|
||||
|
||||
v0.3.3
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3.2
|
||||
Fixed Visual Studio
|
||||
|
||||
v0.3.1 :
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3
|
||||
HC mode : compression levels 2-26
|
||||
|
||||
v0.2.2
|
||||
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
|
||||
|
||||
v0.2.1
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
|
||||
v0.2.0
|
||||
**Breaking format change**
|
||||
Faster decompression speed
|
||||
Can still decode v0.1 format
|
||||
|
||||
v0.1.3
|
||||
fix uninitialization warning, reported by Evan Nemerson
|
||||
|
||||
v0.1.2
|
||||
frame concatenation support
|
||||
|
||||
v0.1.1
|
||||
fix compression bug
|
||||
detects write-flush errors
|
||||
|
||||
v0.1.0
|
||||
first release
|
||||
@@ -1,5 +0,0 @@
|
||||
# Code of Conduct
|
||||
|
||||
Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
|
||||
Please read the [full text](https://code.fb.com/codeofconduct/)
|
||||
so that you can understand what actions will and will not be tolerated.
|
||||
+2
-366
@@ -26,370 +26,6 @@ to do this once to work on any of Facebook's open source projects.
|
||||
|
||||
Complete your CLA here: <https://code.facebook.com/cla>
|
||||
|
||||
## Workflow
|
||||
Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
|
||||
will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
|
||||
related changes into a branch.
|
||||
|
||||
Our contribution process works in three main stages:
|
||||
1. Local development
|
||||
* Update:
|
||||
* Checkout your fork of zstd if you have not already
|
||||
```
|
||||
git checkout https://github.com/<username>/zstd
|
||||
cd zstd
|
||||
```
|
||||
* Update your local dev branch
|
||||
```
|
||||
git pull https://github.com/facebook/zstd dev
|
||||
git push origin dev
|
||||
```
|
||||
* Topic and development:
|
||||
* Make a new branch on your fork about the topic you're developing for
|
||||
```
|
||||
# branch names should be consise but sufficiently informative
|
||||
git checkout -b <branch-name>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Make commits and push
|
||||
```
|
||||
# make some changes =
|
||||
git add -u && git commit -m <message>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Note: run local tests to ensure that your changes didn't break existing functionality
|
||||
* Quick check
|
||||
```
|
||||
make shortest
|
||||
```
|
||||
* Longer check
|
||||
```
|
||||
make test
|
||||
```
|
||||
2. Code Review and CI tests
|
||||
* Ensure CI tests pass:
|
||||
* Before sharing anything to the community, make sure that all CI tests pass on your local fork.
|
||||
See our section on setting up your CI environment for more information on how to do this.
|
||||
* Ensure that static analysis passes on your development machine. See the Static Analysis section
|
||||
below to see how to do this.
|
||||
* Create a pull request:
|
||||
* When you are ready to share you changes to the community, create a pull request from your branch
|
||||
to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
|
||||
page.
|
||||
* From there, select the branch where you made changes as your source branch and facebook:dev
|
||||
as the destination.
|
||||
* Examine the diff presented between the two branches to make sure there is nothing unexpected.
|
||||
* Write a good pull request description:
|
||||
* While there is no strict template that our contributors follow, we would like them to
|
||||
sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
|
||||
at least indirectly, address the following points.
|
||||
* Is this pull request important and why?
|
||||
* Is it addressing an issue? If so, what issue? (provide links for convenience please)
|
||||
* Is this a new feature? If so, why is it useful and/or necessary?
|
||||
* Are there background references and documents that reviewers should be aware of to properly assess this change?
|
||||
* Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
|
||||
* Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
|
||||
* Submit the pull request and iterate with feedback.
|
||||
3. Merge and Release
|
||||
* Getting approval:
|
||||
* You will have to iterate on your changes with feedback from other collaborators to reach a point
|
||||
where your pull request can be safely merged.
|
||||
* To avoid too many comments on style and convention, make sure that you have a
|
||||
look at our style section below before creating a pull request.
|
||||
* Eventually, someone from the zstd team will approve your pull request and not long after merge it into
|
||||
the dev branch.
|
||||
* Housekeeping:
|
||||
* Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
|
||||
the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
|
||||
issue at hand, then please indicate this by requesting that an issue be closed by commenting.
|
||||
* Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
|
||||
that the change must make it to an official zstd release for it to be meaningful. We recommend
|
||||
that contributers track the activity on their pull request and corresponding issue(s) page(s) until
|
||||
their change makes it to the next release of zstd. Users will often discover bugs in your code or
|
||||
suggest ways to refine and improve your initial changes even after the pull request is merged.
|
||||
|
||||
## Static Analysis
|
||||
Static analysis is a process for examining the correctness or validity of a program without actually
|
||||
executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
|
||||
static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
|
||||
|
||||
Once installed, you can ensure that our static analysis tests pass on your local development machine
|
||||
by running:
|
||||
```
|
||||
make staticAnalyze
|
||||
```
|
||||
|
||||
In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
|
||||
just `contrib/largeNbDicts` and nothing else, you can run:
|
||||
|
||||
```
|
||||
scan-build make -C contrib/largeNbDicts largeNbDicts
|
||||
```
|
||||
|
||||
### Pitfalls of static analysis
|
||||
`scan-build` is part of our regular CI suite. Other static analyzers are not.
|
||||
|
||||
It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
|
||||
|
||||
- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
|
||||
- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
|
||||
- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
|
||||
+ torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
|
||||
+ sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
|
||||
- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
|
||||
|
||||
This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.
|
||||
|
||||
## Performance
|
||||
Performance is extremely important for zstd and we only merge pull requests whose performance
|
||||
landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
|
||||
This high bar for performance means that every PR which has the potential to
|
||||
impact performance takes a very long time for us to properly review. That being said, we
|
||||
always welcome contributions to improve performance (or worsen performance for the trade-off of
|
||||
something else). Please keep the following in mind before submitting a performance related PR:
|
||||
|
||||
1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
|
||||
very well documented via past Github issues and pull requests. It may be the case that your
|
||||
particular performance optimization has already been considered in the past. Please take some
|
||||
time to search through old issues and pull requests using keywords specific to your
|
||||
would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
|
||||
on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
|
||||
it will be helpful for you to have context from that topic's history before contributing.
|
||||
2. The distinction between noise and actual performance gains can unfortunately be very subtle
|
||||
especially when microbenchmarking extremely small wins or losses. The only remedy to getting
|
||||
something subtle merged is extensive benchmarking. You will be doing us a great favor if you
|
||||
take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
|
||||
benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
|
||||
your changes on every single processor and os out there (and neither will we) but do that best
|
||||
you can:) We've adding some things to think about when benchmarking below in the Benchmarking
|
||||
Performance section which might be helpful for you.
|
||||
3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
|
||||
legitimate thing to do as long as it does not harm the overall performance health of Zstd.
|
||||
This is a hard balance to strike but please keep in mind other aspects of Zstd when
|
||||
submitting changes that are clang-specific, windows-specific, etc.
|
||||
|
||||
## Benchmarking Performance
|
||||
Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
|
||||
testing over theoretical speculation. This guide it not perfect but for most scenarios, it
|
||||
is a good place to start.
|
||||
|
||||
### Stability
|
||||
Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
|
||||
benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
|
||||
will typically not be stable enough to obtain reliable benchmark results. If you can get your
|
||||
hands on a desktop, this is usually a better scenario.
|
||||
|
||||
Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
|
||||
do a little more work to ensure that you are in fact measuring the changes you've made not and
|
||||
noise. Here are some things you can do to make your benchmarks more stable:
|
||||
|
||||
1. The most simple thing you can do to drastically improve the stability of your benchmark is
|
||||
to run it multiple times and then aggregate the results of those runs. As a general rule of
|
||||
thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
|
||||
you will have to aggregate over to get reliable results. Here are some additional things to keep in
|
||||
mind when running multiple trials:
|
||||
* How you aggregate your samples are important. You might be tempted to use the mean of your
|
||||
results. While this is certainly going to be a more stable number than a raw single sample
|
||||
benchmark number, you might have more luck by taking the median. The mean is not robust to
|
||||
outliers whereas the median is. Better still, you could simply take the fastest speed your
|
||||
benchmark achieved on each run since that is likely the fastest your process will be
|
||||
capable of running your code. In our experience, this (aggregating by just taking the sample
|
||||
with the fastest running time) has been the most stable approach.
|
||||
* The more samples you have, the more stable your benchmarks should be. You can verify
|
||||
your improved stability by looking at the size of your confidence intervals as you
|
||||
increase your sample count. These should get smaller and smaller. Eventually hopefully
|
||||
smaller than the performance win you are expecting.
|
||||
* Most processors will take some time to get `hot` when running anything. The observations
|
||||
you collect during that time period will very different from the true performance number. Having
|
||||
a very large number of sample will help alleviate this problem slightly but you can also
|
||||
address is directly by simply not including the first `n` iterations of your benchmark in
|
||||
your aggregations. You can determine `n` by simply looking at the results from each iteration
|
||||
and then hand picking a good threshold after which the variance in results seems to stabilize.
|
||||
2. You cannot really get reliable benchmarks if your host machine is simultaneously running
|
||||
another cpu/memory-intensive application in the background. If you are running benchmarks on your
|
||||
personal laptop for instance, you should close all applications (including your code editor and
|
||||
browser) before running your benchmarks. You might also have invisible background applications
|
||||
running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
|
||||
on Windows. You will get more stable benchmark results of you end those processes as well.
|
||||
* If you have multiple cores, you can even run your benchmark on a reserved core to prevent
|
||||
pollution from other OS and user processes. There are a number of ways to do this depending
|
||||
on your OS:
|
||||
* On linux boxes, you have use https://github.com/lpechacek/cpuset.
|
||||
* On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
|
||||
* On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
|
||||
3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
|
||||
Dynamically linking your library will introduce some added variation (not a large amount but
|
||||
definitely some). Statically linking libzstd will be more stable. Static libraries should
|
||||
be enabled by default when building zstd.
|
||||
4. Use a profiler with a good high resolution timer. See the section below on profiling for
|
||||
details on this.
|
||||
5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
|
||||
6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
|
||||
tmpfs avoids touching a real storage system, which can have a pretty big variability.
|
||||
|
||||
Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
|
||||
|
||||
### Zstd benchmark
|
||||
The fastest signal you can get regarding your performance changes is via the in-build zstd cli
|
||||
bench option. You can run Zstd as you typically would for your scenario using some set of options
|
||||
and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
|
||||
for that options you have just provided. If you want to look at the internals of how this
|
||||
benchmarking script works, you can check out programs/benchzstd.c
|
||||
|
||||
For example: say you have made a change that you believe improves the speed of zstd level 1. The
|
||||
very first thing you should use to asses whether you actually achieved any sort of improvement
|
||||
is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
|
||||
specify a running time for your benchmark in seconds (default is 3 seconds).
|
||||
Usually, the longer the running time, the more stable your results will be.
|
||||
|
||||
```
|
||||
$ git checkout <commit-before-your-change>
|
||||
$ make && cp zstd zstd-old
|
||||
$ git checkout <commit-after-your-change>
|
||||
$ make && cp zstd zstd-new
|
||||
$ zstd-old -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s
|
||||
$ zstd-new -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s
|
||||
```
|
||||
|
||||
Unless your performance win is large enough to be visible despite the intrinsic noise
|
||||
on your computer, benchzstd alone will likely not be enough to validate the impact of your
|
||||
changes. For example, the results of the example above indicate that effectively nothing
|
||||
changed but there could be a small <3% improvement that the noise on the host machine
|
||||
obscured. So unless you see a large performance win (10-15% consistently) using just
|
||||
this method of evaluation will not be sufficient.
|
||||
|
||||
### Profiling
|
||||
There are a number of great profilers out there. We're going to briefly mention how you can
|
||||
profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
|
||||
on windows.
|
||||
|
||||
Say you have an idea for a change that you think will provide some good performance gains
|
||||
for level 1 compression on Zstd. Typically this means, you have identified a section of
|
||||
code that you think can be made to run faster.
|
||||
|
||||
The first thing you will want to do is make sure that the piece of code is actually taking up
|
||||
a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
|
||||
0.0001% of the total running time. Luckily, there are tools to help with this.
|
||||
Profilers will let you see how much time your code spends inside a particular function.
|
||||
If your target code snippit is only part of a function, it might be worth trying to
|
||||
isolate that snippit by moving it to its own function (this is usually not necessary but
|
||||
might be).
|
||||
|
||||
Most profilers (including the profilers dicusssed below) will generate a call graph of
|
||||
functions for you. Your goal will be to find your function of interest in this call grapch
|
||||
and then inspect the time spent inside of it. You might also want to to look at the
|
||||
annotated assembly which most profilers will provide you with.
|
||||
|
||||
#### Instruments
|
||||
We will once again consider the scenario where you think you've identified a piece of code
|
||||
whose performance can be improved upon. Follow these steps to profile your code using
|
||||
Instruments.
|
||||
|
||||
1. Open Instruments
|
||||
2. Select `Time Profiler` from the list of standard templates
|
||||
3. Close all other applications except for your instruments window and your terminal
|
||||
4. Run your benchmarking script from your terminal window
|
||||
* You will want a benchmark that runs for at least a few seconds (5 seconds will
|
||||
usually be long enough). This way the profiler will have something to work with
|
||||
and you will have ample time to attach your profiler to this process:)
|
||||
* I will just use benchzstd as my bencharmking script for this example:
|
||||
```
|
||||
$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
|
||||
```
|
||||
5. Once you run your benchmarking script, switch back over to instruments and attach your
|
||||
process to the time profiler. You can do this by:
|
||||
* Clicking on the `All Processes` drop down in the top left of the toolbar.
|
||||
* Selecting your process from the dropdown. In my case, it is just going to be labled
|
||||
`zstd`
|
||||
* Hitting the bright red record circle button on the top left of the toolbar
|
||||
6. You profiler will now start collecting metrics from your bencharking script. Once
|
||||
you think you have collected enough samples (usually this is the case after 3 seconds of
|
||||
recording), stop your profiler.
|
||||
7. Make sure that in toolbar of the bottom window, `profile` is selected.
|
||||
8. You should be able to see your call graph.
|
||||
* If you don't see the call graph or an incomplete call graph, make sure you have compiled
|
||||
zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
|
||||
you will have to supply the `-g` flag alone with your build script. You might also
|
||||
have to provide the `-fno-omit-frame-pointer` flag
|
||||
9. Dig down the graph to find your function call and then inspect it by double clicking
|
||||
the list item. You will be able to see the annotated source code and the assembly side by
|
||||
side.
|
||||
|
||||
#### Perf
|
||||
|
||||
This wiki has a pretty detailed tutorial on getting started working with perf so we'll
|
||||
leave you to check that out of you're getting started:
|
||||
|
||||
https://perf.wiki.kernel.org/index.php/Tutorial
|
||||
|
||||
Some general notes on perf:
|
||||
* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
|
||||
counter statistics. Perf uses a high resolution timer and this is likely one
|
||||
of the first things your team will run when assessing your PR.
|
||||
* Perf has a long list of hardware counters that can be viewed with `perf --list`.
|
||||
When measuring optimizations, something worth trying is to make sure the handware
|
||||
counters you expect to be impacted by your change are in fact being so. For example,
|
||||
if you expect the L1 cache misses to decrease with your change, you can look at the
|
||||
counter `L1-dcache-load-misses`
|
||||
* Perf hardware counters will not work on a virtual machine.
|
||||
|
||||
#### Visual Studio
|
||||
|
||||
TODO
|
||||
|
||||
|
||||
## Setting up continuous integration (CI) on your fork
|
||||
Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
|
||||
are well tested before they make it to an official release. Specifically, we use the platforms
|
||||
travis-ci, circle-ci, and appveyor.
|
||||
|
||||
Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
|
||||
The easiest way to run these CI tests on your own before submitting a PR to our dev branch
|
||||
is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
|
||||
instructions for doing this.
|
||||
|
||||
### travis-ci
|
||||
Follow these steps to link travis-ci with your github fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://travis-ci.org/
|
||||
3. Click 'Sign in with Github' on the top right
|
||||
4. Click 'Authorize travis-ci'
|
||||
5. Click 'Activate all repositories using Github Apps'
|
||||
6. Select 'Only select repositories' and select your fork of zstd from the drop down
|
||||
7. Click 'Approve and Install'
|
||||
8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
|
||||
9. Click 'Authorize travis-pro'
|
||||
10. You should have travis set up on your fork now.
|
||||
|
||||
### circle-ci
|
||||
TODO
|
||||
|
||||
### appveyor
|
||||
Follow these steps to link circle-ci with your girhub fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://www.appveyor.com/
|
||||
3. Click 'Sign in' on the top right
|
||||
4. Select 'Github' on the left panel
|
||||
5. Click 'Authorize appveyor'
|
||||
6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
|
||||
7. You should have appveyor set up on your fork now.
|
||||
|
||||
### General notes on CI
|
||||
CI tests run every time a pull request (PR) is created or updated. The exact tests
|
||||
that get run will depend on the destination branch you specify. Some tests take
|
||||
longer to run than others. Currently, our CI is set up to run a short
|
||||
series of tests when creating a PR to the dev branch and a longer series of tests
|
||||
when creating a PR to the master branch. You can look in the configuration files
|
||||
of the respective CI platform for more information on what gets run when.
|
||||
|
||||
Most people will just want to create a PR with the destination set to their local dev
|
||||
branch of zstd. You can then find the status of the tests on the PR's page. You can also
|
||||
re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
|
||||
|
||||
## Issues
|
||||
We use GitHub issues to track public bugs. Please ensure your description is
|
||||
clear and has sufficient instructions to be able to reproduce the issue.
|
||||
@@ -398,9 +34,9 @@ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
|
||||
disclosure of security bugs. In those cases, please go through the process
|
||||
outlined on that page and do not file a public issue.
|
||||
|
||||
## Coding Style
|
||||
## Coding Style
|
||||
* 4 spaces for indentation rather than tabs
|
||||
|
||||
## License
|
||||
By contributing to Zstandard, you agree that your contributions will be licensed
|
||||
under both the [LICENSE](LICENSE) file and the [COPYING](COPYING) file in the root directory of this source tree.
|
||||
under the [LICENSE](LICENSE) file in the root directory of this source tree.
|
||||
|
||||
@@ -1,339 +0,0 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
@@ -0,0 +1,11 @@
|
||||
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
|
||||
|
||||
The examples provided by Facebook are for non-commercial testing and evaluation
|
||||
purposes only. Facebook reserves all rights not expressly granted.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
FACEBOOK BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -1,250 +1,104 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# verbose mode (print commands) on V=1 or VERBOSE=1
|
||||
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
BUILDIR = build
|
||||
ZWRAPDIR = zlibWrapper
|
||||
TESTDIR = tests
|
||||
FUZZDIR = $(TESTDIR)/fuzz
|
||||
|
||||
# Define nul output
|
||||
VOID = /dev/null
|
||||
|
||||
# When cross-compiling from linux to windows, you might
|
||||
# need to specify this as "Windows." Fedora build fails
|
||||
# without it.
|
||||
#
|
||||
# Note: mingw-w64 build from linux to windows does not
|
||||
# fail on other tested distros (ubuntu, debian) even
|
||||
# without manually specifying the TARGET_SYSTEM.
|
||||
TARGET_SYSTEM ?= $(OS)
|
||||
|
||||
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
## default: Build lib-release and zstd-release
|
||||
.PHONY: default
|
||||
default: lib-release zstd-release
|
||||
default: lib zstd-release
|
||||
|
||||
.PHONY: all
|
||||
all: allmost examples manual contrib
|
||||
all: allmost
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
|
||||
|
||||
.PHONY: allmost
|
||||
allmost: allzstd zlibwrapper
|
||||
|
||||
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||
.PHONY: allzstd
|
||||
allzstd: lib-all
|
||||
$(Q)$(MAKE) -C $(PRGDIR) all
|
||||
$(Q)$(MAKE) -C $(TESTDIR) all
|
||||
allmost:
|
||||
$(MAKE) -C $(ZSTDDIR) all
|
||||
$(MAKE) -C $(PRGDIR) all
|
||||
$(MAKE) -C $(TESTDIR) all
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
|
||||
.PHONY: all32
|
||||
all32:
|
||||
$(MAKE) -C $(PRGDIR) zstd32
|
||||
$(MAKE) -C $(TESTDIR) all32
|
||||
|
||||
.PHONY: lib lib-release libzstd.a
|
||||
lib-all : lib
|
||||
lib lib-release lib-all :
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
.PHONY: lib
|
||||
lib:
|
||||
@$(MAKE) -C $(ZSTDDIR)
|
||||
|
||||
.PHONY: zstd zstd-release
|
||||
zstd zstd-release:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)cp $(PRGDIR)/zstd$(EXT) .
|
||||
.PHONY: zstd
|
||||
zstd:
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) .
|
||||
|
||||
.PHONY: zstd-release
|
||||
zstd-release:
|
||||
@$(MAKE) -C $(PRGDIR)
|
||||
cp $(PRGDIR)/zstd$(EXT) .
|
||||
|
||||
.PHONY: zstdmt
|
||||
zstdmt:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
|
||||
.PHONY: zlibwrapper
|
||||
zlibwrapper: lib
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
zlibwrapper:
|
||||
$(MAKE) -C $(ZWRAPDIR) test
|
||||
|
||||
## test: run long-duration tests
|
||||
.PHONY: test
|
||||
DEBUGLEVEL ?= 1
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
|
||||
test:
|
||||
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
|
||||
|
||||
## shortest: same as `make check`
|
||||
.PHONY: shortest
|
||||
shortest:
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
## check: run basic tests for `zstd` cli
|
||||
.PHONY: check
|
||||
check: shortest
|
||||
|
||||
.PHONY: automated_benchmarking
|
||||
automated_benchmarking:
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: benchmarking
|
||||
benchmarking: automated_benchmarking
|
||||
|
||||
## examples: build all examples in `/examples` directory
|
||||
.PHONY: examples
|
||||
examples: lib
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
|
||||
|
||||
## manual: generate API documentation in html format
|
||||
.PHONY: manual
|
||||
manual:
|
||||
$(MAKE) -C contrib/gen_html $@
|
||||
|
||||
## man: generate man page
|
||||
.PHONY: man
|
||||
man:
|
||||
$(MAKE) -C programs $@
|
||||
|
||||
## contrib: build all supported projects in `/contrib` directory
|
||||
.PHONY: contrib
|
||||
contrib: lib
|
||||
$(MAKE) -C contrib/pzstd all
|
||||
$(MAKE) -C contrib/seekable_format/examples all
|
||||
$(MAKE) -C contrib/largeNbDicts all
|
||||
cd contrib/single_file_libs/ ; ./build_decoder_test.sh
|
||||
cd contrib/single_file_libs/ ; ./build_library_test.sh
|
||||
|
||||
.PHONY: cleanTabs
|
||||
cleanTabs:
|
||||
cd contrib; ./cleanTabs
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C examples/ $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
$(Q)$(RM) -r lz4
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C examples/ $@ > $(VOID)
|
||||
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
@echo Cleaning completed
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, macOS, Hurd and some BSD targets
|
||||
#------------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, OSX, Hurd and some BSD targets
|
||||
#------------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
|
||||
HOST_OS = POSIX
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||
.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
#
|
||||
## list: Print all targets and their descriptions (if provided)
|
||||
.PHONY: list
|
||||
list:
|
||||
$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
|
||||
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
|
||||
{ \
|
||||
printf "Target Name\tDescription\n"; \
|
||||
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
|
||||
for target in $$TARGETS; do \
|
||||
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
|
||||
description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
|
||||
printf "$$target\t$$description\n"; \
|
||||
done \
|
||||
} | column -t -s $$'\t'
|
||||
|
||||
.PHONY: install armtest usan asan uasan
|
||||
install:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: travis-install
|
||||
travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
.PHONY: gcc5build
|
||||
gcc5build: clean
|
||||
gcc-5 -v
|
||||
CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
|
||||
|
||||
.PHONY: gcc6build
|
||||
gcc6build: clean
|
||||
gcc-6 -v
|
||||
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
|
||||
|
||||
.PHONY: gcc7build
|
||||
gcc7build: clean
|
||||
gcc-7 -v
|
||||
CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
|
||||
|
||||
.PHONY: clangbuild
|
||||
clangbuild: clean
|
||||
clang -v
|
||||
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
|
||||
|
||||
m32build: clean
|
||||
gcc -v
|
||||
$(MAKE) all32
|
||||
|
||||
armbuild: clean
|
||||
CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
aarch64build: clean
|
||||
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
ppcbuild: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
|
||||
ppc64build: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
|
||||
|
||||
armfuzz: clean
|
||||
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
aarch64fuzz: clean
|
||||
ld -v
|
||||
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
ppcfuzz: clean
|
||||
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
ppc64fuzz: clean
|
||||
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
.PHONY: cxxtest
|
||||
cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
|
||||
cxxtest: clean
|
||||
$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)" # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
|
||||
gpptest: clean
|
||||
CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
gcc5test: clean
|
||||
gcc-5 -v
|
||||
@@ -254,165 +108,84 @@ gcc6test: clean
|
||||
gcc-6 -v
|
||||
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
|
||||
|
||||
clangtest: clean
|
||||
clang -v
|
||||
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
|
||||
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
aarch64test:
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
|
||||
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
ppctest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
|
||||
|
||||
ppc64test: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
|
||||
|
||||
arm-ppc-compilation:
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
|
||||
|
||||
regressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest
|
||||
|
||||
uasanregressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
|
||||
|
||||
msanregressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
|
||||
|
||||
# run UBsan with -fsanitize-recover=signed-integer-overflow
|
||||
# due to a bug in UBsan when doing pointer subtraction
|
||||
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
|
||||
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
|
||||
|
||||
asan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
msan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
|
||||
|
||||
msan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason
|
||||
|
||||
asan32: clean
|
||||
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
|
||||
|
||||
uasan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
tsan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
|
||||
|
||||
apt-install:
|
||||
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
|
||||
|
||||
apt-add-repo:
|
||||
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
|
||||
sudo apt-get update -y -qq
|
||||
|
||||
ppcinstall:
|
||||
APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
|
||||
|
||||
arminstall:
|
||||
APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
|
||||
|
||||
valgrindinstall:
|
||||
APT_PACKAGES="valgrind" $(MAKE) apt-install
|
||||
|
||||
libc6install:
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc7install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc8install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
|
||||
|
||||
gpp6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
|
||||
|
||||
clang38install:
|
||||
APT_PACKAGES="clang-3.8" $(MAKE) apt-install
|
||||
|
||||
# Ubuntu 14.04 ships a too-old lz4
|
||||
lz4install:
|
||||
[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles"
|
||||
endif
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# target specific tests
|
||||
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||
cmakebuild:
|
||||
cmaketest:
|
||||
cmake --version
|
||||
$(RM) -r $(BUILDIR)/cmake/build
|
||||
mkdir $(BUILDIR)/cmake/build
|
||||
cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build -j4;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build install;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
|
||||
cd $(BUILDIR)/cmake/build; ctest -V -L Medium
|
||||
cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||
|
||||
c89build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c89 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
c90test: clean
|
||||
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
||||
|
||||
gnu90build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
|
||||
gnu90test: clean
|
||||
CFLAGS="-std=gnu90" $(MAKE) all
|
||||
|
||||
c99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c99 -Werror" $(MAKE) allmost
|
||||
c99test: clean
|
||||
CFLAGS="-std=c99" $(MAKE) allmost
|
||||
|
||||
gnu99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
|
||||
gnu99test: clean
|
||||
CFLAGS="-std=gnu99" $(MAKE) all
|
||||
|
||||
c11build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c11 -Werror" $(MAKE) allmost
|
||||
c11test: clean
|
||||
CFLAGS="-std=c11" $(MAKE) allmost
|
||||
|
||||
bmix64build: clean
|
||||
$(CC) -v
|
||||
bmix64test: clean
|
||||
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
bmix32build: clean
|
||||
$(CC) -v
|
||||
bmix32test: clean
|
||||
CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
bmi32build: clean
|
||||
$(CC) -v
|
||||
bmi32test: clean
|
||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
# static analyzer test uses clang's scan-build
|
||||
# does not analyze zlibWrapper, due to detected issues in zlib source code
|
||||
staticAnalyze: SCANBUILD ?= scan-build
|
||||
staticAnalyze:
|
||||
$(CC) -v
|
||||
CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
|
||||
staticAnalyze: clean
|
||||
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
|
||||
endif
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
v1.1.3
|
||||
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
|
||||
cli : new : experimental target `make zstdmt`, with multi-threading support
|
||||
cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell
|
||||
cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
|
||||
cli : fix zstdless on Mac OS-X, by Andrew Janke
|
||||
cli : fix #232 "compress non-files"
|
||||
dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
|
||||
API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
|
||||
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
|
||||
API : new : ZDICT_finalizeDictionary()
|
||||
API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
|
||||
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
|
||||
build : support for Solaris target, by Przemyslaw Skibinski
|
||||
doc : clarified specification, by Andrew Purcell
|
||||
|
||||
v1.1.2
|
||||
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
|
||||
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
|
||||
API : zbuff : changed : prototypes now generate deprecation warnings
|
||||
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
|
||||
lib : changed : only public ZSTD_ symbols are now exposed
|
||||
lib : changed : reduced usage of stack memory
|
||||
lib : fixed : several corner case bugs, by Nick Terrell
|
||||
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
|
||||
cli : new : preserve file attributes
|
||||
cli : new : added zstdless and zstdgrep tools
|
||||
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
|
||||
cli : fixed : zstdcat
|
||||
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
|
||||
install : better compatibility with FreeBSD, by Dimitry Andric
|
||||
source tree : changed : zbuff source files moved to lib/deprecated
|
||||
|
||||
v1.1.1
|
||||
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
|
||||
New : doc/zstd_manual.html, by Przemyslaw Skibinski
|
||||
Improved : slightly better compression ratio at --ultra levels (>= 20)
|
||||
Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
|
||||
Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
|
||||
Added : example/multiple_streaming_compression.c
|
||||
Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
|
||||
Updated man page
|
||||
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
|
||||
|
||||
v1.1.0
|
||||
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
||||
added : NetBSD install target (#338)
|
||||
Improved : speed for batches of small files
|
||||
Improved : speed of zlib wrapper, by Przemyslaw Skibinski
|
||||
Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
|
||||
Fixed : CLI -d output to stdout by default when input is stdin (#322)
|
||||
Fixed : CLI correctly detects console on Mac OS-X
|
||||
Fixed : CLI supports recursive mode `-r` on Mac OS-X
|
||||
Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
|
||||
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
|
||||
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
|
||||
Fixed : zstd-pgo, reported by octoploid (#329)
|
||||
|
||||
v1.0.0
|
||||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.1
|
||||
New streaming API
|
||||
Changed : --ultra now enables levels beyond 19
|
||||
Changed : -i# now selects benchmark time in second
|
||||
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
|
||||
Fixed : speed regression on specific patterns (#272)
|
||||
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
|
||||
Fixed : ICC compilation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.0
|
||||
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
|
||||
New : Build on FreeBSD and DragonFly, thanks to JrMarino
|
||||
Changed : modified API : ZSTD_compressEnd()
|
||||
Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
|
||||
Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
|
||||
Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
|
||||
Fixed : checksum correctly checked in single-pass mode
|
||||
Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
|
||||
Modified : minor compression level adaptations
|
||||
Updated : compression format specification to v0.2.0
|
||||
changed : zstd.h moved to /lib directory
|
||||
|
||||
v0.7.5
|
||||
Transition version, supporting decoding of v0.8.x
|
||||
|
||||
v0.7.4
|
||||
Added : homebrew for Mac, by Daniel Cade
|
||||
Added : more examples
|
||||
Fixed : segfault when using small dictionaries, reported by Felix Handte
|
||||
Modified : default compression level for CLI is now 3
|
||||
Updated : specification, to v0.1.1
|
||||
|
||||
v0.7.3
|
||||
New : compression format specification
|
||||
New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
|
||||
New : `ZSTD_getDecompressedSize()`
|
||||
New : OpenBSD target, by Juan Francisco Cantero Hurtado
|
||||
New : `examples` directory
|
||||
fixed : dictBuilder using HC levels, reported by Bartosz Taudul
|
||||
fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
|
||||
fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
|
||||
modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
|
||||
modified : legacy functions no longer need magic number
|
||||
|
||||
v0.7.2
|
||||
fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
|
||||
fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
|
||||
fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
|
||||
|
||||
v0.7.1
|
||||
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
|
||||
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
|
||||
fixed : corruption issue, reported by cj
|
||||
modified : checksum enabled by default in command line mode
|
||||
|
||||
v0.7.0
|
||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||
New : Command `--rm`, to remove source file after successful de/compression
|
||||
New : Visual build scripts, by Christophe Chevalier
|
||||
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
||||
New : Frame checksum support
|
||||
New : Support pass-through mode (when using `-df`)
|
||||
API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
|
||||
API : create dictionary files from custom content, by Giuseppe Ottaviano
|
||||
API : support for custom malloc/free functions
|
||||
New : controllable Dictionary ID
|
||||
New : Support for skippable frames
|
||||
|
||||
v0.6.1
|
||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||
New : Ability to compile compressor / decompressor separately
|
||||
Changed : new lib directory structure
|
||||
Fixed : Legacy codec v0.5 compatible with dictionary decompression
|
||||
Fixed : Decoder corruption error (#173)
|
||||
Fixed : null-string roundtrip (#176)
|
||||
New : benchmark mode can select directory as input
|
||||
Experimental : midipix support, VMS support
|
||||
|
||||
v0.6.0
|
||||
Stronger high compression modes, thanks to Przemyslaw Skibinski
|
||||
API : ZSTD_getFrameParams() provides size of decompressed content
|
||||
New : highest compression modes require `--ultra` command to fully unleash their capacity
|
||||
Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
|
||||
|
||||
v0.5.1
|
||||
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
|
||||
Changed : Dictionary builder integrated into libzstd and zstd cli
|
||||
Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
|
||||
Fix : high compression modes for big-endian platforms
|
||||
New : zstd cli : `-t` | `--test` command
|
||||
|
||||
v0.5.0
|
||||
New : dictionary builder utility
|
||||
Changed : streaming & dictionary API
|
||||
Improved : better compression of small data
|
||||
|
||||
v0.4.7
|
||||
Improved : small compression speed improvement in HC mode
|
||||
Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
|
||||
fix : bt search bug
|
||||
|
||||
v0.4.6
|
||||
fix : fast compression mode on Windows
|
||||
New : cmake configuration file, thanks to Artyom Dymchenko
|
||||
Improved : high compression mode on repetitive data
|
||||
New : block-level API
|
||||
New : ZSTD_duplicateCCtx()
|
||||
|
||||
v0.4.5
|
||||
new : -m/--multiple : compress/decompress multiple files
|
||||
|
||||
v0.4.4
|
||||
Fixed : high compression modes for Windows 32 bits
|
||||
new : external dictionary API extended to buffered mode and accessible through command line
|
||||
new : windows DLL project, thanks to Christophe Chevalier
|
||||
|
||||
v0.4.3 :
|
||||
new : external dictionary API
|
||||
new : zstd-frugal
|
||||
|
||||
v0.4.2 :
|
||||
Generic minor improvements for small blocks
|
||||
Fixed : big-endian compatibility, by Peter Harris (#85)
|
||||
|
||||
v0.4.1
|
||||
Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
|
||||
removed `zstd.c`
|
||||
|
||||
v0.4.0
|
||||
Command line utility compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
|
||||
v0.3.6
|
||||
small blocks params
|
||||
|
||||
v0.3.5
|
||||
minor generic compression improvements
|
||||
|
||||
v0.3.4
|
||||
Faster fast cLevels
|
||||
|
||||
v0.3.3
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3.2
|
||||
Fixed Visual Studio
|
||||
|
||||
v0.3.1 :
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3
|
||||
HC mode : compression levels 2-26
|
||||
|
||||
v0.2.2
|
||||
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
|
||||
|
||||
v0.2.1
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
|
||||
v0.2.0
|
||||
**Breaking format change**
|
||||
Faster decompression speed
|
||||
Can still decode v0.1 format
|
||||
|
||||
v0.1.3
|
||||
fix uninitialization warning, reported by Evan Nemerson
|
||||
|
||||
v0.1.2
|
||||
frame concatenation support
|
||||
|
||||
v0.1.1
|
||||
fix compression bug
|
||||
detects write-flush errors
|
||||
|
||||
v0.1.0
|
||||
first release
|
||||
@@ -0,0 +1,33 @@
|
||||
Additional Grant of Patent Rights Version 2
|
||||
|
||||
"Software" means the Zstandard software distributed by Facebook, Inc.
|
||||
|
||||
Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
|
||||
("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
|
||||
(subject to the termination provision below) license under any Necessary
|
||||
Claims, to make, have made, use, sell, offer to sell, import, and otherwise
|
||||
transfer the Software. For avoidance of doubt, no license is granted under
|
||||
Facebook’s rights in any patent claims that are infringed by (i) modifications
|
||||
to the Software made by you or any third party or (ii) the Software in
|
||||
combination with any software or other technology.
|
||||
|
||||
The license granted hereunder will terminate, automatically and without notice,
|
||||
if you (or any of your subsidiaries, corporate affiliates or agents) initiate
|
||||
directly or indirectly, or take a direct financial interest in, any Patent
|
||||
Assertion: (i) against Facebook or any of its subsidiaries or corporate
|
||||
affiliates, (ii) against any party if such Patent Assertion arises in whole or
|
||||
in part from any software, technology, product or service of Facebook or any of
|
||||
its subsidiaries or corporate affiliates, or (iii) against any party relating
|
||||
to the Software. Notwithstanding the foregoing, if Facebook or any of its
|
||||
subsidiaries or corporate affiliates files a lawsuit alleging patent
|
||||
infringement against you in the first instance, and you respond by filing a
|
||||
patent infringement counterclaim in that lawsuit against that party that is
|
||||
unrelated to the Software, the license granted hereunder will not terminate
|
||||
under section (i) of this paragraph due to such counterclaim.
|
||||
|
||||
A "Necessary Claim" is a claim of a patent owned by Facebook that is
|
||||
necessarily infringed by the Software standing alone.
|
||||
|
||||
A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
|
||||
or contributory infringement or inducement to infringe any patent, including a
|
||||
cross-claim or counterclaim.
|
||||
@@ -1,100 +1,62 @@
|
||||
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
It is provided as an open-source BSD-licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst` and `.gz` files.
|
||||
For other programming languages,
|
||||
you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
||||
Should your project require another programming language,
|
||||
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|Branch |Status |
|
||||
|------------|---------|
|
||||
|master | [](https://travis-ci.org/facebook/zstd) |
|
||||
|dev | [](https://travis-ci.org/facebook/zstd) |
|
||||
|
||||
**Development branch status:**
|
||||
|
||||
[![Build Status][travisDevBadge]][travisLink]
|
||||
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
||||
[![Build status][CircleDevBadge]][CircleLink]
|
||||
[![Build status][CirrusDevBadge]][CirrusLink]
|
||||
[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
|
||||
|
||||
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
||||
[travisLink]: https://travis-ci.org/facebook/zstd
|
||||
[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
|
||||
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
|
||||
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
||||
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
||||
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
|
||||
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
|
||||
[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
|
||||
[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
|
||||
|
||||
## Benchmarks
|
||||
|
||||
For reference, several fast compression algorithms were tested and compared
|
||||
on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
|
||||
with a Core i9-9900K CPU @ 5.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
compiled with [gcc] 9.3.0,
|
||||
on the [Silesia compression corpus].
|
||||
As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.4.0, with the [Silesia compression corpus].
|
||||
|
||||
[lzbench]: https://github.com/inikep/lzbench
|
||||
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||
[gcc]: https://gcc.gnu.org/
|
||||
|
||||
| Compressor name | Ratio | Compression| Decompress.|
|
||||
| --------------- | ------| -----------| ---------- |
|
||||
| **zstd 1.4.5 -1** | 2.884 | 500 MB/s | 1660 MB/s |
|
||||
| zlib 1.2.11 -1 | 2.743 | 90 MB/s | 400 MB/s |
|
||||
| brotli 1.0.7 -0 | 2.703 | 400 MB/s | 450 MB/s |
|
||||
| **zstd 1.4.5 --fast=1** | 2.434 | 570 MB/s | 2200 MB/s |
|
||||
| **zstd 1.4.5 --fast=3** | 2.312 | 640 MB/s | 2300 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 560 MB/s | 710 MB/s |
|
||||
| **zstd 1.4.5 --fast=5** | 2.178 | 700 MB/s | 2420 MB/s |
|
||||
| lzo1x 2.10 -1 | 2.106 | 690 MB/s | 820 MB/s |
|
||||
| lz4 1.9.2 | 2.101 | 740 MB/s | 4530 MB/s |
|
||||
| **zstd 1.4.5 --fast=7** | 2.096 | 750 MB/s | 2480 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 410 MB/s | 860 MB/s |
|
||||
| snappy 1.1.8 | 2.073 | 560 MB/s | 1790 MB/s |
|
||||
|
||||
[zlib]: http://www.zlib.net/
|
||||
| Name | Ratio | C.speed | D.speed |
|
||||
|-------------------------|-------|--------:|--------:|
|
||||
| | | MB/s | MB/s |
|
||||
| **zstd 0.8.2 -1** |**2.877**| **330** | **940** |
|
||||
| [zlib] 1.2.8 deflate -1 | 2.730 | 95 | 360 |
|
||||
| brotli 0.4 -0 | 2.708 | 320 | 375 |
|
||||
| QuickLZ 1.5 | 2.237 | 510 | 605 |
|
||||
| LZO 2.09 | 2.106 | 610 | 870 |
|
||||
| [LZ4] r131 | 2.101 | 620 | 3100 |
|
||||
| Snappy 1.1.3 | 2.091 | 480 | 1600 |
|
||||
| LZF 3.6 | 2.077 | 375 | 790 |
|
||||
|
||||
[zlib]:http://www.zlib.net/
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
The negative compression levels, specified with `--fast=#`,
|
||||
offer faster compression and decompression speed in exchange for some loss in
|
||||
compression ratio compared to level 1, as seen in the table above.
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increments.
|
||||
Decompression speed is preserved and remains roughly the same at all settings,
|
||||
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
|
||||
The following tests were run
|
||||
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)
|
||||
with a Core i7-6700K CPU @ 4.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
compiled with [gcc] 7.3.0,
|
||||
on the [Silesia compression corpus].
|
||||
The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus].
|
||||
|
||||
Compression Speed vs Ratio | Decompression Speed
|
||||
---------------------------|--------------------
|
||||
 | 
|
||||
 | 
|
||||
|
||||
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
|
||||
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
||||
Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
|
||||
For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
|
||||
|
||||
|
||||
## The case for Small Data compression
|
||||
### The case for Small Data compression
|
||||
|
||||
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
||||
|
||||
The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon.
|
||||
|
||||
To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data.
|
||||
Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
|
||||
Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
|
||||
Using this dictionary, the compression ratio achievable on small data improves dramatically.
|
||||
|
||||
The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
|
||||
It consists of roughly 10K records weighing about 1KB each.
|
||||
The following example uses the `github-users` [sample set](https://www.dropbox.com/s/mnktkomhkjbf1i2/github_users.tar.zst?dl=0), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
|
||||
It consists of roughly 10K records weighting about 1KB each.
|
||||
|
||||
Compression Ratio | Compression Speed | Decompression Speed
|
||||
------------------|-------------------|--------------------
|
||||
@@ -107,93 +69,66 @@ Training works if there is some correlation in a family of small data samples. T
|
||||
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
||||
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
### Dictionary compression How To:
|
||||
#### Dictionary compression How To :
|
||||
|
||||
1. Create the dictionary
|
||||
1) Create the dictionary
|
||||
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
|
||||
2. Compress with dictionary
|
||||
2) Compress with dictionary
|
||||
|
||||
`zstd -D dictionaryName FILE`
|
||||
`zstd -D dictionaryName FILE`
|
||||
|
||||
3. Decompress with dictionary
|
||||
3) Decompress with dictionary
|
||||
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
|
||||
|
||||
## Build instructions
|
||||
### Build
|
||||
|
||||
### Makefile
|
||||
Once you have the repository cloned, there are multiple ways provided to build Zstandard.
|
||||
|
||||
If your system is compatible with standard `make` (or `gmake`),
|
||||
invoking `make` in root directory will generate `zstd` cli in root directory.
|
||||
#### Makefile
|
||||
|
||||
Other available options include:
|
||||
- `make install` : create and install zstd cli, library and man pages
|
||||
- `make check` : create and run `zstd`, tests its behavior on local platform
|
||||
If your system is compatible with a standard `make` (or `gmake`) binary generator,
|
||||
you can simply run it at the root directory.
|
||||
It will generate `zstd` within root directory.
|
||||
|
||||
### cmake
|
||||
Other available options include :
|
||||
- `make install` : create and install zstd binary, library and man page
|
||||
- `make test` : create and run `zstd` and test tools on local platform
|
||||
|
||||
#### cmake
|
||||
|
||||
A `cmake` project generator is provided within `build/cmake`.
|
||||
It can generate Makefiles or other build scripts
|
||||
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||
|
||||
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||
#### Visual (Windows)
|
||||
|
||||
### Meson
|
||||
|
||||
A Meson project is provided within [`build/meson`](build/meson). Follow
|
||||
build instructions in that directory.
|
||||
|
||||
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
||||
example about how Meson is used to build this project.
|
||||
|
||||
Note that default build type is **release**.
|
||||
|
||||
### VCPKG
|
||||
You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
|
||||
|
||||
git clone https://github.com/Microsoft/vcpkg.git
|
||||
cd vcpkg
|
||||
./bootstrap-vcpkg.sh
|
||||
./vcpkg integrate install
|
||||
./vcpkg install zstd
|
||||
|
||||
The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
|
||||
If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
|
||||
|
||||
### Visual Studio (Windows)
|
||||
|
||||
Going into `build` directory, you will find additional possibilities:
|
||||
- Projects for Visual Studio 2005, 2008 and 2010.
|
||||
+ VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
|
||||
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
||||
Going into `build` directory, you will find additional possibilities :
|
||||
- Projects for Visual Studio 2005, 2008 and 2010
|
||||
+ VS2010 project is compatible with VS2012, VS2013 and VS2015
|
||||
- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
|
||||
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
||||
|
||||
### Buck
|
||||
|
||||
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
||||
The output binary will be in `buck-out/gen/programs/`.
|
||||
### Status
|
||||
|
||||
## Testing
|
||||
|
||||
You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory.
|
||||
Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary.
|
||||
For information on CI testing, please refer to TESTING.md
|
||||
|
||||
## Status
|
||||
|
||||
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
||||
Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amounts of data in multiple formats and use cases.
|
||||
Zstandard is considered safe for production environments.
|
||||
|
||||
## License
|
||||
### License
|
||||
|
||||
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
||||
Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS).
|
||||
|
||||
## Contributing
|
||||
### Contributing
|
||||
|
||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
||||
The "dev" branch is the one where all contributions will be merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
|
||||
Direct commit to "master" are not permitted.
|
||||
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
||||
|
||||
### Miscellaneous
|
||||
|
||||
Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
Testing
|
||||
=======
|
||||
|
||||
Zstandard CI testing is split up into three sections:
|
||||
short, medium, and long tests.
|
||||
|
||||
Short Tests
|
||||
-----------
|
||||
Short tests run on CircleCI for new commits on every branch and pull request.
|
||||
They consist of the following tests:
|
||||
- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
|
||||
- Compilation on various versions of gcc, clang, and g++
|
||||
- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
|
||||
|
||||
Medium Tests
|
||||
------------
|
||||
Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
|
||||
They consist of the following tests:
|
||||
- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
|
||||
Msan on x86_64
|
||||
- `tests/playTests.sh --test-large-data`
|
||||
- Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
|
||||
- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
|
||||
- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
|
||||
- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
|
||||
|
||||
Long Tests
|
||||
----------
|
||||
Long tests run on all commits to `master` branch,
|
||||
and once a day on the current version of `dev` branch,
|
||||
on TravisCI.
|
||||
They consist of the following tests:
|
||||
- Entire test suite (including fuzzers and some other specialized tests) on:
|
||||
- x86_64 and x86 with UBsan and Asan
|
||||
- x86_64 with Msan
|
||||
- ARM, AArch64, PowerPC, and PowerPC64
|
||||
- Streaming mode fuzzer with Tsan (for the `zstdmt` testing)
|
||||
- ZlibWrapper tests, including under valgrind
|
||||
- Versions test (ensuring `zstd` can decode files from all previous versions)
|
||||
- `pzstd` with asan and tsan, as well as in 32-bits mode
|
||||
- Testing `zstd` with legacy mode off
|
||||
- Testing `zbuff` (old streaming API)
|
||||
- Entire test suite and make install on macOS
|
||||
+109
-220
@@ -1,110 +1,87 @@
|
||||
# Following tests are run _only_ on master branch
|
||||
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
|
||||
# or a branch `visual*`, they will intentionnally trigger `master` tests
|
||||
version: 1.0.{build}
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
|
||||
PLATFORM: "mingw64"
|
||||
- COMPILER: "gcc"
|
||||
MAKE_PARAMS: "make test"
|
||||
PLATFORM: "mingw32"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Debug"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Debug"
|
||||
PLATFORM: "Win32"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Release"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Release"
|
||||
PLATFORM: "Win32"
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- appveyorTest
|
||||
- /visual*/
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib"
|
||||
ARTIFACT: "true"
|
||||
BUILD: "true"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x86"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static"
|
||||
ARTIFACT: "true"
|
||||
BUILD: "true"
|
||||
- COMPILER: "clang"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
|
||||
BUILD: "true"
|
||||
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: ""
|
||||
TEST: "cmake"
|
||||
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "x64"
|
||||
CONFIGURATION: "Debug"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "Win32"
|
||||
CONFIGURATION: "Debug"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "x64"
|
||||
CONFIGURATION: "Release"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "Win32"
|
||||
CONFIGURATION: "Release"
|
||||
|
||||
install:
|
||||
install:
|
||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET PATH_ORIGINAL=%PATH%
|
||||
- if [%HOST%]==[mingw] (
|
||||
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
|
||||
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
|
||||
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
|
||||
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
|
||||
)
|
||||
- IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
|
||||
SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
|
||||
- MKDIR bin
|
||||
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
|
||||
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
|
||||
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
|
||||
COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
|
||||
COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
|
||||
) else (
|
||||
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
|
||||
)
|
||||
|
||||
build_script:
|
||||
- if [%HOST%]==[mingw] (
|
||||
( if [%PLATFORM%]==[x64] (
|
||||
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
|
||||
) else if [%PLATFORM%]==[x86] (
|
||||
SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
|
||||
) )
|
||||
build_script:
|
||||
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
|
||||
- if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
|
||||
- if [%PLATFORM%]==[mingw64] (
|
||||
make clean &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building clang &&
|
||||
ECHO *** &&
|
||||
ECHO make %CLANG_PARAMS% &&
|
||||
make %CLANG_PARAMS% &&
|
||||
COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building cmake for %PLATFORM% &&
|
||||
ECHO *** &&
|
||||
mkdir build\cmake\build &&
|
||||
cd build\cmake\build &&
|
||||
cmake -G "Visual Studio 14 2015 Win64" .. &&
|
||||
cd ..\..\.. &&
|
||||
make clean &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building pzstd for %PLATFORM% &&
|
||||
ECHO *** &&
|
||||
make -C contrib\pzstd googletest-mingw64 &&
|
||||
make -C contrib\pzstd pzstd.exe &&
|
||||
make -C contrib\pzstd tests &&
|
||||
make -C contrib\pzstd check &&
|
||||
make -C contrib\pzstd clean
|
||||
)
|
||||
- if [%HOST%]==[mingw] if [%BUILD%]==[true] (
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
ECHO *** &&
|
||||
ECHO *** Building %PLATFORM% &&
|
||||
ECHO *** &&
|
||||
make -v &&
|
||||
sh -c "%COMPILER% -v" &&
|
||||
ECHO Building zlib to static link &&
|
||||
SET "CC=%COMPILER%" &&
|
||||
sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
|
||||
sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
|
||||
ECHO Building zstd &&
|
||||
SET "CPPFLAGS=-I../../zlib" &&
|
||||
SET "LDFLAGS=../../zlib/libz.a" &&
|
||||
sh -c "%SCRIPT%" &&
|
||||
( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
|
||||
ECHO Creating artifacts &&
|
||||
ECHO %cd% &&
|
||||
lib\dll\example\build_package.bat &&
|
||||
make -C programs DEBUGFLAGS= clean zstd &&
|
||||
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
||||
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
||||
cp zstd.exe ..\bin\zstd.exe &&
|
||||
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
||||
cd zstd &&
|
||||
git archive --format=tar master -o zstd-src.tar &&
|
||||
..\zstd -19 zstd-src.tar &&
|
||||
appveyor PushArtifact zstd-src.tar.zst &&
|
||||
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
||||
appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
|
||||
cd ..\..\bin\ &&
|
||||
7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
|
||||
appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
|
||||
)
|
||||
cc -v &&
|
||||
ECHO %MAKE_PARAMS% &&
|
||||
sh -c %MAKE_PARAMS%
|
||||
)
|
||||
- if [%HOST%]==[visual] (
|
||||
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
|
||||
COPY programs\zstd.exe bin\zstd.exe &&
|
||||
appveyor PushArtifact bin\zstd.exe
|
||||
)
|
||||
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] (
|
||||
COPY programs\zstd.exe bin\zstd32.exe &&
|
||||
appveyor PushArtifact bin\zstd32.exe
|
||||
)
|
||||
- if [%COMPILER%]==[gcc] make clean
|
||||
- if [%COMPILER%]==[visual] (
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
|
||||
ECHO *** &&
|
||||
@@ -155,135 +132,47 @@
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
|
||||
)
|
||||
|
||||
test_script:
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET "CC=gcc"
|
||||
- SET "CXX=g++"
|
||||
- if [%TEST%]==[cmake] (
|
||||
mkdir build\cmake\build &&
|
||||
cd build\cmake\build &&
|
||||
cmake -G "Visual Studio 14 2015 Win64" .. &&
|
||||
cd ..\..\.. &&
|
||||
make clean
|
||||
- SET FUZZERTEST=-T1mn
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
if [%PLATFORM%]==[mingw64] tests\fuzzer_clang.exe %FUZZERTEST%
|
||||
)
|
||||
- SET "FUZZERTEST=-T30s"
|
||||
- if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||
CD tests &&
|
||||
SET ZSTD_BIN=./zstd.exe&&
|
||||
SET DATAGEN_BIN=./datagen.exe&&
|
||||
SET ZSTD=./zstd.exe &&
|
||||
sh -e playTests.sh --test-large-data &&
|
||||
fullbench.exe -i1 &&
|
||||
fullbench.exe -i1 -P0 &&
|
||||
fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
)
|
||||
|
||||
artifacts:
|
||||
- path: bin\zstd.exe
|
||||
- path: bin\zstd32.exe
|
||||
|
||||
# The following tests are for regular pushes
|
||||
# into `dev` or some feature branch
|
||||
# There run less tests, for shorter feedback loop
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "cygwin"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x86"
|
||||
SCRIPT: "CFLAGS=-Werror make -j allzstd"
|
||||
- COMPILER: "clang"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
||||
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "x64"
|
||||
CONFIGURATION: "Debug"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "Win32"
|
||||
CONFIGURATION: "Debug"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "x64"
|
||||
CONFIGURATION: "Release"
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
PLATFORM: "Win32"
|
||||
CONFIGURATION: "Release"
|
||||
|
||||
install:
|
||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET PATH_ORIGINAL=%PATH%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
ECHO Installing Cygwin Packages &&
|
||||
C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
|
||||
gcc-g++,^
|
||||
gcc,^
|
||||
cmake,^
|
||||
make
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
|
||||
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
|
||||
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
|
||||
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
|
||||
)
|
||||
- IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
|
||||
SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
|
||||
)
|
||||
|
||||
build_script:
|
||||
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
set CHERE_INVOKING=yes &&
|
||||
set CC=%COMPILER% &&
|
||||
C:\cygwin64\bin\bash --login -c "
|
||||
set -e;
|
||||
cd build/cmake;
|
||||
CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
|
||||
make -j4;
|
||||
ctest -V -L Medium;
|
||||
"
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
( if [%PLATFORM%]==[x64] (
|
||||
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
|
||||
) else if [%PLATFORM%]==[x86] (
|
||||
SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
|
||||
) ) &&
|
||||
make -v &&
|
||||
sh -c "%COMPILER% -v" &&
|
||||
set "CC=%COMPILER%" &&
|
||||
sh -c "%SCRIPT%"
|
||||
)
|
||||
- if [%HOST%]==[visual] (
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
|
||||
MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
|
||||
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
|
||||
MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
|
||||
)
|
||||
|
||||
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[mingw] (
|
||||
set "CC=%COMPILER%" &&
|
||||
make check
|
||||
)
|
||||
deploy:
|
||||
- provider: GitHub
|
||||
auth_token:
|
||||
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
|
||||
artifact: bin\zstd.exe
|
||||
force_update: true
|
||||
on:
|
||||
branch: autobuild
|
||||
COMPILER: gcc
|
||||
PLATFORM: "mingw64"
|
||||
appveyor_repo_tag: true
|
||||
- provider: GitHub
|
||||
auth_token:
|
||||
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
|
||||
artifact: bin\zstd32.exe
|
||||
force_update: true
|
||||
on:
|
||||
branch: autobuild
|
||||
COMPILER: gcc
|
||||
PLATFORM: "mingw32"
|
||||
appveyor_repo_tag: true
|
||||
|
||||
+6
-19
@@ -8,26 +8,13 @@
|
||||
*.user
|
||||
*.opendb
|
||||
|
||||
VS2008/bin/
|
||||
VS2005/
|
||||
VS2008/
|
||||
VS2010/bin/
|
||||
VS2010/zwrapbench/
|
||||
VS2012/
|
||||
VS2013/
|
||||
VS2015/
|
||||
Studio*
|
||||
VS2012/bin/
|
||||
VS2013/bin/
|
||||
VS2015/bin/
|
||||
|
||||
# CMake
|
||||
cmake/build/
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
CMakeScripts
|
||||
Testing
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
install_manifest.txt
|
||||
compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
build
|
||||
lib
|
||||
!cmake/lib
|
||||
!meson/lib
|
||||
cmake/
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ Projects for various integrated development environments (IDE)
|
||||
|
||||
The following projects are included with the zstd distribution:
|
||||
- `cmake` - CMake project contributed by Artyom Dymchenko
|
||||
- `VS2005` - Visual Studio 2005 Project (this project has been moved to the contrib directory and will no longer be supported)
|
||||
- `VS2005` - Visual Studio 2005 project
|
||||
- `VS2008` - Visual Studio 2008 project
|
||||
- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
|
||||
- `VS_scripts` - command line scripts prepared for Visual Studio compilation without IDE
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -121,7 +121,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -196,7 +196,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -274,7 +274,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -44,7 +44,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -120,7 +120,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -194,7 +194,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -271,7 +271,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -328,16 +328,20 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
@@ -349,99 +353,27 @@
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\pool.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\threading.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
RelativePath="..\..\..\tests\fullbench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\huf_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\tests\fullbench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.c"
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
@@ -450,10 +382,6 @@
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||
>
|
||||
@@ -490,56 +418,20 @@
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_internal.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.h"
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
|
||||
@@ -44,8 +44,8 @@
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -120,8 +120,8 @@
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -194,8 +194,8 @@
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -271,8 +271,8 @@
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -328,14 +328,6 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
@@ -344,10 +336,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -364,10 +352,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -412,50 +396,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
@@ -514,10 +458,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -534,48 +474,12 @@
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.h"
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -122,7 +122,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -197,7 +197,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -275,7 +275,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -333,19 +333,7 @@
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchzstd.c"
|
||||
RelativePath="..\..\..\programs\bench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
@@ -360,10 +348,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -372,10 +356,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -424,30 +404,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -484,26 +444,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
@@ -558,10 +498,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -582,6 +518,10 @@
|
||||
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
@@ -618,46 +558,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -121,7 +121,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -195,7 +195,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -272,7 +272,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -332,10 +332,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -352,10 +348,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -396,50 +388,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -526,10 +478,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -546,48 +494,12 @@
|
||||
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_double_fast.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_lazy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_ldm.h"
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
|
||||
@@ -155,7 +155,6 @@
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\datagencli.c" />
|
||||
</ItemGroup>
|
||||
@@ -165,4 +164,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -166,16 +166,12 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -186,4 +182,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -156,61 +156,29 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_opt.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\pool.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\threading.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
|
||||
@@ -67,22 +67,22 @@
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
@@ -90,7 +90,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -105,7 +105,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -122,7 +122,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -142,7 +142,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -156,10 +156,8 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
@@ -167,25 +165,12 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_opt.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\fuzzer.c" />
|
||||
</ItemGroup>
|
||||
@@ -198,21 +183,10 @@
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
|
||||
@@ -20,10 +20,8 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
@@ -31,24 +29,13 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_opt.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -80,18 +67,8 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ResourceCompile Include="libzstd-dll.rc" />
|
||||
@@ -172,7 +149,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -192,7 +169,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -212,10 +189,11 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -233,13 +211,14 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -20,10 +20,8 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
@@ -31,24 +29,13 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_opt.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -80,43 +67,38 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>libzstd</RootNamespace>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
|
||||
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
@@ -136,36 +118,35 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -185,7 +166,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -205,10 +186,11 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -226,13 +208,14 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
@@ -27,25 +26,13 @@
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_opt.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -55,10 +42,7 @@
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchzstd.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\dibio.c" />
|
||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||
@@ -70,24 +54,13 @@
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
|
||||
@@ -96,7 +69,7 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchzstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\dibio.h" />
|
||||
<ClInclude Include="..\..\..\programs\fileio.h" />
|
||||
@@ -182,7 +155,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -198,7 +171,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -216,7 +189,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -237,7 +210,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
|
||||
+17
-27
@@ -7,58 +7,48 @@ Here are a few command lines for reference :
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build.VS2013.cmd
|
||||
build\build.VS2013.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `bin\Release\{ARCH}\` folder.
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build.generic.cmd VS2013 Win32 Release v120`
|
||||
- x64: `build.generic.cmd VS2013 x64 Release v120`
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Release v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Release v120`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build.generic.cmd VS2013 Win32 Debug v120`
|
||||
- x64: `build.generic.cmd VS2013 x64 Debug v120`
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Debug v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Debug v120`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr140.dll
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build.VS2015.cmd
|
||||
build\build.VS2015.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `bin\Release\{ARCH}\` folder.
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build.generic.cmd VS2015 Win32 Release v140`
|
||||
- x64: `build.generic.cmd VS2015 x64 Release v140`
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Release v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Release v140`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build.generic.cmd VS2015 Win32 Debug v140`
|
||||
- x64: `build.generic.cmd VS2015 x64 Debug v140`
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Debug v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Debug v140`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr120.dll
|
||||
|
||||
This capability is offered through `build.generic.cmd` using proper arguments:
|
||||
You need to invoke `build\build.generic.cmd` with the proper arguments:
|
||||
|
||||
**For Win32**
|
||||
```batch
|
||||
build.generic.cmd VS2015 Win32 Release v120
|
||||
build\build.generic.cmd VS2015 Win32 Release v120
|
||||
```
|
||||
The result of the build will be in the `bin\Release\Win32\` folder.
|
||||
The result of the build will be in the `build\bin\Release\Win32\` folder.
|
||||
|
||||
**For x64**
|
||||
```batch
|
||||
build.generic.cmd VS2015 x64 Release v120
|
||||
build\build.generic.cmd VS2015 x64 Release v120
|
||||
```
|
||||
The result of the build will be in the `bin\Release\x64\` folder.
|
||||
The result of the build will be in the `build\bin\Release\x64\` folder.
|
||||
|
||||
If you want Debug builds, replace `Release` with `Debug`.
|
||||
|
||||
### Build with Visual Studio 2017
|
||||
|
||||
`build.VS2017.cmd`, contributed by [@HaydnTrigg](https://github.com/HaydnTrigg),
|
||||
will build both the `Release Win32` and `Release x64` versions
|
||||
of the first VS2017 variant it finds, in this priority order :
|
||||
Enterprise > Professional > Community
|
||||
|
||||
Alternatively, it's possible to target a specific version,
|
||||
using appropriate script, such as `build.VS2017Enterprise.cmd` for example.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017 Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017 x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Community Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Community x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Enterprise Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Enterprise x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Professional Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Professional x64 Release v141
|
||||
@@ -19,29 +19,19 @@ GOTO build
|
||||
:display_help
|
||||
|
||||
echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
|
||||
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, VS2017, ...)
|
||||
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, ...)
|
||||
echo msbuild_platform: Platform (x64 or Win32)
|
||||
echo msbuild_configuration: VS configuration (Release or Debug)
|
||||
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140, v141)
|
||||
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140)
|
||||
|
||||
EXIT /B 1
|
||||
|
||||
:build
|
||||
|
||||
SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
|
||||
SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
|
||||
IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
|
||||
IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
|
||||
IF %msbuild_version% == VS2017 (
|
||||
IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
|
||||
IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
|
||||
IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
|
||||
)
|
||||
rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ?
|
||||
|
||||
SET project="%~p0\..\VS2010\zstd.sln"
|
||||
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
# cmake working directory
|
||||
cmakeBuild
|
||||
|
||||
# cmake artefacts
|
||||
# cmake producted
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
cmake_uninstall.cmake
|
||||
*.1
|
||||
|
||||
+19
-186
@@ -2,202 +2,35 @@
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR)
|
||||
|
||||
# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
|
||||
# Set and use the newest cmake policies that are validated to work
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
|
||||
if("${CMAKE_MAJOR_VERSION}" LESS 3)
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
|
||||
"${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
else()
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
|
||||
endif()
|
||||
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
|
||||
PROJECT(zstd)
|
||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
|
||||
|
||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
|
||||
OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
# Parse version
|
||||
include(GetZstdLibraryVersion)
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=1)
|
||||
ELSE (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if( CMAKE_MAJOR_VERSION LESS 3 )
|
||||
## Provide cmake 3+ behavior for older versions of cmake
|
||||
project(zstd)
|
||||
set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
|
||||
set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
|
||||
set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
|
||||
set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
|
||||
enable_language(C) # Main library is in C
|
||||
enable_language(CXX) # Testing contributed code also utilizes CXX
|
||||
else()
|
||||
project(zstd
|
||||
VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
|
||||
LANGUAGES C # Main library is in C
|
||||
CXX # Testing contributed code also utilizes CXX
|
||||
)
|
||||
endif()
|
||||
message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
|
||||
set(zstd_HOMEPAGE_URL "http://www.zstd.net")
|
||||
set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
|
||||
|
||||
# Set a default build type if none was specified
|
||||
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
||||
message(STATUS "Setting build type to 'Release' as none was specified.")
|
||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
|
||||
# Set the possible values of build type for cmake-gui
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
|
||||
endif()
|
||||
|
||||
include(GNUInstallDirs)
|
||||
ADD_SUBDIRECTORY(lib)
|
||||
ADD_SUBDIRECTORY(programs)
|
||||
ADD_SUBDIRECTORY(tests)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add extra compilation flags
|
||||
#-----------------------------------------------------------------------------
|
||||
include(AddZstdCompilationFlags)
|
||||
ADD_ZSTD_COMPILATION_FLAGS()
|
||||
INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
|
||||
ADD_EXTRA_COMPILATION_FLAGS()
|
||||
|
||||
# Always hide XXHash symbols
|
||||
add_definitions(-DXXH_NAMESPACE=ZSTD_)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Installation variables
|
||||
#-----------------------------------------------------------------------------
|
||||
message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
|
||||
message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Options
|
||||
#-----------------------------------------------------------------------------
|
||||
|
||||
# Legacy support
|
||||
option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=5)
|
||||
else ()
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=0)
|
||||
endif ()
|
||||
|
||||
# Multi-threading support
|
||||
option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
|
||||
else ()
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
|
||||
endif ()
|
||||
|
||||
option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
|
||||
option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
|
||||
|
||||
# Respect the conventional CMake option for enabling tests if it was specified on the first configure
|
||||
if (BUILD_TESTING)
|
||||
set(ZSTD_BUILD_TESTS_default ON)
|
||||
else()
|
||||
set(ZSTD_BUILD_TESTS_default OFF)
|
||||
endif()
|
||||
option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
|
||||
if (MSVC)
|
||||
option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# External dependencies
|
||||
#-----------------------------------------------------------------------------
|
||||
if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if(CMAKE_USE_PTHREADS_INIT)
|
||||
set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add source directories
|
||||
#-----------------------------------------------------------------------------
|
||||
add_subdirectory(lib)
|
||||
|
||||
option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
|
||||
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build shared library to build zstd CLI")
|
||||
endif ()
|
||||
|
||||
add_subdirectory(programs)
|
||||
endif ()
|
||||
|
||||
if (ZSTD_BUILD_TESTS)
|
||||
enable_testing()
|
||||
if (NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build static library to build tests")
|
||||
endif ()
|
||||
|
||||
add_subdirectory(tests)
|
||||
endif ()
|
||||
|
||||
if (ZSTD_BUILD_CONTRIB)
|
||||
add_subdirectory(contrib)
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add clean-all target
|
||||
#-----------------------------------------------------------------------------
|
||||
add_custom_target(clean-all
|
||||
ADD_CUSTOM_TARGET(clean-all
|
||||
COMMAND ${CMAKE_BUILD_TOOL} clean
|
||||
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
|
||||
)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Generate Package Config files
|
||||
#
|
||||
# This section is based on the boiler plate code from:
|
||||
# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
|
||||
#-----------------------------------------------------------------------------
|
||||
include(CMakePackageConfigHelpers)
|
||||
write_basic_package_version_file(
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
VERSION ${zstd_VERSION}
|
||||
COMPATIBILITY SameMajorVersion
|
||||
)
|
||||
|
||||
# A Package Config file that works from the build directory
|
||||
export(EXPORT zstdExports
|
||||
FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
|
||||
NAMESPACE zstd::
|
||||
)
|
||||
configure_file(zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
|
||||
COPYONLY
|
||||
)
|
||||
|
||||
# A Package Config file that works from the installation directory
|
||||
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
|
||||
install(EXPORT zstdExports
|
||||
FILE zstdTargets.cmake
|
||||
NAMESPACE zstd::
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
install(FILES
|
||||
zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
MACRO(ADD_EXTRA_COMPILATION_FLAGS)
|
||||
include(CheckCXXCompilerFlag)
|
||||
include(CheckCCompilerFlag)
|
||||
if (CMAKE_COMPILER_IS_GNUCXX OR MINGW) #Not only UNIX but also WIN32 for MinGW
|
||||
|
||||
set(POSITION_INDEPENDENT_CODE_FLAG "-fPIC")
|
||||
CHECK_C_COMPILER_FLAG(${POSITION_INDEPENDENT_CODE_FLAG} POSITION_INDEPENDENT_CODE_FLAG_ALLOWED)
|
||||
if (POSITION_INDEPENDENT_CODE_FLAG_ALLOWED)
|
||||
MESSAGE("Compiler flag ${POSITION_INDEPENDENT_CODE_FLAG} allowed")
|
||||
set(ACTIVATE_POSITION_INDEPENDENT_CODE_FLAG "ON" CACHE BOOL "activate -fPIC flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${POSITION_INDEPENDENT_CODE_FLAG} not allowed")
|
||||
endif (POSITION_INDEPENDENT_CODE_FLAG_ALLOWED)
|
||||
|
||||
set(WARNING_UNDEF "-Wundef")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_UNDEF} WARNING_UNDEF_ALLOWED)
|
||||
if (WARNING_UNDEF_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_UNDEF} allowed")
|
||||
set(ACTIVATE_WARNING_UNDEF "ON" CACHE BOOL "activate -Wundef flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_UNDEF} not allowed")
|
||||
endif (WARNING_UNDEF_ALLOWED)
|
||||
|
||||
set(WARNING_SHADOW "-Wshadow")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_SHADOW} WARNING_SHADOW_ALLOWED)
|
||||
if (WARNING_SHADOW_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_SHADOW} allowed")
|
||||
set(ACTIVATE_WARNING_SHADOW "ON" CACHE BOOL "activate -Wshadow flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_SHADOW} not allowed")
|
||||
endif (WARNING_SHADOW_ALLOWED)
|
||||
|
||||
set(WARNING_CAST_ALIGN "-Wcast-align")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_CAST_ALIGN} WARNING_CAST_ALIGN_ALLOWED)
|
||||
if (WARNING_CAST_ALIGN_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_CAST_ALIGN} allowed")
|
||||
set(ACTIVATE_WARNING_CAST_ALIGN "ON" CACHE BOOL "activate -Wcast-align flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_CAST_ALIGN} not allowed")
|
||||
endif (WARNING_CAST_ALIGN_ALLOWED)
|
||||
|
||||
set(WARNING_CAST_QUAL "-Wcast-qual")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_CAST_QUAL} WARNING_CAST_QUAL_ALLOWED)
|
||||
if (WARNING_CAST_QUAL_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_CAST_QUAL} allowed")
|
||||
set(ACTIVATE_WARNING_CAST_QUAL "ON" CACHE BOOL "activate -Wcast-qual flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_CAST_QUAL} not allowed")
|
||||
endif (WARNING_CAST_QUAL_ALLOWED)
|
||||
|
||||
set(WARNING_STRICT_PROTOTYPES "-Wstrict-prototypes")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_STRICT_PROTOTYPES} WARNING_STRICT_PROTOTYPES_ALLOWED)
|
||||
if (WARNING_STRICT_PROTOTYPES_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_STRICT_PROTOTYPES} allowed")
|
||||
set(ACTIVATE_WARNING_STRICT_PROTOTYPES "ON" CACHE BOOL "activate -Wstrict-prototypes flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_STRICT_PROTOTYPES} not allowed")
|
||||
endif (WARNING_STRICT_PROTOTYPES_ALLOWED)
|
||||
|
||||
set(WARNING_ALL "-Wall")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_ALL} WARNING_ALL_ALLOWED)
|
||||
if (WARNING_ALL_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_ALL} allowed")
|
||||
set(ACTIVATE_WARNING_ALL "ON" CACHE BOOL "activate -Wall flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_ALL} not allowed")
|
||||
endif (WARNING_ALL_ALLOWED)
|
||||
|
||||
set(WARNING_EXTRA "-Wextra")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_EXTRA} WARNING_EXTRA_ALLOWED)
|
||||
if (WARNING_EXTRA_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_EXTRA} allowed")
|
||||
set(ACTIVATE_WARNING_EXTRA "ON" CACHE BOOL "activate -Wextra flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_EXTRA} not allowed")
|
||||
endif (WARNING_EXTRA_ALLOWED)
|
||||
|
||||
set(WARNING_FLOAT_EQUAL "-Wfloat-equal")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_FLOAT_EQUAL} WARNING_FLOAT_EQUAL_ALLOWED)
|
||||
if (WARNING_FLOAT_EQUAL_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_FLOAT_EQUAL} allowed")
|
||||
set(ACTIVATE_WARNING_FLOAT_EQUAL "OFF" CACHE BOOL "activate -Wfloat-equal flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_FLOAT_EQUAL} not allowed")
|
||||
endif (WARNING_FLOAT_EQUAL_ALLOWED)
|
||||
|
||||
set(WARNING_SIGN_CONVERSION "-Wsign-conversion")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_SIGN_CONVERSION} WARNING_SIGN_CONVERSION_ALLOWED)
|
||||
if (WARNING_SIGN_CONVERSION_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_SIGN_CONVERSION} allowed")
|
||||
set(ACTIVATE_WARNING_SIGN_CONVERSION "OFF" CACHE BOOL "activate -Wsign-conversion flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_SIGN_CONVERSION} not allowed")
|
||||
endif (WARNING_SIGN_CONVERSION_ALLOWED)
|
||||
|
||||
if (ACTIVATE_POSITION_INDEPENDENT_CODE_FLAG)
|
||||
list(APPEND CMAKE_C_FLAGS ${POSITION_INDEPENDENT_CODE_FLAG})
|
||||
else ()
|
||||
string(REPLACE ${POSITION_INDEPENDENT_CODE_FLAG} "" CMAKE_C_FLAGS "${POSITION_INDEPENDENT_CODE_FLAG}")
|
||||
endif (ACTIVATE_POSITION_INDEPENDENT_CODE_FLAG)
|
||||
|
||||
if (ACTIVATE_WARNING_UNDEF)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_UNDEF})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_UNDEF})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_UNDEF} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_UNDEF} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_UNDEF)
|
||||
|
||||
if (ACTIVATE_WARNING_SHADOW)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_SHADOW})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_SHADOW})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_SHADOW} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_SHADOW} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_SHADOW)
|
||||
|
||||
if (ACTIVATE_WARNING_CAST_QUAL)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_CAST_QUAL})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_CAST_QUAL})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_CAST_QUAL} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_CAST_QUAL} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_CAST_QUAL)
|
||||
|
||||
if (ACTIVATE_WARNING_CAST_ALIGN)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_CAST_ALIGN})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_CAST_ALIGN})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_CAST_ALIGN} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_CAST_ALIGN} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_CAST_ALIGN)
|
||||
|
||||
if (ACTIVATE_WARNING_STRICT_PROTOTYPES)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_STRICT_PROTOTYPES)
|
||||
|
||||
if (ACTIVATE_WARNING_ALL)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_ALL})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_ALL})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_ALL} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_ALL} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_ALL)
|
||||
|
||||
if (ACTIVATE_WARNING_EXTRA)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_EXTRA})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_EXTRA})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_EXTRA} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_EXTRA} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_EXTRA)
|
||||
|
||||
if (ACTIVATE_WARNING_FLOAT_EQUAL)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_FLOAT_EQUAL})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_FLOAT_EQUAL})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_FLOAT_EQUAL} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_FLOAT_EQUAL} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_FLOAT_EQUAL)
|
||||
|
||||
if (ACTIVATE_WARNING_SIGN_CONVERSION)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_SIGN_CONVERSION})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_SIGN_CONVERSION})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_SIGN_CONVERSION} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_SIGN_CONVERSION} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_SIGN_CONVERSION)
|
||||
|
||||
#Set c++11 by default
|
||||
list(APPEND CMAKE_CXX_FLAGS "-std=c++11")
|
||||
|
||||
#Set c99 by default
|
||||
list(APPEND CMAKE_C_FLAGS "-std=c99")
|
||||
|
||||
elseif (MSVC)
|
||||
# Add specific compilation flags for Windows Visual
|
||||
|
||||
set(WARNING_ALL "/Wall")
|
||||
CHECK_C_COMPILER_FLAG(${WARNING_ALL} WARNING_ALL_ALLOWED)
|
||||
if (WARNING_ALL_ALLOWED)
|
||||
MESSAGE("Compiler flag ${WARNING_ALL} allowed")
|
||||
set(ACTIVATE_WARNING_ALL "OFF" CACHE BOOL "activate /Wall flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${WARNING_ALL} not allowed")
|
||||
endif (WARNING_ALL_ALLOWED)
|
||||
|
||||
set(RTC_FLAG "/RTC1")
|
||||
CHECK_C_COMPILER_FLAG(${RTC_FLAG} RTC_FLAG_ALLOWED)
|
||||
if (RTC_FLAG_ALLOWED)
|
||||
MESSAGE("Compiler flag ${RTC_FLAG} allowed")
|
||||
set(ACTIVATE_RTC_FLAG "ON" CACHE BOOL "activate /RTC1 flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${RTC_FLAG} not allowed")
|
||||
endif (RTC_FLAG_ALLOWED)
|
||||
|
||||
set(ZC_FLAG "/Zc:forScope")
|
||||
CHECK_C_COMPILER_FLAG(${ZC_FLAG} ZC_FLAG_ALLOWED)
|
||||
if (ZC_FLAG_ALLOWED)
|
||||
MESSAGE("Compiler flag ${ZC_FLAG} allowed")
|
||||
set(ACTIVATE_ZC_FLAG "ON" CACHE BOOL "activate /Zc:forScope flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${ZC_FLAG} not allowed")
|
||||
endif (ZC_FLAG_ALLOWED)
|
||||
|
||||
set(GD_FLAG "/Gd")
|
||||
CHECK_C_COMPILER_FLAG(${GD_FLAG} GD_FLAG_ALLOWED)
|
||||
if (GD_FLAG_ALLOWED)
|
||||
MESSAGE("Compiler flag ${GD_FLAG} allowed")
|
||||
set(ACTIVATE_GD_FLAG "ON" CACHE BOOL "activate /Gd flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${GD_FLAG} not allowed")
|
||||
endif (GD_FLAG_ALLOWED)
|
||||
|
||||
set(ANALYZE_FLAG "/analyze:stacksize25000")
|
||||
CHECK_C_COMPILER_FLAG(${ANALYZE_FLAG} ANALYZE_FLAG_ALLOWED)
|
||||
if (ANALYZE_FLAG_ALLOWED)
|
||||
MESSAGE("Compiler flag ${ANALYZE_FLAG} allowed")
|
||||
set(ACTIVATE_ANALYZE_FLAG "ON" CACHE BOOL "activate /ANALYZE flag")
|
||||
else ()
|
||||
MESSAGE("Compiler flag ${ANALYZE_FLAG} not allowed")
|
||||
endif (ANALYZE_FLAG_ALLOWED)
|
||||
|
||||
if (ACTIVATE_WARNING_ALL)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${WARNING_ALL})
|
||||
list(APPEND CMAKE_C_FLAGS ${WARNING_ALL})
|
||||
else ()
|
||||
string(REPLACE ${WARNING_ALL} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${WARNING_ALL} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_WARNING_ALL)
|
||||
|
||||
# Only for DEBUG version
|
||||
if (ACTIVATE_RTC_FLAG)
|
||||
list(APPEND CMAKE_CXX_FLAGS_DEBUG ${RTC_FLAG})
|
||||
list(APPEND CMAKE_C_FLAGS_DEBUG ${RTC_FLAG})
|
||||
else ()
|
||||
string(REPLACE ${RTC_FLAG} "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
|
||||
string(REPLACE ${RTC_FLAG} "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
|
||||
endif (ACTIVATE_RTC_FLAG)
|
||||
|
||||
if (ACTIVATE_ZC_FLAG)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${ZC_FLAG})
|
||||
list(APPEND CMAKE_C_FLAGS ${ZC_FLAG})
|
||||
else ()
|
||||
string(REPLACE ${ZC_FLAG} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${ZC_FLAG} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_ZC_FLAG)
|
||||
|
||||
if (ACTIVATE_GD_FLAG)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${GD_FLAG})
|
||||
list(APPEND CMAKE_C_FLAGS ${GD_FLAG})
|
||||
else ()
|
||||
string(REPLACE ${GD_FLAG} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${GD_FLAG} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_GD_FLAG)
|
||||
|
||||
if (ACTIVATE_ANALYZE_FLAG)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${ANALYZE_FLAG})
|
||||
list(APPEND CMAKE_C_FLAGS ${ANALYZE_FLAG})
|
||||
else ()
|
||||
string(REPLACE ${ANALYZE_FLAG} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${ANALYZE_FLAG} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_ANALYZE_FLAG)
|
||||
|
||||
if (MSVC80 OR MSVC90 OR MSVC10 OR MSVC11)
|
||||
# To avoid compiler warning (level 4) C4571, compile with /EHa if you still want
|
||||
# your catch(...) blocks to catch structured exceptions.
|
||||
list(APPEND CMAKE_CXX_FLAGS "/EHa")
|
||||
endif (MSVC80 OR MSVC90 OR MSVC10 OR MSVC11)
|
||||
|
||||
set(MULTITHREADED_COMPILATION "/MP")
|
||||
MESSAGE("Compiler flag ${MULTITHREADED_COMPILATION} allowed")
|
||||
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate /MP flag")
|
||||
|
||||
if (ACTIVATE_MULTITHREADED_COMPILATION)
|
||||
list(APPEND CMAKE_CXX_FLAGS ${MULTITHREADED_COMPILATION})
|
||||
list(APPEND CMAKE_C_FLAGS ${MULTITHREADED_COMPILATION})
|
||||
else ()
|
||||
string(REPLACE ${MULTITHREADED_COMPILATION} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE ${MULTITHREADED_COMPILATION} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
endif (ACTIVATE_MULTITHREADED_COMPILATION)
|
||||
|
||||
#For exceptions
|
||||
list(APPEND CMAKE_CXX_FLAGS "/EHsc")
|
||||
list(APPEND CMAKE_C_FLAGS "/EHsc")
|
||||
|
||||
# UNICODE SUPPORT
|
||||
list(APPEND CMAKE_CXX_FLAGS "/D_UNICODE /DUNICODE")
|
||||
list(APPEND CMAKE_C_FLAGS "/D_UNICODE /DUNICODE")
|
||||
endif ()
|
||||
|
||||
# Remove duplicates compilation flags
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
separate_arguments(${flag_var})
|
||||
list(REMOVE_DUPLICATES ${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
set(${flag_var} "${${flag_var}}" CACHE STRING "common build flags" FORCE)
|
||||
ENDFOREACH (flag_var)
|
||||
|
||||
if (MSVC)
|
||||
# Replace /MT to /MD flag
|
||||
# Replace /O2 to /O3 flag
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
STRING(REGEX REPLACE "/MT" "/MD" ${flag_var} "${${flag_var}}")
|
||||
STRING(REGEX REPLACE "/O2" "/Ox" ${flag_var} "${${flag_var}}")
|
||||
ENDFOREACH (flag_var)
|
||||
endif ()
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STRING "Updated flags" FORCE)
|
||||
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}" CACHE STRING "Updated flags" FORCE)
|
||||
set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}" CACHE STRING "Updated flags" FORCE)
|
||||
|
||||
ENDMACRO(ADD_EXTRA_COMPILATION_FLAGS)
|
||||
@@ -1,78 +0,0 @@
|
||||
include(CheckCXXCompilerFlag)
|
||||
include(CheckCCompilerFlag)
|
||||
|
||||
function(EnableCompilerFlag _flag _C _CXX)
|
||||
string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
|
||||
string(REGEX REPLACE "[^A-Za-z0-9]+" "_" varname "${varname}")
|
||||
string(REGEX REPLACE "^_+" "" varname "${varname}")
|
||||
string(TOUPPER "${varname}" varname)
|
||||
if (_C)
|
||||
CHECK_C_COMPILER_FLAG(${_flag} C_FLAG_${varname})
|
||||
if (C_FLAG_${varname})
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
|
||||
endif ()
|
||||
endif ()
|
||||
if (_CXX)
|
||||
CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
|
||||
if (CXX_FLAG_${varname})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
|
||||
endif ()
|
||||
endif ()
|
||||
endfunction()
|
||||
|
||||
macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
|
||||
#Set c++11 by default
|
||||
EnableCompilerFlag("-std=c++11" false true)
|
||||
#Set c99 by default
|
||||
EnableCompilerFlag("-std=c99" true false)
|
||||
EnableCompilerFlag("-Wall" true true)
|
||||
EnableCompilerFlag("-Wextra" true true)
|
||||
EnableCompilerFlag("-Wundef" true true)
|
||||
EnableCompilerFlag("-Wshadow" true true)
|
||||
EnableCompilerFlag("-Wcast-align" true true)
|
||||
EnableCompilerFlag("-Wcast-qual" true true)
|
||||
EnableCompilerFlag("-Wstrict-prototypes" true false)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
elseif (MSVC) # Add specific compilation flags for Windows Visual
|
||||
|
||||
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
|
||||
if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
|
||||
EnableCompilerFlag("/MP" true true)
|
||||
endif ()
|
||||
|
||||
# UNICODE SUPPORT
|
||||
EnableCompilerFlag("/D_UNICODE" true true)
|
||||
EnableCompilerFlag("/DUNICODE" true true)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Remove duplicates compilation flags
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if( ${flag_var} )
|
||||
separate_arguments(${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
|
||||
if (MSVC AND ZSTD_USE_STATIC_RUNTIME)
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if ( ${flag_var} )
|
||||
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
endif ()
|
||||
|
||||
endmacro()
|
||||
@@ -1,49 +0,0 @@
|
||||
# Find LibLZ4
|
||||
#
|
||||
# Find LibLZ4 headers and library
|
||||
#
|
||||
# Result Variables
|
||||
#
|
||||
# LIBLZ4_FOUND - True if lz4 is found
|
||||
# LIBLZ4_INCLUDE_DIRS - lz4 headers directories
|
||||
# LIBLZ4_LIBRARIES - lz4 libraries
|
||||
# LIBLZ4_VERSION_MAJOR - The major version of lz4
|
||||
# LIBLZ4_VERSION_MINOR - The minor version of lz4
|
||||
# LIBLZ4_VERSION_RELEASE - The release version of lz4
|
||||
# LIBLZ4_VERSION_STRING - version number string (e.g. 1.8.3)
|
||||
#
|
||||
# Hints
|
||||
#
|
||||
# Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library
|
||||
|
||||
set(_LIBLZ4_ROOT_HINTS
|
||||
ENV LZ4_ROOT_DIR)
|
||||
|
||||
find_path( LIBLZ4_INCLUDE_DIR lz4.h
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
find_library( LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
|
||||
if(LIBLZ4_INCLUDE_DIR)
|
||||
file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")
|
||||
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")
|
||||
|
||||
set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
|
||||
unset(LIBLZ4_HEADER_CONTENT)
|
||||
endif()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4 REQUIRED_VARS LIBLZ4_INCLUDE_DIR
|
||||
LIBLZ4_LIBRARY
|
||||
VERSION_VAR LIBLZ4_VERSION_STRING
|
||||
FAIL_MESSAGE "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
|
||||
set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )
|
||||
@@ -1,10 +0,0 @@
|
||||
function(GetZstdLibraryVersion _header _major _minor _patch)
|
||||
# Read file content
|
||||
file(READ ${_header} CONTENT)
|
||||
|
||||
string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}")
|
||||
set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# Cmake contributions
|
||||
|
||||
Contributions to the cmake build configurations are welcome. Please
|
||||
use case sensitivity that matches modern (ie. cmake version 2.6 and above)
|
||||
conventions of using lower-case for commands, and upper-case for
|
||||
variables.
|
||||
|
||||
## How to build
|
||||
|
||||
As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
|
||||
To do this, you can create a new directory and build in it:
|
||||
```sh
|
||||
cd build/cmake
|
||||
mkdir builddir
|
||||
cd builddir
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
Then you can clean all cmake caches by simply delete the new directory:
|
||||
```sh
|
||||
rm -rf build/cmake/builddir
|
||||
```
|
||||
|
||||
And of course, you can directly build in build/cmake:
|
||||
```sh
|
||||
cd build/cmake
|
||||
cmake
|
||||
make
|
||||
```
|
||||
|
||||
To show cmake build options, you can:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -LH ..
|
||||
```
|
||||
|
||||
Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
|
||||
make
|
||||
```
|
||||
|
||||
### referring
|
||||
[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
|
||||
|
||||
## CMake Style Recommendations
|
||||
|
||||
### Indent all code correctly, i.e. the body of
|
||||
|
||||
* if/else/endif
|
||||
* foreach/endforeach
|
||||
* while/endwhile
|
||||
* macro/endmacro
|
||||
* function/endfunction
|
||||
|
||||
Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
|
||||
spaces for indenting as is used in the rest of the file. Do not use tabs.
|
||||
|
||||
### Upper/lower casing
|
||||
|
||||
Most important: use consistent upper- or lowercasing within one file !
|
||||
|
||||
In general, the all-lowercase style is preferred.
|
||||
|
||||
So, this is recommended:
|
||||
|
||||
```
|
||||
add_executable(foo foo.c)
|
||||
```
|
||||
|
||||
These forms are discouraged
|
||||
|
||||
```
|
||||
ADD_EXECUTABLE(bar bar.c)
|
||||
Add_Executable(hello hello.c)
|
||||
aDd_ExEcUtAbLe(blub blub.c)
|
||||
```
|
||||
|
||||
### End commands
|
||||
To make the code easier to read, use empty commands for endforeach(), endif(),
|
||||
endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
|
||||
|
||||
For example, do this:
|
||||
|
||||
```
|
||||
if(FOOVAR)
|
||||
some_command(...)
|
||||
else()
|
||||
another_command(...)
|
||||
endif()
|
||||
```
|
||||
|
||||
and not this:
|
||||
|
||||
```
|
||||
if(BARVAR)
|
||||
some_other_command(...)
|
||||
endif(BARVAR)
|
||||
```
|
||||
|
||||
### Other resources for best practices
|
||||
|
||||
https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
endif()
|
||||
endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
|
||||
file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
|
||||
string(REGEX REPLACE "\n" ";" files "${files}")
|
||||
@@ -15,8 +15,8 @@ foreach(file ${files})
|
||||
)
|
||||
if(NOT "${rm_retval}" STREQUAL 0)
|
||||
message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
|
||||
endif()
|
||||
else()
|
||||
endif(NOT "${rm_retval}" STREQUAL 0)
|
||||
else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
|
||||
endif()
|
||||
endforeach()
|
||||
endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
endforeach(file)
|
||||
@@ -1,13 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(contrib)
|
||||
|
||||
add_subdirectory(pzstd)
|
||||
add_subdirectory(gen_html)
|
||||
@@ -1,30 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(gen_html)
|
||||
include(GetZstdLibraryVersion)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
|
||||
set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
|
||||
|
||||
add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp)
|
||||
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
|
||||
set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
|
||||
add_custom_target(zstd_manual.html ALL
|
||||
${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
|
||||
DEPENDS gen_html COMMENT "Update zstd manual")
|
||||
|
||||
install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
|
||||
@@ -1,32 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(pzstd)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
|
||||
|
||||
add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if (CMAKE_USE_PTHREADS_INIT)
|
||||
target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
|
||||
install(TARGETS pzstd RUNTIME DESTINATION "bin")
|
||||
@@ -1,2 +0,0 @@
|
||||
# cmake build artefact
|
||||
libzstd.pc
|
||||
+151
-129
@@ -1,57 +1,82 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
# * Copyright (c) 2014-present, Yann Collet, Facebook, Inc.
|
||||
# * All rights reserved.
|
||||
# *
|
||||
# * This source code is licensed under the BSD-style license found in the
|
||||
# * LICENSE file in the root directory of this source tree. An additional grant
|
||||
# * of patent rights can be found in the PATENTS file in the same directory.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You can contact the author at :
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
project(libzstd)
|
||||
# Get library version based on information from input content (use regular exp)
|
||||
function(GetLibraryVersion _content _outputVar1 _outputVar2 _outputVar3)
|
||||
string(REGEX MATCHALL ".*define ZSTD_VERSION_MAJOR+.* ([0-9]+).*define ZSTD_VERSION_MINOR+.* ([0-9]+).*define ZSTD_VERSION_RELEASE+.* ([0-9]+)" VERSION_REGEX "${_content}")
|
||||
SET(${_outputVar1} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
SET(${_outputVar2} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
SET(${_outputVar3} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
||||
option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||
PROJECT(libzstd)
|
||||
|
||||
if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build at least one flavor of libzstd")
|
||||
endif()
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define project root directory
|
||||
SET(ROOT_DIR ../../..)
|
||||
|
||||
# Define library directory, where sources and header files are located
|
||||
include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
|
||||
file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
|
||||
file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
|
||||
file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
|
||||
file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
|
||||
file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
|
||||
# Read file content
|
||||
FILE(READ ${LIBRARY_DIR}/zstd.h HEADER_CONTENT)
|
||||
|
||||
set(Sources
|
||||
${CommonSources}
|
||||
${CompressSources}
|
||||
${DecompressSources}
|
||||
${DictBuilderSources}
|
||||
${DeprecatedSources})
|
||||
# Parse version
|
||||
GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
|
||||
MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
|
||||
file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
|
||||
file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
|
||||
file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
|
||||
file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
|
||||
file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
|
||||
SET(Sources
|
||||
${LIBRARY_DIR}/common/entropy_common.c
|
||||
${LIBRARY_DIR}/common/fse_decompress.c
|
||||
${LIBRARY_DIR}/common/threading.c
|
||||
${LIBRARY_DIR}/common/pool.c
|
||||
${LIBRARY_DIR}/common/zstd_common.c
|
||||
${LIBRARY_DIR}/common/error_private.c
|
||||
${LIBRARY_DIR}/common/xxhash.c
|
||||
${LIBRARY_DIR}/compress/fse_compress.c
|
||||
${LIBRARY_DIR}/compress/huf_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_compress.c
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.c
|
||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/cover.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_compress.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
|
||||
|
||||
set(Headers
|
||||
SET(Headers
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${CommonHeaders}
|
||||
${CompressHeaders}
|
||||
${DecompressHeaders}
|
||||
${DictBuilderHeaders}
|
||||
${DeprecatedHeaders})
|
||||
${LIBRARY_DIR}/common/pool.h
|
||||
${LIBRARY_DIR}/common/threading.h
|
||||
${LIBRARY_DIR}/common/bitstream.h
|
||||
${LIBRARY_DIR}/common/error_private.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
${LIBRARY_DIR}/common/fse.h
|
||||
${LIBRARY_DIR}/common/huf.h
|
||||
${LIBRARY_DIR}/common/mem.h
|
||||
${LIBRARY_DIR}/common/zstd_internal.h
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
include_directories(${LIBRARY_LEGACY_DIR})
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
|
||||
|
||||
set(Sources ${Sources}
|
||||
SET(Sources ${Sources}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
||||
@@ -60,7 +85,7 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.c)
|
||||
|
||||
set(Headers ${Headers}
|
||||
SET(Headers ${Headers}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.h
|
||||
@@ -69,108 +94,105 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v05.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.h)
|
||||
endif ()
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/libzstd-dll)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Split project to static and shared libraries build
|
||||
set(library_targets)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
list(APPEND library_targets libzstd_shared)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_shared ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif()
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
add_library(libzstd_static STATIC ${Sources} ${Headers})
|
||||
list(APPEND library_targets libzstd_static)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_static ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
|
||||
ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
|
||||
# Add specific compile definitions for MSVC project
|
||||
if (MSVC)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET_TARGET_PROPERTIES(libzstd_static PROPERTIES COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (MSVC)
|
||||
|
||||
# With MSVC static library needs to be renamed to avoid conflict with import library
|
||||
if (MSVC)
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd_static)
|
||||
else ()
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd)
|
||||
endif ()
|
||||
# Define library base name
|
||||
IF (MSVC)
|
||||
SET(LIBRARY_BASE_NAME zstdlib)
|
||||
ELSE ()
|
||||
SET(LIBRARY_BASE_NAME libzstd)
|
||||
ENDIF (MSVC)
|
||||
|
||||
IF (MSVC)
|
||||
IF (CMAKE_SIZEOF_VOID_P MATCHES "8")
|
||||
SET(LIBRARY_ARCH_SUFFIX "_x64")
|
||||
ELSE ()
|
||||
SET(LIBRARY_ARCH_SUFFIX "_x86")
|
||||
ENDIF (CMAKE_SIZEOF_VOID_P MATCHES "8")
|
||||
ELSE ()
|
||||
SET(LIBRARY_ARCH_SUFFIX "")
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Define static and shared library names
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_target_properties(
|
||||
libzstd_shared
|
||||
PROPERTIES
|
||||
OUTPUT_NAME zstd
|
||||
VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
|
||||
SOVERSION ${zstd_VERSION_MAJOR})
|
||||
endif ()
|
||||
SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}${LIBRARY_ARCH_SUFFIX} CACHE STRING "Static library output name")
|
||||
SET(SHARED_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}.${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}${LIBRARY_ARCH_SUFFIX} CACHE STRING "Shared library output name")
|
||||
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_target_properties(
|
||||
libzstd_static
|
||||
PROPERTIES
|
||||
POSITION_INDEPENDENT_CODE On
|
||||
OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
|
||||
endif ()
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_static
|
||||
PROPERTIES
|
||||
PREFIX ""
|
||||
OUTPUT_NAME ${STATIC_LIBRARY_OUTPUT_NAME})
|
||||
|
||||
if (UNIX OR MINGW)
|
||||
# pkg-config
|
||||
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
set(LIBDIR "${CMAKE_INSTALL_LIBDIR}")
|
||||
set(INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
set(VERSION "${zstd_VERSION}")
|
||||
add_custom_target(libzstd.pc ALL
|
||||
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
|
||||
-DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
|
||||
COMMENT "Creating pkg-config file")
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_shared
|
||||
PROPERTIES
|
||||
PREFIX ""
|
||||
OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME})
|
||||
|
||||
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
|
||||
endif ()
|
||||
IF (UNIX)
|
||||
IF ("${PREFIX}" STREQUAL "")
|
||||
SET(PREFIX /usr/local)
|
||||
ENDIF()
|
||||
MESSAGE("the variable PREFIX=${PREFIX}")
|
||||
SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib)
|
||||
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
|
||||
|
||||
# install target
|
||||
install(FILES
|
||||
"${LIBRARY_DIR}/zstd.h"
|
||||
"${LIBRARY_DIR}/dictBuilder/zdict.h"
|
||||
"${LIBRARY_DIR}/common/zstd_errors.h"
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
# install target
|
||||
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
|
||||
install(TARGETS ${library_targets}
|
||||
EXPORT zstdExports
|
||||
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
||||
)
|
||||
# Create symlinks and setup this files
|
||||
SET(SHARED_LIBRARY_LINK ${SHARED_LIBRARY_OUTPUT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
SET(SHARED_LIBRARY_SYMLINK1 ${LIBRARY_BASE_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
SET(SHARED_LIBRARY_SYMLINK2 ${LIBRARY_BASE_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${LIBVER_MAJOR})
|
||||
|
||||
# uninstall target
|
||||
if (NOT TARGET uninstall)
|
||||
configure_file(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
SET(SHARED_LIBRARY_LINK_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_LINK})
|
||||
SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1})
|
||||
SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2})
|
||||
|
||||
if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
|
||||
FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH})
|
||||
endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
|
||||
|
||||
if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
|
||||
FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH})
|
||||
endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
|
||||
|
||||
ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
|
||||
COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
|
||||
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
|
||||
COMMENT "Generating symbolic link")
|
||||
|
||||
ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
|
||||
COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
|
||||
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
|
||||
COMMENT "Generating symbolic link")
|
||||
|
||||
INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
|
||||
# uninstall target
|
||||
CONFIGURE_FILE(
|
||||
"${CMAKE_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
IMMEDIATE @ONLY)
|
||||
|
||||
add_custom_target(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
endif ()
|
||||
ADD_CUSTOM_TARGET(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
ENDIF (UNIX)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
configure_file("${IN}" "${OUT}" @ONLY)
|
||||
@@ -1,5 +1,3 @@
|
||||
# produced by make
|
||||
zstd
|
||||
zstd-frugal
|
||||
unzstd
|
||||
zstdcat
|
||||
|
||||
@@ -1,135 +1,42 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# * All rights reserved.
|
||||
# *
|
||||
# * This source code is licensed under the BSD-style license found in the
|
||||
# * LICENSE file in the root directory of this source tree. An additional grant
|
||||
# * of patent rights can be found in the PATENTS file in the same directory.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You can contact the author at :
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
project(programs)
|
||||
PROJECT(programs)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define project root directory
|
||||
SET(ROOT_DIR ../../..)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
endif ()
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (ZSTD_PROGRAMS_LINK_SHARED)
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
|
||||
else ()
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstd)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
endif ()
|
||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
||||
|
||||
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
target_link_libraries(zstd rt)
|
||||
endif ()
|
||||
install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
|
||||
if (UNIX)
|
||||
add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
|
||||
add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
|
||||
add_custom_target(zstd.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
|
||||
COMMENT "Copying manpage zstd.1")
|
||||
add_custom_target(zstdgrep.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
|
||||
COMMENT "Copying manpage zstdgrep.1")
|
||||
add_custom_target(zstdless.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
|
||||
COMMENT "Copying manpage zstdless.1")
|
||||
add_custom_target(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
|
||||
add_custom_target(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
|
||||
|
||||
# Define MAN_INSTALL_DIR if necessary
|
||||
if (MAN_INSTALL_DIR)
|
||||
else ()
|
||||
set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1)
|
||||
endif ()
|
||||
|
||||
install(FILES
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/unzstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
|
||||
DESTINATION "${MAN_INSTALL_DIR}")
|
||||
|
||||
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
|
||||
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
endif ()
|
||||
|
||||
# Add multi-threading support definitions
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
|
||||
if (UNIX)
|
||||
target_link_libraries(zstd ${THREADS_LIBS})
|
||||
|
||||
add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
option(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
|
||||
option(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
|
||||
option(ZSTD_LZ4_SUPPORT "LZ4 SUPPORT" OFF)
|
||||
|
||||
# Add gzip support
|
||||
if (ZSTD_ZLIB_SUPPORT)
|
||||
find_package(ZLIB REQUIRED)
|
||||
|
||||
if (ZLIB_FOUND)
|
||||
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${ZLIB_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "zlib library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Add lzma support
|
||||
if (ZSTD_LZMA_SUPPORT)
|
||||
find_package(LibLZMA REQUIRED)
|
||||
|
||||
if (LIBLZMA_FOUND)
|
||||
include_directories(${LIBLZMA_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZMA_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lzma library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Add lz4 support
|
||||
if (ZSTD_LZ4_SUPPORT)
|
||||
find_package(LibLZ4 REQUIRED)
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
include_directories(${LIBLZ4_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZ4_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZ4COMPRESS;ZSTD_LZ4DECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lz4 library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (UNIX)
|
||||
ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
|
||||
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
||||
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
ENDIF (UNIX)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ################################################################
|
||||
# zstd - Makefile
|
||||
# Copyright (C) Yann Collet 2014-present
|
||||
# Copyright (C) Yann Collet 2014-2016
|
||||
# All rights reserved.
|
||||
#
|
||||
# BSD license
|
||||
@@ -30,77 +30,29 @@
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
project(tests)
|
||||
PROJECT(tests)
|
||||
|
||||
# name: Cache variable name. The value is expected to be a semicolon-separated
|
||||
# list of command line flags
|
||||
# default_value: Value to initialize the option with. Can be space separated.
|
||||
function(AddTestFlagsOption name default_value doc)
|
||||
string(STRIP "${default_value}" default_value)
|
||||
string(REGEX REPLACE " +" ";" default_value "${default_value}")
|
||||
set(${name} ${default_value} CACHE STRING "${doc}")
|
||||
mark_as_advanced(${name})
|
||||
endfunction()
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
# Define project root directory
|
||||
SET(ROOT_DIR ../../..)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
|
||||
include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
|
||||
SET(TESTS_DIR ${ROOT_DIR}/tests)
|
||||
INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
target_link_libraries(datagen libzstd_static)
|
||||
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c)
|
||||
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
||||
|
||||
#
|
||||
# fullbench
|
||||
#
|
||||
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
|
||||
target_link_libraries(fullbench libzstd_static)
|
||||
add_test(NAME fullbench COMMAND fullbench)
|
||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
|
||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||
|
||||
#
|
||||
# fuzzer
|
||||
#
|
||||
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
|
||||
target_link_libraries(fuzzer libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
|
||||
add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
|
||||
# Disable the timeout since the run time is too long for the default timeout of
|
||||
# 1500 seconds and varies considerably between low-end and high-end CPUs.
|
||||
# set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
|
||||
IF (UNIX)
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
|
||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||
|
||||
#
|
||||
# zstreamtest
|
||||
#
|
||||
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
|
||||
target_link_libraries(zstreamtest libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
|
||||
add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
|
||||
|
||||
#
|
||||
# playTests.sh
|
||||
#
|
||||
AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "--test-large-data"
|
||||
"Semicolon-separated list of flags to pass to the playTests.sh test")
|
||||
add_test(NAME playTests COMMAND sh -c "${TESTS_DIR}/playTests.sh" ${ZSTD_PLAYTESTS_FLAGS})
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
|
||||
"ZSTD_BIN=$<TARGET_FILE:zstd>"
|
||||
"DATAGEN_BIN=$<TARGET_FILE:datagen>"
|
||||
)
|
||||
else()
|
||||
message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
|
||||
set_tests_properties(playTests PROPERTIES DISABLED YES)
|
||||
endif()
|
||||
|
||||
# Label the "Medium" set of tests (see TESTING.md)
|
||||
set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
|
||||
|
||||
if (UNIX)
|
||||
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
|
||||
target_link_libraries(paramgrill libzstd_static m) #m is math library
|
||||
endif ()
|
||||
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
TARGET_LINK_LIBRARIES(datagen libzstd_static)
|
||||
ENDIF (UNIX)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import re
|
||||
|
||||
|
||||
def find_version_tuple(filepath):
|
||||
version_file_data = None
|
||||
with open(filepath) as fd:
|
||||
version_file_data = fd.read()
|
||||
|
||||
patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
|
||||
"""
|
||||
regex = re.compile(patterns, re.MULTILINE)
|
||||
version_match = regex.search(version_file_data)
|
||||
if version_match:
|
||||
return version_match.groups()
|
||||
raise Exception("Unable to find version string")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
|
||||
parser.add_argument('file', help='path to lib/zstd.h')
|
||||
args = parser.parse_args()
|
||||
version_tuple = find_version_tuple(args.file)
|
||||
print('.'.join(version_tuple))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
# This file should be synced with https://github.com/lzutao/meson-symlink
|
||||
|
||||
import os
|
||||
import pathlib # since Python 3.4
|
||||
|
||||
|
||||
def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
|
||||
if not install_dir.exists():
|
||||
install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
|
||||
if not install_dir.is_dir():
|
||||
raise NotADirectoryError(install_dir)
|
||||
|
||||
new_dst = install_dir.joinpath(dst)
|
||||
if new_dst.is_symlink() and os.readlink(new_dst) == src:
|
||||
print('File exists: {!r} -> {!r}'.format(new_dst, src))
|
||||
return
|
||||
print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
|
||||
new_dst.symlink_to(src, target_is_directory=dst_is_dir)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Install a symlink',
|
||||
usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
|
||||
'example:\n'
|
||||
' {0} dash sh /bin'.format(pathlib.Path(__file__).name))
|
||||
parser.add_argument('source', help='target to link')
|
||||
parser.add_argument('dest', help='link name')
|
||||
parser.add_argument('install_dir', help='installation directory')
|
||||
parser.add_argument('-d', '--isdir',
|
||||
action='store_true',
|
||||
help='dest is a directory')
|
||||
parser.add_argument('-m', '--mode',
|
||||
help='directory mode on creating if not exist',
|
||||
default='0o755')
|
||||
args = parser.parse_args()
|
||||
|
||||
dir_mode = int(args.mode, 8)
|
||||
|
||||
meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
|
||||
install_dir = pathlib.Path(meson_destdir, args.install_dir)
|
||||
install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
Meson build system for zstandard
|
||||
================================
|
||||
|
||||
Meson is a build system designed to optimize programmer productivity.
|
||||
It aims to do this by providing simple, out-of-the-box support for
|
||||
modern software development tools and practices, such as unit tests,
|
||||
coverage reports, Valgrind, CCache and the like.
|
||||
|
||||
This Meson build system is provided with no guarantee and maintained
|
||||
by Dima Krasner \<dima@dimakrasner.com\>.
|
||||
|
||||
It outputs one `libzstd`, either shared or static, depending on
|
||||
`default_library` option.
|
||||
|
||||
## How to build
|
||||
|
||||
`cd` to this meson directory (`build/meson`)
|
||||
|
||||
```sh
|
||||
meson setup -Dbin_programs=true -Dbin_contrib=true builddir
|
||||
cd builddir
|
||||
ninja # to build
|
||||
ninja install # to install
|
||||
```
|
||||
|
||||
You might want to install it in staging directory:
|
||||
|
||||
```sh
|
||||
DESTDIR=./staging ninja install
|
||||
```
|
||||
|
||||
To configure build options, use:
|
||||
|
||||
```sh
|
||||
meson configure
|
||||
```
|
||||
|
||||
See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).
|
||||
@@ -1,30 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html'))
|
||||
|
||||
gen_html = executable('gen_html',
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
|
||||
include_directories: gen_html_includes,
|
||||
native: true,
|
||||
install: false)
|
||||
|
||||
# Update zstd manual
|
||||
zstd_manual_html = custom_target('zstd_manual.html',
|
||||
output : 'zstd_manual.html',
|
||||
command : [gen_html,
|
||||
zstd_version,
|
||||
join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
|
||||
'@OUTPUT@'],
|
||||
install : false)
|
||||
@@ -1,12 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
subdir('pzstd')
|
||||
subdir('gen_html')
|
||||
@@ -1,24 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd'))
|
||||
pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
|
||||
pzstd = executable('pzstd',
|
||||
pzstd_sources,
|
||||
cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
|
||||
include_directories: pzstd_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: true)
|
||||
@@ -1,131 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'lib/compress'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated'))]
|
||||
|
||||
libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/hist.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
|
||||
|
||||
# Explicit define legacy support
|
||||
add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
|
||||
language: 'c')
|
||||
|
||||
if legacy_level == 0
|
||||
message('Legacy support: DISABLED')
|
||||
else
|
||||
# See ZSTD_LEGACY_SUPPORT of lib/README.md
|
||||
message('Enable legacy support back to version 0.@0@'.format(legacy_level))
|
||||
|
||||
libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
|
||||
foreach i : [1, 2, 3, 4, 5, 6, 7]
|
||||
if legacy_level <= i
|
||||
libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
|
||||
endif
|
||||
endforeach
|
||||
endif
|
||||
|
||||
libzstd_deps = []
|
||||
if use_multi_thread
|
||||
message('Enable multi-threading support')
|
||||
add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
|
||||
libzstd_deps = [ thread_dep ]
|
||||
endif
|
||||
|
||||
libzstd_c_args = []
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
libzstd_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
|
||||
libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
|
||||
'-DZSTD_HEAPMODE=0',
|
||||
'-D_CONSOLE',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
else
|
||||
libzstd_c_args += ['-DZSTD_HEAPMODE=0',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
endif
|
||||
endif
|
||||
|
||||
mingw_ansi_stdio_flags = []
|
||||
if host_machine_os == os_windows and cc_id == compiler_gcc
|
||||
mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
|
||||
endif
|
||||
libzstd_c_args += mingw_ansi_stdio_flags
|
||||
|
||||
libzstd_debug_cflags = []
|
||||
if use_debug
|
||||
libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
|
||||
if cc_id == compiler_gcc or cc_id == compiler_clang
|
||||
libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
|
||||
'-Wdeclaration-after-statement', '-Wstrict-prototypes',
|
||||
'-Wundef', '-Wpointer-arith', '-Wvla',
|
||||
'-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
|
||||
'-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
|
||||
endif
|
||||
endif
|
||||
libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
|
||||
|
||||
libzstd = library('zstd',
|
||||
libzstd_sources,
|
||||
include_directories: libzstd_includes,
|
||||
c_args: libzstd_c_args,
|
||||
dependencies: libzstd_deps,
|
||||
install: true,
|
||||
version: zstd_libversion)
|
||||
|
||||
libzstd_dep = declare_dependency(link_with: libzstd,
|
||||
include_directories: libzstd_includes)
|
||||
|
||||
pkgconfig.generate(libzstd,
|
||||
name: 'libzstd',
|
||||
filebase: 'libzstd',
|
||||
description: 'fast lossless compression algorithm library',
|
||||
version: zstd_libversion,
|
||||
url: 'http://www.zstd.net/')
|
||||
|
||||
install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))
|
||||
@@ -1,147 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
project('zstd',
|
||||
['c', 'cpp'],
|
||||
license: ['BSD', 'GPLv2'],
|
||||
default_options : [
|
||||
'c_std=gnu99',
|
||||
'cpp_std=c++11',
|
||||
'buildtype=release'
|
||||
],
|
||||
version: 'DUMMY',
|
||||
meson_version: '>=0.47.0')
|
||||
|
||||
cc = meson.get_compiler('c')
|
||||
cxx = meson.get_compiler('cpp')
|
||||
pkgconfig = import('pkgconfig')
|
||||
python3 = import('python').find_installation()
|
||||
windows_mod = import('windows')
|
||||
|
||||
host_machine_os = host_machine.system()
|
||||
os_windows = 'windows'
|
||||
os_linux = 'linux'
|
||||
os_darwin = 'darwin'
|
||||
os_freebsd = 'freebsd'
|
||||
os_sun = 'sunos'
|
||||
|
||||
cc_id = cc.get_id()
|
||||
compiler_gcc = 'gcc'
|
||||
compiler_clang = 'clang'
|
||||
compiler_msvc = 'msvc'
|
||||
|
||||
zstd_version = meson.project_version()
|
||||
|
||||
zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
|
||||
GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py')
|
||||
r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file)
|
||||
if r.returncode() == 0
|
||||
zstd_version = r.stdout().strip()
|
||||
message('Project version is now: @0@'.format(zstd_version))
|
||||
else
|
||||
error('Cannot find project version in @0@'.format(zstd_h_file))
|
||||
endif
|
||||
|
||||
zstd_libversion = zstd_version
|
||||
|
||||
# =============================================================================
|
||||
# Installation directories
|
||||
# =============================================================================
|
||||
|
||||
zstd_prefix = get_option('prefix')
|
||||
zstd_bindir = get_option('bindir')
|
||||
zstd_datadir = get_option('datadir')
|
||||
zstd_mandir = get_option('mandir')
|
||||
zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
|
||||
|
||||
# =============================================================================
|
||||
# Project options
|
||||
# =============================================================================
|
||||
|
||||
# Built-in options
|
||||
use_debug = get_option('debug')
|
||||
buildtype = get_option('buildtype')
|
||||
default_library_type = get_option('default_library')
|
||||
|
||||
# Custom options
|
||||
debug_level = get_option('debug_level')
|
||||
legacy_level = get_option('legacy_level')
|
||||
use_backtrace = get_option('backtrace')
|
||||
use_static_runtime = get_option('static_runtime')
|
||||
|
||||
bin_programs = get_option('bin_programs')
|
||||
bin_contrib = get_option('bin_contrib')
|
||||
bin_tests = get_option('bin_tests')
|
||||
|
||||
feature_multi_thread = get_option('multi_thread')
|
||||
feature_zlib = get_option('zlib')
|
||||
feature_lzma = get_option('lzma')
|
||||
feature_lz4 = get_option('lz4')
|
||||
|
||||
# =============================================================================
|
||||
# Dependencies
|
||||
# =============================================================================
|
||||
|
||||
libm_dep = cc.find_library('m', required: bin_tests)
|
||||
thread_dep = dependency('threads', required: feature_multi_thread)
|
||||
use_multi_thread = thread_dep.found()
|
||||
# Arguments in dependency should be equivalent to those passed to pkg-config
|
||||
zlib_dep = dependency('zlib', required: feature_zlib)
|
||||
use_zlib = zlib_dep.found()
|
||||
lzma_dep = dependency('liblzma', required: feature_lzma)
|
||||
use_lzma = lzma_dep.found()
|
||||
lz4_dep = dependency('liblz4', required: feature_lz4)
|
||||
use_lz4 = lz4_dep.found()
|
||||
|
||||
# =============================================================================
|
||||
# Compiler flags
|
||||
# =============================================================================
|
||||
|
||||
add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
|
||||
|
||||
if [compiler_gcc, compiler_clang].contains(cc_id)
|
||||
common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
|
||||
if cc_id == compiler_clang
|
||||
# Should use Meson's own --werror build option
|
||||
#common_warning_flags += '-Werror'
|
||||
common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
|
||||
endif
|
||||
cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
|
||||
cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
|
||||
add_project_arguments(cc_compile_flags, language : 'c')
|
||||
add_project_arguments(cxx_compile_flags, language : 'cpp')
|
||||
elif cc_id == compiler_msvc
|
||||
msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
|
||||
if use_multi_thread
|
||||
msvc_compile_flags += '/MP'
|
||||
endif
|
||||
if use_static_runtime
|
||||
msvc_compile_flags += '/MT'
|
||||
endif
|
||||
add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
|
||||
endif
|
||||
|
||||
# =============================================================================
|
||||
# Subdirs
|
||||
# =============================================================================
|
||||
|
||||
subdir('lib')
|
||||
|
||||
if bin_programs
|
||||
subdir('programs')
|
||||
endif
|
||||
|
||||
if bin_tests
|
||||
subdir('tests')
|
||||
endif
|
||||
|
||||
if bin_contrib
|
||||
subdir('contrib')
|
||||
endif
|
||||
@@ -1,36 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
|
||||
|
||||
option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
|
||||
description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
|
||||
option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
|
||||
description: 'Enable run-time debug. See lib/common/debug.h')
|
||||
option('backtrace', type: 'boolean', value: false,
|
||||
description: 'Display a stack backtrace when execution generates a runtime exception')
|
||||
option('static_runtime', type: 'boolean', value: false,
|
||||
description: 'Link to static run-time libraries on MSVC')
|
||||
|
||||
option('bin_programs', type: 'boolean', value: true,
|
||||
description: 'Enable programs build')
|
||||
option('bin_tests', type: 'boolean', value: false,
|
||||
description: 'Enable tests build')
|
||||
option('bin_contrib', type: 'boolean', value: false,
|
||||
description: 'Enable contrib build')
|
||||
|
||||
option('multi_thread', type: 'feature', value: 'enabled',
|
||||
description: 'Enable multi-threading when pthread is detected')
|
||||
option('zlib', type: 'feature', value: 'auto',
|
||||
description: 'Enable zlib support')
|
||||
option('lzma', type: 'feature', value: 'auto',
|
||||
description: 'Enable lzma support')
|
||||
option('lz4', type: 'feature', value: 'auto',
|
||||
description: 'Enable lz4 support')
|
||||
@@ -1,104 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/dibio.c')]
|
||||
|
||||
zstd_c_args = libzstd_debug_cflags
|
||||
if use_multi_thread
|
||||
zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
|
||||
endif
|
||||
|
||||
zstd_deps = [ libzstd_dep ]
|
||||
if use_zlib
|
||||
zstd_deps += [ zlib_dep ]
|
||||
zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lzma
|
||||
zstd_deps += [ lzma_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lz4
|
||||
zstd_deps += [ lz4_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
|
||||
endif
|
||||
|
||||
export_dynamic_on_windows = false
|
||||
# explicit backtrace enable/disable for Linux & Darwin
|
||||
if not use_backtrace
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=0'
|
||||
elif use_debug and host_machine_os == os_windows # MinGW target
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=1'
|
||||
export_dynamic_on_windows = true
|
||||
endif
|
||||
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
zstd_programs_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
|
||||
endif
|
||||
endif
|
||||
|
||||
zstd = executable('zstd',
|
||||
zstd_programs_sources,
|
||||
c_args: zstd_c_args,
|
||||
dependencies: zstd_deps,
|
||||
export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
|
||||
install: true)
|
||||
|
||||
zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c')]
|
||||
|
||||
# Minimal target, with only zstd compression and decompression.
|
||||
# No bench. No legacy.
|
||||
executable('zstd-frugal',
|
||||
zstd_frugal_sources,
|
||||
dependencies: libzstd_dep,
|
||||
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
|
||||
install: true)
|
||||
|
||||
install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless'),
|
||||
install_dir: zstd_bindir)
|
||||
|
||||
# =============================================================================
|
||||
# Programs and manpages installing
|
||||
# =============================================================================
|
||||
|
||||
install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless.1'))
|
||||
|
||||
InstallSymlink_py = '../InstallSymlink.py'
|
||||
zstd_man1_dir = join_paths(zstd_mandir, 'man1')
|
||||
bin_EXT = host_machine_os == os_windows ? '.exe' : ''
|
||||
man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
|
||||
|
||||
foreach f : ['zstdcat', 'unzstd']
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, f + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
|
||||
endforeach
|
||||
|
||||
if use_multi_thread
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, 'zstdmt' + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
|
||||
endif
|
||||
@@ -1,206 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
|
||||
os_freebsd, 'netbsd', 'dragonfly', os_sun]
|
||||
|
||||
# =============================================================================
|
||||
# Test flags
|
||||
# =============================================================================
|
||||
|
||||
FUZZER_FLAGS = ['--no-big-tests']
|
||||
FUZZERTEST = '-T200s'
|
||||
ZSTREAM_TESTTIME = '-T90s'
|
||||
DECODECORPUS_TESTTIME = '-T30'
|
||||
ZSTDRTTEST = ['--test-large-data']
|
||||
|
||||
# =============================================================================
|
||||
# Executables
|
||||
# =============================================================================
|
||||
|
||||
test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
|
||||
|
||||
datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/datagencli.c')]
|
||||
datagen = executable('datagen',
|
||||
datagen_sources,
|
||||
c_args: [ '-DNDEBUG' ],
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fullbench.c')]
|
||||
fullbench = executable('fullbench',
|
||||
fullbench_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fuzzer.c')]
|
||||
fuzzer = executable('fuzzer',
|
||||
fuzzer_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/seqgen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
|
||||
zstreamtest = executable('zstreamtest',
|
||||
zstreamtest_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'tests/paramgrill.c')]
|
||||
paramgrill = executable('paramgrill',
|
||||
paramgrill_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
|
||||
roundTripCrash = executable('roundTripCrash',
|
||||
roundTripCrash_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
|
||||
longmatch = executable('longmatch',
|
||||
longmatch_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
|
||||
invalidDictionaries = executable('invalidDictionaries',
|
||||
invalidDictionaries_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
|
||||
legacy = executable('legacy',
|
||||
legacy_sources,
|
||||
# Use -Dlegacy_level build option to control it
|
||||
#c_args: '-DZSTD_LEGACY_SUPPORT=4',
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
endif
|
||||
|
||||
decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
|
||||
decodecorpus = executable('decodecorpus',
|
||||
decodecorpus_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/poolTests.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c')]
|
||||
poolTests = executable('poolTests',
|
||||
poolTests_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: false)
|
||||
|
||||
checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
|
||||
checkTag = executable('checkTag',
|
||||
checkTag_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
# =============================================================================
|
||||
# Tests (Use "meson test --list" to list all tests)
|
||||
# =============================================================================
|
||||
|
||||
if tests_supported_oses.contains(host_machine_os)
|
||||
valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
|
||||
valgrindTest_py = files('valgrindTest.py')
|
||||
test('valgrindTest',
|
||||
valgrindTest_py,
|
||||
args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
|
||||
depends: [zstd, datagen, fuzzer, fullbench],
|
||||
timeout: 600) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
if host_machine_os != os_windows
|
||||
playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
|
||||
test('test-zstd',
|
||||
playTests_sh,
|
||||
args: ZSTDRTTEST,
|
||||
env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
|
||||
depends: [datagen],
|
||||
workdir: meson.current_build_dir(),
|
||||
timeout: 2800) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
test('test-fullbench-1',
|
||||
fullbench,
|
||||
args: ['-i1'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
test('test-fullbench-2',
|
||||
fullbench,
|
||||
args: ['-i1', '-P0'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
|
||||
if use_zlib
|
||||
test('test-fuzzer',
|
||||
fuzzer,
|
||||
args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
|
||||
timeout: 480)
|
||||
endif
|
||||
|
||||
test('test-zstream-1',
|
||||
zstreamtest,
|
||||
args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 240)
|
||||
test('test-zstream-2',
|
||||
zstreamtest,
|
||||
args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-zstream-3',
|
||||
zstreamtest,
|
||||
args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-longmatch', longmatch, timeout: 36)
|
||||
test('test-invalidDictionaries', invalidDictionaries) # should be fast
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
test('test-legacy', legacy) # should be fast
|
||||
endif
|
||||
test('test-decodecorpus',
|
||||
decodecorpus,
|
||||
args: ['-t', DECODECORPUS_TESTTIME],
|
||||
timeout: 60)
|
||||
test('test-poolTests', poolTests) # should be fast
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
|
||||
VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
|
||||
|
||||
print('\n ---- valgrind tests : memory analyzer ----')
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
|
||||
|
||||
if subprocess.call([*VALGRIND_ARGS, zstd],
|
||||
stdout=subprocess.DEVNULL) == 0:
|
||||
raise subprocess.CalledProcessError('zstd without argument should have failed')
|
||||
|
||||
with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmp_fd:
|
||||
with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
|
||||
stdin=p1.stdout) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
|
||||
with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
|
||||
subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
|
||||
parser.add_argument('valgrind', help='valgrind path')
|
||||
parser.add_argument('zstd', help='zstd path')
|
||||
parser.add_argument('datagen', help='datagen path')
|
||||
parser.add_argument('fuzzer', help='fuzzer path')
|
||||
parser.add_argument('fullbench', help='fullbench path')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
valgrind = args.valgrind
|
||||
zstd = args.zstd
|
||||
datagen = args.datagen
|
||||
fuzzer = args.fuzzer
|
||||
fullbench = args.fullbench
|
||||
|
||||
valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,3 +0,0 @@
|
||||
## Project Support Notice
|
||||
|
||||
The VS2005 Project directory has been moved to the contrib directory in order to indicate that it will no longer be supported.
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/bin/sh
|
||||
sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c}
|
||||
@@ -1 +0,0 @@
|
||||
check_flipped_bits
|
||||
@@ -1,35 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2019-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
.PHONY: all
|
||||
all: check_flipped_bits
|
||||
|
||||
ZSTDLIBDIR ?= ../../lib
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \
|
||||
-I$(ZSTDLIBDIR)/decompress
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls -Wmissing-prototypes
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
.PHONY: $(ZSTDLIBDIR)/libzstd.a
|
||||
$(ZSTDLIBDIR)/libzstd.a:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
|
||||
$(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f check_flipped_bits
|
||||
@@ -1,400 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2019-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef struct {
|
||||
char *input;
|
||||
size_t input_size;
|
||||
|
||||
char *perturbed; /* same size as input */
|
||||
|
||||
char *output;
|
||||
size_t output_size;
|
||||
|
||||
const char *dict_file_name;
|
||||
const char *dict_file_dir_name;
|
||||
int32_t dict_id;
|
||||
char *dict;
|
||||
size_t dict_size;
|
||||
ZSTD_DDict* ddict;
|
||||
|
||||
ZSTD_DCtx* dctx;
|
||||
|
||||
int success_count;
|
||||
int error_counts[ZSTD_error_maxCode];
|
||||
} stuff_t;
|
||||
|
||||
static void free_stuff(stuff_t* stuff) {
|
||||
free(stuff->input);
|
||||
free(stuff->output);
|
||||
ZSTD_freeDDict(stuff->ddict);
|
||||
free(stuff->dict);
|
||||
ZSTD_freeDCtx(stuff->dctx);
|
||||
}
|
||||
|
||||
static void usage(void) {
|
||||
fprintf(stderr, "check_flipped_bits input_filename [-d dict] [-D dict_dir]\n");
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "Arguments:\n");
|
||||
fprintf(stderr, " -d file: path to a dictionary file to use.\n");
|
||||
fprintf(stderr, " -D dir : path to a directory, with files containing dictionaries, of the\n"
|
||||
" form DICTID.zstd-dict, e.g., 12345.zstd-dict.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void print_summary(stuff_t* stuff) {
|
||||
int error_code;
|
||||
fprintf(stderr, "%9d successful decompressions\n", stuff->success_count);
|
||||
for (error_code = 0; error_code < ZSTD_error_maxCode; error_code++) {
|
||||
int count = stuff->error_counts[error_code];
|
||||
if (count) {
|
||||
fprintf(
|
||||
stderr, "%9d failed decompressions with message: %s\n",
|
||||
count, ZSTD_getErrorString(error_code));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static char* readFile(const char* filename, size_t* size) {
|
||||
struct stat statbuf;
|
||||
int ret;
|
||||
FILE* f;
|
||||
char *buf;
|
||||
size_t bytes_read;
|
||||
|
||||
ret = stat(filename, &statbuf);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "stat failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
if ((statbuf.st_mode & S_IFREG) != S_IFREG) {
|
||||
fprintf(stderr, "Input must be regular file\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*size = statbuf.st_size;
|
||||
|
||||
f = fopen(filename, "r");
|
||||
if (f == NULL) {
|
||||
fprintf(stderr, "fopen failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
buf = malloc(*size);
|
||||
if (buf == NULL) {
|
||||
fprintf(stderr, "malloc failed\n");
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes_read = fread(buf, 1, *size, f);
|
||||
if (bytes_read != *size) {
|
||||
fprintf(stderr, "failed to read whole file\n");
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = fclose(f);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "fclose failed: %m\n");
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDict(const char* filename, char **buf, size_t* size, int32_t* dict_id) {
|
||||
ZSTD_DDict* ddict;
|
||||
*buf = readFile(filename, size);
|
||||
if (*buf == NULL) {
|
||||
fprintf(stderr, "Opening dictionary file '%s' failed\n", filename);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ddict = ZSTD_createDDict_advanced(*buf, *size, ZSTD_dlm_byRef, ZSTD_dct_auto, ZSTD_defaultCMem);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict.\n");
|
||||
return NULL;
|
||||
}
|
||||
if (dict_id != NULL) {
|
||||
*dict_id = ZSTD_getDictID_fromDDict(ddict);
|
||||
}
|
||||
return ddict;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDictByID(stuff_t *stuff, int32_t dict_id, char **buf, size_t* size) {
|
||||
if (stuff->dict_file_dir_name == NULL) {
|
||||
return NULL;
|
||||
} else {
|
||||
size_t dir_name_len = strlen(stuff->dict_file_dir_name);
|
||||
int dir_needs_separator = 0;
|
||||
size_t dict_file_name_alloc_size = dir_name_len + 1 /* '/' */ + 10 /* max int32_t len */ + strlen(".zstd-dict") + 1 /* '\0' */;
|
||||
char *dict_file_name = malloc(dict_file_name_alloc_size);
|
||||
ZSTD_DDict* ddict;
|
||||
int32_t read_dict_id;
|
||||
if (dict_file_name == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (dir_name_len > 0 && stuff->dict_file_dir_name[dir_name_len - 1] != '/') {
|
||||
dir_needs_separator = 1;
|
||||
}
|
||||
|
||||
snprintf(
|
||||
dict_file_name,
|
||||
dict_file_name_alloc_size,
|
||||
"%s%s%u.zstd-dict",
|
||||
stuff->dict_file_dir_name,
|
||||
dir_needs_separator ? "/" : "",
|
||||
dict_id);
|
||||
|
||||
/* fprintf(stderr, "Loading dict %u from '%s'.\n", dict_id, dict_file_name); */
|
||||
|
||||
ddict = readDict(dict_file_name, buf, size, &read_dict_id);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", dict_file_name);
|
||||
free(dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
if (read_dict_id != dict_id) {
|
||||
fprintf(stderr, "Read dictID (%u) does not match expected (%u).\n", read_dict_id, dict_id);
|
||||
free(dict_file_name);
|
||||
ZSTD_freeDDict(ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
free(dict_file_name);
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
static int init_stuff(stuff_t* stuff, int argc, char *argv[]) {
|
||||
const char* input_filename;
|
||||
|
||||
if (argc < 2) {
|
||||
usage();
|
||||
}
|
||||
|
||||
input_filename = argv[1];
|
||||
stuff->input_size = 0;
|
||||
stuff->input = readFile(input_filename, &stuff->input_size);
|
||||
if (stuff->input == NULL) {
|
||||
fprintf(stderr, "Failed to read input file.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->perturbed = malloc(stuff->input_size);
|
||||
if (stuff->perturbed == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
memcpy(stuff->perturbed, stuff->input, stuff->input_size);
|
||||
|
||||
stuff->output_size = ZSTD_DStreamOutSize();
|
||||
stuff->output = malloc(stuff->output_size);
|
||||
if (stuff->output == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->dict_file_name = NULL;
|
||||
stuff->dict_file_dir_name = NULL;
|
||||
stuff->dict_id = 0;
|
||||
stuff->dict = NULL;
|
||||
stuff->dict_size = 0;
|
||||
stuff->ddict = NULL;
|
||||
|
||||
if (argc > 2) {
|
||||
if (!strcmp(argv[2], "-d")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else
|
||||
if (!strcmp(argv[2], "-D")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_dir_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
}
|
||||
|
||||
if (stuff->dict_file_dir_name) {
|
||||
int32_t dict_id = ZSTD_getDictID_fromFrame(stuff->input, stuff->input_size);
|
||||
if (dict_id != 0) {
|
||||
stuff->ddict = readDictByID(stuff, dict_id, &stuff->dict, &stuff->dict_size);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create cached ddict.\n");
|
||||
return 0;
|
||||
}
|
||||
stuff->dict_id = dict_id;
|
||||
}
|
||||
} else
|
||||
if (stuff->dict_file_name) {
|
||||
stuff->ddict = readDict(stuff->dict_file_name, &stuff->dict, &stuff->dict_size, &stuff->dict_id);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", stuff->dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->dctx = ZSTD_createDCtx();
|
||||
if (stuff->dctx == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->success_count = 0;
|
||||
memset(stuff->error_counts, 0, sizeof(stuff->error_counts));
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int test_decompress(stuff_t* stuff) {
|
||||
size_t ret;
|
||||
ZSTD_inBuffer in = {stuff->perturbed, stuff->input_size, 0};
|
||||
ZSTD_outBuffer out = {stuff->output, stuff->output_size, 0};
|
||||
ZSTD_DCtx* dctx = stuff->dctx;
|
||||
int32_t custom_dict_id = ZSTD_getDictID_fromFrame(in.src, in.size);
|
||||
char *custom_dict = NULL;
|
||||
size_t custom_dict_size = 0;
|
||||
ZSTD_DDict* custom_ddict = NULL;
|
||||
|
||||
if (custom_dict_id != 0 && custom_dict_id != stuff->dict_id) {
|
||||
/* fprintf(stderr, "Instead of dict %u, this perturbed blob wants dict %u.\n", stuff->dict_id, custom_dict_id); */
|
||||
custom_ddict = readDictByID(stuff, custom_dict_id, &custom_dict, &custom_dict_size);
|
||||
}
|
||||
|
||||
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_DCtx_refDDict(dctx, custom_ddict);
|
||||
} else {
|
||||
ZSTD_DCtx_refDDict(dctx, stuff->ddict);
|
||||
}
|
||||
|
||||
while (in.pos != in.size) {
|
||||
out.pos = 0;
|
||||
ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||
|
||||
if (ZSTD_isError(ret)) {
|
||||
unsigned int code = ZSTD_getErrorCode(ret);
|
||||
if (code >= ZSTD_error_maxCode) {
|
||||
fprintf(stderr, "Received unexpected error code!\n");
|
||||
exit(1);
|
||||
}
|
||||
stuff->error_counts[code]++;
|
||||
/*
|
||||
fprintf(
|
||||
stderr, "Decompression failed: %s\n", ZSTD_getErrorName(ret));
|
||||
*/
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->success_count++;
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bits(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t bit;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (bit = 0; bit < 8; bit++) {
|
||||
unsigned char new_val = old_val ^ (1 << bit);
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Flipping byte %zu bit %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, bit, old_val, new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bytes(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t new_val;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (new_val = 0; new_val < 256; new_val++) {
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Changing byte %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, old_val, (unsigned char)new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
stuff_t stuff;
|
||||
|
||||
if(!init_stuff(&stuff, argc, argv)) {
|
||||
fprintf(stderr, "Failed to init.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (test_decompress(&stuff)) {
|
||||
fprintf(stderr, "Blob already decompresses successfully!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
perturb_bits(&stuff);
|
||||
|
||||
perturb_bytes(&stuff);
|
||||
|
||||
print_summary(&stuff);
|
||||
|
||||
free_stuff(&stuff);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
# Dockerfile
|
||||
# First image to build the binary
|
||||
FROM alpine as builder
|
||||
|
||||
RUN apk --no-cache add make gcc libc-dev
|
||||
COPY . /src
|
||||
RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install
|
||||
|
||||
# Second minimal image to only keep the built binary
|
||||
FROM alpine
|
||||
|
||||
# Copy the built files
|
||||
COPY --from=builder /pkg /
|
||||
|
||||
# Copy the license as well
|
||||
RUN mkdir -p /usr/local/share/licenses/zstd
|
||||
COPY --from=builder /src/LICENSE /usr/local/share/licences/zstd/
|
||||
|
||||
# Just run `zstd` if no other command is given
|
||||
CMD ["/usr/local/bin/zstd"]
|
||||
@@ -1,20 +0,0 @@
|
||||
|
||||
## Requirement
|
||||
|
||||
The `Dockerfile` script requires a version of `docker` >= 17.05
|
||||
|
||||
## Installing docker
|
||||
|
||||
The official docker install docs use a ppa with a modern version available:
|
||||
https://docs.docker.com/install/linux/docker-ce/ubuntu/
|
||||
|
||||
## How to run
|
||||
|
||||
`docker build -t zstd .`
|
||||
|
||||
## test
|
||||
|
||||
```
|
||||
echo foo | docker run -i --rm zstd | docker run -i --rm zstd zstdcat
|
||||
foo
|
||||
```
|
||||
+13
-28
@@ -1,24 +1,18 @@
|
||||
# ################################################################
|
||||
# ##########################################################################
|
||||
# Copyright (c) 2016-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ##########################################################################
|
||||
|
||||
CXXFLAGS ?= -O3
|
||||
CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||
CXXFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
|
||||
|
||||
ZSTDAPI = ../../lib/zstd.h
|
||||
ZSTDMANUAL = ../../doc/zstd_manual.html
|
||||
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)`
|
||||
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
|
||||
LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
@@ -29,23 +23,14 @@ EXT =
|
||||
endif
|
||||
|
||||
|
||||
.PHONY: default
|
||||
.PHONY: default gen_html
|
||||
|
||||
default: gen_html
|
||||
|
||||
.PHONY: all
|
||||
all: manual
|
||||
|
||||
gen_html: gen_html.cpp
|
||||
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
||||
$(CXX) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
$(ZSTDMANUAL): gen_html $(ZSTDAPI)
|
||||
echo "Update zstd manual in /doc"
|
||||
./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
|
||||
|
||||
.PHONY: manual
|
||||
manual: gen_html $(ZSTDMANUAL)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@$(RM) gen_html$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
* Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
@@ -19,7 +19,7 @@ void trim(string& s, string characters)
|
||||
{
|
||||
size_t p = s.find_first_not_of(characters);
|
||||
s.erase(0, p);
|
||||
|
||||
|
||||
p = s.find_last_not_of(characters);
|
||||
if (string::npos != p)
|
||||
s.erase(p+1);
|
||||
@@ -48,7 +48,7 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator)
|
||||
line = input[linenum];
|
||||
|
||||
if (terminator.empty() && line.empty()) { linenum--; break; }
|
||||
|
||||
|
||||
epos = line.find(terminator);
|
||||
if (!terminator.empty() && epos!=string::npos) {
|
||||
out.push_back(line);
|
||||
@@ -168,11 +168,7 @@ int main(int argc, char *argv[]) {
|
||||
sout << "<pre><b>";
|
||||
for (l=0; l<lines.size(); l++) {
|
||||
// fprintf(stderr, "line[%d]=%s\n", l, lines[l].c_str());
|
||||
string fline = lines[l];
|
||||
if (fline.substr(0, 12) == "ZSTDLIB_API " ||
|
||||
fline.substr(0, 12) == string(12, ' '))
|
||||
fline = fline.substr(12);
|
||||
print_line(sout, fline);
|
||||
print_line(sout, lines[l]);
|
||||
}
|
||||
sout << "</b><p>";
|
||||
for (l=0; l<comments.size(); l++) {
|
||||
@@ -221,4 +217,4 @@ int main(int argc, char *argv[]) {
|
||||
ostream << "</html>" << endl << "</body>" << endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
# build artifacts
|
||||
largeNbDicts
|
||||
@@ -1,58 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
PROGDIR = ../../programs
|
||||
LIBDIR = ../../lib
|
||||
|
||||
LIBZSTD = $(LIBDIR)/libzstd.a
|
||||
|
||||
CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=gnu99
|
||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
|
||||
|
||||
default: largeNbDicts
|
||||
|
||||
all : largeNbDicts
|
||||
|
||||
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
.PHONY: $(LIBZSTD)
|
||||
$(LIBZSTD):
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
|
||||
|
||||
benchfn.o: $(PROGDIR)/benchfn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
timefn.o: $(PROGDIR)/timefn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
datagen.o: $(PROGDIR)/datagen.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
util.o: $(PROGDIR)/util.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
xxhash.o : $(LIBDIR)/common/xxhash.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
clean:
|
||||
$(RM) *.o
|
||||
$(MAKE) -C $(LIBDIR) clean > /dev/null
|
||||
$(RM) largeNbDicts
|
||||
@@ -1,25 +0,0 @@
|
||||
largeNbDicts
|
||||
=====================
|
||||
|
||||
`largeNbDicts` is a benchmark test tool
|
||||
dedicated to the specific scenario of
|
||||
dictionary decompression using a very large number of dictionaries.
|
||||
When dictionaries are constantly changing, they are always "cold",
|
||||
suffering from increased latency due to cache misses.
|
||||
|
||||
The tool is created in a bid to investigate performance for this scenario,
|
||||
and experiment mitigation techniques.
|
||||
|
||||
Command line :
|
||||
```
|
||||
largeNbDicts [Options] filename(s)
|
||||
|
||||
Options :
|
||||
-r : recursively load all files in subdirectories (default: off)
|
||||
-B# : split input into blocks of size # (default: no split)
|
||||
-# : use compression level # (default: 3)
|
||||
-D # : use # as a dictionary (default: create one)
|
||||
-i# : nb benchmark rounds (default: 6)
|
||||
--nbDicts=# : set nb of dictionaries to # (default: one per block)
|
||||
-h : help (this text)
|
||||
```
|
||||
@@ -1,979 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* largeNbDicts
|
||||
* This is a benchmark test tool
|
||||
* dedicated to the specific case of dictionary decompression
|
||||
* using a very large nb of dictionaries
|
||||
* thus suffering latency from lots of cache misses.
|
||||
* It's created in a bid to investigate performance and find optimizations. */
|
||||
|
||||
|
||||
/*--- Dependencies ---*/
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, abort */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <limits.h> /* UINT_MAX */
|
||||
#include <assert.h> /* assert */
|
||||
|
||||
#include "util.h"
|
||||
#include "benchfn.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*--- Constants --- */
|
||||
|
||||
#define KB *(1<<10)
|
||||
#define MB *(1<<20)
|
||||
|
||||
#define BLOCKSIZE_DEFAULT 0 /* no slicing into blocks */
|
||||
#define DICTSIZE (4 KB)
|
||||
#define CLEVEL_DEFAULT 3
|
||||
|
||||
#define BENCH_TIME_DEFAULT_S 6
|
||||
#define RUN_TIME_DEFAULT_MS 1000
|
||||
#define BENCH_TIME_DEFAULT_MS (BENCH_TIME_DEFAULT_S * RUN_TIME_DEFAULT_MS)
|
||||
|
||||
#define DISPLAY_LEVEL_DEFAULT 3
|
||||
|
||||
#define BENCH_SIZE_MAX (1200 MB)
|
||||
|
||||
|
||||
/*--- Macros ---*/
|
||||
|
||||
#define CONTROL(c) { if (!(c)) abort(); }
|
||||
#undef MIN
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
|
||||
/*--- Display Macros ---*/
|
||||
|
||||
#define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
|
||||
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
|
||||
|
||||
|
||||
/*--- buffer_t ---*/
|
||||
|
||||
typedef struct {
|
||||
void* ptr;
|
||||
size_t size;
|
||||
size_t capacity;
|
||||
} buffer_t;
|
||||
|
||||
static const buffer_t kBuffNull = { NULL, 0, 0 };
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t createBuffer(size_t capacity)
|
||||
{
|
||||
assert(capacity > 0);
|
||||
void* const ptr = malloc(capacity);
|
||||
if (ptr==NULL) return kBuffNull;
|
||||
|
||||
buffer_t buffer;
|
||||
buffer.ptr = ptr;
|
||||
buffer.capacity = capacity;
|
||||
buffer.size = 0;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static void freeBuffer(buffer_t buff)
|
||||
{
|
||||
free(buff.ptr);
|
||||
}
|
||||
|
||||
|
||||
static void fillBuffer_fromHandle(buffer_t* buff, FILE* f)
|
||||
{
|
||||
size_t const readSize = fread(buff->ptr, 1, buff->capacity, f);
|
||||
buff->size = readSize;
|
||||
}
|
||||
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t createBuffer_fromFile(const char* fileName)
|
||||
{
|
||||
U64 const fileSize = UTIL_getFileSize(fileName);
|
||||
size_t const bufferSize = (size_t) fileSize;
|
||||
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull;
|
||||
assert((U64)bufferSize == fileSize); /* check overflow */
|
||||
|
||||
{ FILE* const f = fopen(fileName, "rb");
|
||||
if (f == NULL) return kBuffNull;
|
||||
|
||||
buffer_t buff = createBuffer(bufferSize);
|
||||
CONTROL(buff.ptr != NULL);
|
||||
|
||||
fillBuffer_fromHandle(&buff, f);
|
||||
CONTROL(buff.size == buff.capacity);
|
||||
|
||||
fclose(f); /* do nothing specific if fclose() fails */
|
||||
return buff;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t
|
||||
createDictionaryBuffer(const char* dictionaryName,
|
||||
const void* srcBuffer,
|
||||
const size_t* srcBlockSizes, size_t nbBlocks,
|
||||
size_t requestedDictSize)
|
||||
{
|
||||
if (dictionaryName) {
|
||||
DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName);
|
||||
return createBuffer_fromFile(dictionaryName); /* note : result might be kBuffNull */
|
||||
|
||||
} else {
|
||||
|
||||
DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n",
|
||||
(unsigned)requestedDictSize);
|
||||
void* const dictBuffer = malloc(requestedDictSize);
|
||||
CONTROL(dictBuffer != NULL);
|
||||
|
||||
assert(nbBlocks <= UINT_MAX);
|
||||
size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
|
||||
srcBuffer,
|
||||
srcBlockSizes, (unsigned)nbBlocks);
|
||||
CONTROL(!ZSTD_isError(dictSize));
|
||||
|
||||
buffer_t result;
|
||||
result.ptr = dictBuffer;
|
||||
result.capacity = requestedDictSize;
|
||||
result.size = dictSize;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
* Loads `buffer`, with content from files listed within `fileNamesTable`.
|
||||
* Fills `buffer` entirely.
|
||||
* @return : 0 on success, !=0 on error */
|
||||
static int loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char* const * fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0, totalSize = 0;
|
||||
|
||||
for (unsigned n=0; n<nbFiles; n++) {
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
assert(f!=NULL);
|
||||
|
||||
assert(pos <= bufferSize);
|
||||
assert(fileSize <= bufferSize - pos);
|
||||
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
assert(readSize == fileSize);
|
||||
pos += readSize;
|
||||
}
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
totalSize += (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
assert(totalSize == bufferSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*--- slice_collection_t ---*/
|
||||
|
||||
typedef struct {
|
||||
void** slicePtrs;
|
||||
size_t* capacities;
|
||||
size_t nbSlices;
|
||||
} slice_collection_t;
|
||||
|
||||
static const slice_collection_t kNullCollection = { NULL, NULL, 0 };
|
||||
|
||||
static void freeSliceCollection(slice_collection_t collection)
|
||||
{
|
||||
free(collection.slicePtrs);
|
||||
free(collection.capacities);
|
||||
}
|
||||
|
||||
/* shrinkSizes() :
|
||||
* downsizes sizes of slices within collection, according to `newSizes`.
|
||||
* every `newSizes` entry must be <= than its corresponding collection size */
|
||||
void shrinkSizes(slice_collection_t collection,
|
||||
const size_t* newSizes) /* presumed same size as collection */
|
||||
{
|
||||
size_t const nbSlices = collection.nbSlices;
|
||||
for (size_t blockNb = 0; blockNb < nbSlices; blockNb++) {
|
||||
assert(newSizes[blockNb] <= collection.capacities[blockNb]);
|
||||
collection.capacities[blockNb] = newSizes[blockNb];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* splitSlices() :
|
||||
* nbSlices : if == 0, nbSlices is automatically determined from srcSlices and blockSize.
|
||||
* otherwise, creates exactly nbSlices slices,
|
||||
* by either truncating input (when smaller)
|
||||
* or repeating input from beginning */
|
||||
static slice_collection_t
|
||||
splitSlices(slice_collection_t srcSlices, size_t blockSize, size_t nbSlices)
|
||||
{
|
||||
if (blockSize==0) blockSize = (size_t)(-1); /* means "do not cut" */
|
||||
size_t nbSrcBlocks = 0;
|
||||
for (size_t ssnb=0; ssnb < srcSlices.nbSlices; ssnb++) {
|
||||
size_t pos = 0;
|
||||
while (pos <= srcSlices.capacities[ssnb]) {
|
||||
nbSrcBlocks++;
|
||||
pos += blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
if (nbSlices == 0) nbSlices = nbSrcBlocks;
|
||||
|
||||
void** const sliceTable = (void**)malloc(nbSlices * sizeof(*sliceTable));
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
if (sliceTable == NULL || capacities == NULL) {
|
||||
free(sliceTable);
|
||||
free(capacities);
|
||||
return kNullCollection;
|
||||
}
|
||||
|
||||
size_t ssnb = 0;
|
||||
for (size_t sliceNb=0; sliceNb < nbSlices; ) {
|
||||
ssnb = (ssnb + 1) % srcSlices.nbSlices;
|
||||
size_t pos = 0;
|
||||
char* const ptr = (char*)srcSlices.slicePtrs[ssnb];
|
||||
while (pos < srcSlices.capacities[ssnb] && sliceNb < nbSlices) {
|
||||
size_t const size = MIN(blockSize, srcSlices.capacities[ssnb] - pos);
|
||||
sliceTable[sliceNb] = ptr + pos;
|
||||
capacities[sliceNb] = size;
|
||||
sliceNb++;
|
||||
pos += blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
slice_collection_t result;
|
||||
result.nbSlices = nbSlices;
|
||||
result.slicePtrs = sliceTable;
|
||||
result.capacities = capacities;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
static size_t sliceCollection_totalCapacity(slice_collection_t sc)
|
||||
{
|
||||
size_t totalSize = 0;
|
||||
for (size_t n=0; n<sc.nbSlices; n++)
|
||||
totalSize += sc.capacities[n];
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
|
||||
/* --- buffer collection --- */
|
||||
|
||||
typedef struct {
|
||||
buffer_t buffer;
|
||||
slice_collection_t slices;
|
||||
} buffer_collection_t;
|
||||
|
||||
|
||||
static void freeBufferCollection(buffer_collection_t bc)
|
||||
{
|
||||
freeBuffer(bc.buffer);
|
||||
freeSliceCollection(bc.slices);
|
||||
}
|
||||
|
||||
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc)
|
||||
{
|
||||
size_t const bufferSize = sliceCollection_totalCapacity(sc);
|
||||
|
||||
buffer_t buffer = createBuffer(bufferSize);
|
||||
CONTROL(buffer.ptr != NULL);
|
||||
|
||||
size_t const nbSlices = sc.nbSlices;
|
||||
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
|
||||
CONTROL(slices != NULL);
|
||||
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
CONTROL(capacities != NULL);
|
||||
|
||||
char* const ptr = (char*)buffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t n=0; n < nbSlices; n++) {
|
||||
capacities[n] = sc.capacities[n];
|
||||
slices[n] = ptr + pos;
|
||||
pos += capacities[n];
|
||||
}
|
||||
|
||||
buffer_collection_t result;
|
||||
result.buffer = buffer;
|
||||
result.slices.nbSlices = nbSlices;
|
||||
result.slices.capacities = capacities;
|
||||
result.slices.slicePtrs = slices;
|
||||
return result;
|
||||
}
|
||||
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromSliceCollection(slice_collection_t sc)
|
||||
{
|
||||
size_t const bufferSize = sliceCollection_totalCapacity(sc);
|
||||
|
||||
buffer_t buffer = createBuffer(bufferSize);
|
||||
CONTROL(buffer.ptr != NULL);
|
||||
|
||||
size_t const nbSlices = sc.nbSlices;
|
||||
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
|
||||
CONTROL(slices != NULL);
|
||||
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
CONTROL(capacities != NULL);
|
||||
|
||||
char* const ptr = (char*)buffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t n=0; n < nbSlices; n++) {
|
||||
capacities[n] = sc.capacities[n];
|
||||
slices[n] = ptr + pos;
|
||||
pos += capacities[n];
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < nbSlices; i++) {
|
||||
memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]);
|
||||
capacities[i] = sc.capacities[i];
|
||||
}
|
||||
|
||||
buffer_collection_t result;
|
||||
result.buffer = buffer;
|
||||
result.slices.nbSlices = nbSlices;
|
||||
result.slices.capacities = capacities;
|
||||
result.slices.slicePtrs = slices;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromFiles(const char* const * fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
assert(totalSizeToLoad != UTIL_FILESIZE_UNKNOWN);
|
||||
assert(totalSizeToLoad <= BENCH_SIZE_MAX);
|
||||
size_t const loadedSize = (size_t)totalSizeToLoad;
|
||||
assert(loadedSize > 0);
|
||||
void* const srcBuffer = malloc(loadedSize);
|
||||
assert(srcBuffer != NULL);
|
||||
|
||||
assert(nbFiles > 0);
|
||||
size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(*fileSizes));
|
||||
assert(fileSizes != NULL);
|
||||
|
||||
/* Load input buffer */
|
||||
int const errorCode = loadFiles(srcBuffer, loadedSize,
|
||||
fileSizes,
|
||||
fileNamesTable, nbFiles);
|
||||
assert(errorCode == 0);
|
||||
|
||||
void** sliceTable = (void**)malloc(nbFiles * sizeof(*sliceTable));
|
||||
assert(sliceTable != NULL);
|
||||
|
||||
char* const ptr = (char*)srcBuffer;
|
||||
size_t pos = 0;
|
||||
unsigned fileNb = 0;
|
||||
for ( ; (pos < loadedSize) && (fileNb < nbFiles); fileNb++) {
|
||||
sliceTable[fileNb] = ptr + pos;
|
||||
pos += fileSizes[fileNb];
|
||||
}
|
||||
assert(pos == loadedSize);
|
||||
assert(fileNb == nbFiles);
|
||||
|
||||
|
||||
buffer_t buffer;
|
||||
buffer.ptr = srcBuffer;
|
||||
buffer.capacity = loadedSize;
|
||||
buffer.size = loadedSize;
|
||||
|
||||
slice_collection_t slices;
|
||||
slices.slicePtrs = sliceTable;
|
||||
slices.capacities = fileSizes;
|
||||
slices.nbSlices = nbFiles;
|
||||
|
||||
buffer_collection_t bc;
|
||||
bc.buffer = buffer;
|
||||
bc.slices = slices;
|
||||
return bc;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*--- ddict_collection_t ---*/
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DDict** ddicts;
|
||||
size_t nbDDict;
|
||||
} ddict_collection_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CDict** cdicts;
|
||||
size_t nbCDict;
|
||||
} cdict_collection_t;
|
||||
|
||||
static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeCDictCollection(cdict_collection_t cdictc)
|
||||
{
|
||||
for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
|
||||
ZSTD_freeCDict(cdictc.cdicts[dictNb]);
|
||||
}
|
||||
free(cdictc.cdicts);
|
||||
}
|
||||
|
||||
/* returns .buffers=NULL if operation fails */
|
||||
static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel)
|
||||
{
|
||||
ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
|
||||
if (cdicts==NULL) return kNullCDictCollection;
|
||||
for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
|
||||
cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
|
||||
CONTROL(cdicts[dictNb] != NULL);
|
||||
}
|
||||
cdict_collection_t cdictc;
|
||||
cdictc.cdicts = cdicts;
|
||||
cdictc.nbCDict = nbCDict;
|
||||
return cdictc;
|
||||
}
|
||||
|
||||
static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeDDictCollection(ddict_collection_t ddictc)
|
||||
{
|
||||
for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) {
|
||||
ZSTD_freeDDict(ddictc.ddicts[dictNb]);
|
||||
}
|
||||
free(ddictc.ddicts);
|
||||
}
|
||||
|
||||
/* returns .buffers=NULL if operation fails */
|
||||
static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict)
|
||||
{
|
||||
ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*));
|
||||
assert(ddicts != NULL);
|
||||
if (ddicts==NULL) return kNullDDictCollection;
|
||||
for (size_t dictNb=0; dictNb < nbDDict; dictNb++) {
|
||||
ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize);
|
||||
assert(ddicts[dictNb] != NULL);
|
||||
}
|
||||
ddict_collection_t ddictc;
|
||||
ddictc.ddicts = ddicts;
|
||||
ddictc.nbDDict = nbDDict;
|
||||
return ddictc;
|
||||
}
|
||||
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleCDictionaries(cdict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbCDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d];
|
||||
dicts.cdicts[d] = dicts.cdicts[r];
|
||||
dicts.cdicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d1];
|
||||
dicts.cdicts[d1] = dicts.cdicts[d2];
|
||||
dicts.cdicts[d2] = tmpd;
|
||||
}
|
||||
}
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleDDictionaries(ddict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbDDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d];
|
||||
dicts.ddicts[d] = dicts.ddicts[r];
|
||||
dicts.ddicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d1];
|
||||
dicts.ddicts[d1] = dicts.ddicts[d2];
|
||||
dicts.ddicts[d2] = tmpd;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* --- Compression --- */
|
||||
|
||||
/* compressBlocks() :
|
||||
* @return : total compressed size of all blocks,
|
||||
* or 0 if error.
|
||||
*/
|
||||
static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If present, must contain at least nbBlocks fields */
|
||||
slice_collection_t dstBlockBuffers,
|
||||
slice_collection_t srcBlockBuffers,
|
||||
ZSTD_CDict* cdict, int cLevel)
|
||||
{
|
||||
size_t const nbBlocks = srcBlockBuffers.nbSlices;
|
||||
assert(dstBlockBuffers.nbSlices == srcBlockBuffers.nbSlices);
|
||||
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
assert(cctx != NULL);
|
||||
|
||||
size_t totalCSize = 0;
|
||||
for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) {
|
||||
size_t cBlockSize;
|
||||
if (cdict == NULL) {
|
||||
cBlockSize = ZSTD_compressCCtx(cctx,
|
||||
dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb],
|
||||
srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb],
|
||||
cLevel);
|
||||
} else {
|
||||
cBlockSize = ZSTD_compress_usingCDict(cctx,
|
||||
dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb],
|
||||
srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb],
|
||||
cdict);
|
||||
}
|
||||
CONTROL(!ZSTD_isError(cBlockSize));
|
||||
if (cSizes) cSizes[blockNb] = cBlockSize;
|
||||
totalCSize += cBlockSize;
|
||||
}
|
||||
return totalCSize;
|
||||
}
|
||||
|
||||
|
||||
/* --- Benchmark --- */
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CCtx* cctx;
|
||||
size_t nbDicts;
|
||||
size_t dictNb;
|
||||
cdict_collection_t dictionaries;
|
||||
} compressInstructions;
|
||||
|
||||
compressInstructions createCompressInstructions(cdict_collection_t dictionaries)
|
||||
{
|
||||
compressInstructions ci;
|
||||
ci.cctx = ZSTD_createCCtx();
|
||||
CONTROL(ci.cctx != NULL);
|
||||
ci.nbDicts = dictionaries.nbCDict;
|
||||
ci.dictNb = 0;
|
||||
ci.dictionaries = dictionaries;
|
||||
return ci;
|
||||
}
|
||||
|
||||
void freeCompressInstructions(compressInstructions ci)
|
||||
{
|
||||
ZSTD_freeCCtx(ci.cctx);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DCtx* dctx;
|
||||
size_t nbDicts;
|
||||
size_t dictNb;
|
||||
ddict_collection_t dictionaries;
|
||||
} decompressInstructions;
|
||||
|
||||
decompressInstructions createDecompressInstructions(ddict_collection_t dictionaries)
|
||||
{
|
||||
decompressInstructions di;
|
||||
di.dctx = ZSTD_createDCtx();
|
||||
assert(di.dctx != NULL);
|
||||
di.nbDicts = dictionaries.nbDDict;
|
||||
di.dictNb = 0;
|
||||
di.dictionaries = dictionaries;
|
||||
return di;
|
||||
}
|
||||
|
||||
void freeDecompressInstructions(decompressInstructions di)
|
||||
{
|
||||
ZSTD_freeDCtx(di.dctx);
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
compressInstructions* const ci = (compressInstructions*) payload;
|
||||
(void)dstCapacity;
|
||||
|
||||
ZSTD_compress_usingCDict(ci->cctx,
|
||||
dst, srcSize,
|
||||
src, srcSize,
|
||||
ci->dictionaries.cdicts[ci->dictNb]);
|
||||
|
||||
ci->dictNb = ci->dictNb + 1;
|
||||
if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
decompressInstructions* const di = (decompressInstructions*) payload;
|
||||
|
||||
size_t const result = ZSTD_decompress_usingDDict(di->dctx,
|
||||
dst, dstCapacity,
|
||||
src, srcSize,
|
||||
di->dictionaries.ddicts[di->dictNb]);
|
||||
|
||||
di->dictNb = di->dictNb + 1;
|
||||
if (di->dictNb >= di->nbDicts) di->dictNb = 0;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
static int benchMem(slice_collection_t dstBlocks,
|
||||
slice_collection_t srcBlocks,
|
||||
ddict_collection_t ddictionaries,
|
||||
cdict_collection_t cdictionaries,
|
||||
unsigned nbRounds, int benchCompression)
|
||||
{
|
||||
assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
|
||||
|
||||
unsigned const ms_per_round = RUN_TIME_DEFAULT_MS;
|
||||
unsigned const total_time_ms = nbRounds * ms_per_round;
|
||||
|
||||
double bestSpeed = 0.;
|
||||
|
||||
BMK_timedFnState_t* const benchState =
|
||||
BMK_createTimedFnState(total_time_ms, ms_per_round);
|
||||
|
||||
decompressInstructions di = createDecompressInstructions(ddictionaries);
|
||||
compressInstructions ci = createCompressInstructions(cdictionaries);
|
||||
void* payload = benchCompression ? (void*)&ci : (void*)&di;
|
||||
BMK_benchParams_t const bp = {
|
||||
.benchFn = benchCompression ? compress : decompress,
|
||||
.benchPayload = payload,
|
||||
.initFn = NULL,
|
||||
.initPayload = NULL,
|
||||
.errorFn = ZSTD_isError,
|
||||
.blockCount = dstBlocks.nbSlices,
|
||||
.srcBuffers = (const void* const*) srcBlocks.slicePtrs,
|
||||
.srcSizes = srcBlocks.capacities,
|
||||
.dstBuffers = dstBlocks.slicePtrs,
|
||||
.dstCapacities = dstBlocks.capacities,
|
||||
.blockResults = NULL
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, bp);
|
||||
CONTROL(BMK_isSuccessful_runOutcome(outcome));
|
||||
|
||||
BMK_runTime_t const result = BMK_extract_runTime(outcome);
|
||||
double const dTime_ns = result.nanoSecPerRun;
|
||||
double const dTime_sec = (double)dTime_ns / 1000000000;
|
||||
size_t const srcSize = result.sumOfReturn;
|
||||
double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
|
||||
if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
|
||||
if (benchCompression)
|
||||
DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
|
||||
else
|
||||
DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
|
||||
|
||||
fflush(stdout);
|
||||
if (BMK_isCompleted_TimedFn(benchState)) break;
|
||||
}
|
||||
DISPLAY("\n");
|
||||
|
||||
freeDecompressInstructions(di);
|
||||
freeCompressInstructions(ci);
|
||||
BMK_freeTimedFnState(benchState);
|
||||
|
||||
return 0; /* success */
|
||||
}
|
||||
|
||||
|
||||
/*! bench() :
|
||||
* fileName : file to load for benchmarking purpose
|
||||
* dictionary : optional (can be NULL), file to load as dictionary,
|
||||
* if none provided : will be calculated on the fly by the program.
|
||||
* @return : 0 is success, 1+ otherwise */
|
||||
int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
const char* dictionary,
|
||||
size_t blockSize, int clevel,
|
||||
unsigned nbDictMax, unsigned nbBlocks,
|
||||
unsigned nbRounds, int benchCompression)
|
||||
{
|
||||
int result = 0;
|
||||
|
||||
DISPLAYLEVEL(3, "loading %u files... \n", nbFiles);
|
||||
buffer_collection_t const srcs = createBufferCollection_fromFiles(fileNameTable, nbFiles);
|
||||
CONTROL(srcs.buffer.ptr != NULL);
|
||||
buffer_t srcBuffer = srcs.buffer;
|
||||
size_t const srcSize = srcBuffer.size;
|
||||
DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n",
|
||||
(double)srcSize / (1 MB));
|
||||
|
||||
slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize, nbBlocks);
|
||||
nbBlocks = (unsigned)(srcSlices.nbSlices);
|
||||
DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks);
|
||||
if (blockSize)
|
||||
DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
|
||||
DISPLAYLEVEL(3, "\n");
|
||||
size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
|
||||
|
||||
|
||||
size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
|
||||
CONTROL(dstCapacities != NULL);
|
||||
size_t dstBufferCapacity = 0;
|
||||
for (size_t bnb=0; bnb<nbBlocks; bnb++) {
|
||||
dstCapacities[bnb] = ZSTD_compressBound(srcSlices.capacities[bnb]);
|
||||
dstBufferCapacity += dstCapacities[bnb];
|
||||
}
|
||||
|
||||
buffer_t dstBuffer = createBuffer(dstBufferCapacity);
|
||||
CONTROL(dstBuffer.ptr != NULL);
|
||||
|
||||
void** const sliceTable = malloc(nbBlocks * sizeof(*sliceTable));
|
||||
CONTROL(sliceTable != NULL);
|
||||
|
||||
{ char* const ptr = dstBuffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t snb=0; snb < nbBlocks; snb++) {
|
||||
sliceTable[snb] = ptr + pos;
|
||||
pos += dstCapacities[snb];
|
||||
} }
|
||||
|
||||
slice_collection_t dstSlices;
|
||||
dstSlices.capacities = dstCapacities;
|
||||
dstSlices.slicePtrs = sliceTable;
|
||||
dstSlices.nbSlices = nbBlocks;
|
||||
|
||||
|
||||
/* dictionary determination */
|
||||
buffer_t const dictBuffer = createDictionaryBuffer(dictionary,
|
||||
srcs.buffer.ptr,
|
||||
srcs.slices.capacities, srcs.slices.nbSlices,
|
||||
DICTSIZE);
|
||||
CONTROL(dictBuffer.ptr != NULL);
|
||||
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel);
|
||||
CONTROL(cdict != NULL);
|
||||
|
||||
size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel);
|
||||
CONTROL(cTotalSizeNoDict != 0);
|
||||
DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
clevel,
|
||||
(double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
|
||||
|
||||
size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
|
||||
CONTROL(cSizes != NULL);
|
||||
|
||||
size_t const cTotalSize = compressBlocks(cSizes, dstSlices, srcSlices, cdict, clevel);
|
||||
CONTROL(cTotalSize != 0);
|
||||
DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
(unsigned)dictBuffer.size,
|
||||
(double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
|
||||
|
||||
/* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
|
||||
shrinkSizes(dstSlices, cSizes);
|
||||
|
||||
unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
|
||||
|
||||
cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel);
|
||||
CONTROL(cdictionaries.cdicts != NULL);
|
||||
|
||||
ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
|
||||
CONTROL(ddictionaries.ddicts != NULL);
|
||||
|
||||
if (benchCompression) {
|
||||
size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
shuffleCDictionaries(cdictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
} else {
|
||||
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
shuffleDDictionaries(ddictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
}
|
||||
|
||||
/* free all heap objects in reverse order */
|
||||
freeCDictCollection(cdictionaries);
|
||||
freeDDictCollection(ddictionaries);
|
||||
free(cSizes);
|
||||
ZSTD_freeCDict(cdict);
|
||||
freeBuffer(dictBuffer);
|
||||
freeSliceCollection(dstSlices);
|
||||
freeBuffer(dstBuffer);
|
||||
freeSliceCollection(srcSlices);
|
||||
freeBufferCollection(srcs);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* --- Command Line --- */
|
||||
|
||||
/*! readU32FromChar() :
|
||||
* @return : unsigned integer value read from input in `char` format.
|
||||
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
* Note : function will exit() program if digit sequence overflows */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
assert(result <= max); /* check overflow */
|
||||
result *= 10, result += (unsigned)**stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
assert(result <= maxK); /* check overflow */
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') {
|
||||
assert(result <= maxK); /* check overflow */
|
||||
result <<= 10;
|
||||
}
|
||||
(*stringPtr)++; /* skip `K` or `M` */
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check if *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static int longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int usage(const char* exeName)
|
||||
{
|
||||
DISPLAY (" \n");
|
||||
DISPLAY (" %s [Options] filename(s) \n", exeName);
|
||||
DISPLAY (" \n");
|
||||
DISPLAY ("Options : \n");
|
||||
DISPLAY ("-z : benchmark compression (default) \n");
|
||||
DISPLAY ("-d : benchmark decompression \n");
|
||||
DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n");
|
||||
DISPLAY ("-B# : split input into blocks of size # (default: no split) \n");
|
||||
DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
|
||||
DISPLAY ("-D # : use # as a dictionary (default: create one) \n");
|
||||
DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
|
||||
DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
|
||||
DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
|
||||
DISPLAY ("-h : help (this text) \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bad_usage(const char* exeName)
|
||||
{
|
||||
DISPLAY (" bad usage : \n");
|
||||
usage(exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main (int argc, const char** argv)
|
||||
{
|
||||
int recursiveMode = 0;
|
||||
int benchCompression = 1;
|
||||
unsigned nbRounds = BENCH_TIME_DEFAULT_S;
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc < 2) return bad_usage(exeName);
|
||||
|
||||
const char** nameTable = (const char**)malloc((size_t)argc * sizeof(const char*));
|
||||
assert(nameTable != NULL);
|
||||
unsigned nameIdx = 0;
|
||||
|
||||
const char* dictionary = NULL;
|
||||
int cLevel = CLEVEL_DEFAULT;
|
||||
size_t blockSize = BLOCKSIZE_DEFAULT;
|
||||
unsigned nbDicts = 0; /* determine nbDicts automatically: 1 dictionary per block */
|
||||
unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
|
||||
|
||||
for (int argNb = 1; argNb < argc ; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
|
||||
if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
|
||||
if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
|
||||
if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
|
||||
if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
|
||||
if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--dictionary=")) { dictionary = argument; continue; }
|
||||
if (longCommandWArg(&argument, "-B")) { blockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
/* anything that's not a command is a filename */
|
||||
nameTable[nameIdx++] = argument;
|
||||
}
|
||||
|
||||
FileNamesTable* filenameTable;
|
||||
|
||||
if (recursiveMode) {
|
||||
#ifndef UTIL_HAS_CREATEFILELIST
|
||||
assert(0); /* missing capability, do not run */
|
||||
#endif
|
||||
filenameTable = UTIL_createExpandedFNT(nameTable, nameIdx, 1 /* follow_links */);
|
||||
} else {
|
||||
filenameTable = UTIL_assembleFileNamesTable(nameTable, nameIdx, NULL);
|
||||
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
|
||||
}
|
||||
|
||||
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression);
|
||||
|
||||
UTIL_freeFileNamesTable(filenameTable);
|
||||
free(nameTable);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
!lib/zstd
|
||||
!lib/zstd/*
|
||||
*.o
|
||||
*.a
|
||||
@@ -1,122 +0,0 @@
|
||||
From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Tue, 8 Aug 2017 19:20:25 -0700
|
||||
Subject: [PATCH v5 0/5] Add xxhash and zstd modules
|
||||
|
||||
Hi all,
|
||||
|
||||
This patch set adds xxhash, zstd compression, and zstd decompression
|
||||
modules. It also adds zstd support to BtrFS and SquashFS.
|
||||
|
||||
Each patch has relevant summaries, benchmarks, and tests.
|
||||
|
||||
Best,
|
||||
Nick Terrell
|
||||
|
||||
Changelog:
|
||||
|
||||
v1 -> v2:
|
||||
- Make pointer in lib/xxhash.c:394 non-const (1/5)
|
||||
- Use div_u64() for division of u64s (2/5)
|
||||
- Reduce stack usage of ZSTD_compressSequences(), ZSTD_buildSeqTable(),
|
||||
ZSTD_decompressSequencesLong(), FSE_buildDTable(), FSE_decompress_wksp(),
|
||||
HUF_writeCTable(), HUF_readStats(), HUF_readCTable(),
|
||||
HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/5)
|
||||
- No zstd function uses more than 400 B of stack space (2/5)
|
||||
|
||||
v2 -> v3:
|
||||
- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
|
||||
(2/5)
|
||||
- Fix bug in dictionary compression from upstream commit cc1522351f (2/5)
|
||||
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/5)
|
||||
- Change default compression level for BtrFS to 3 (3/5)
|
||||
|
||||
v3 -> v4:
|
||||
- Fix compiler warnings (2/5)
|
||||
- Add missing includes (3/5)
|
||||
- Fix minor linter warnings (3/5, 4/5)
|
||||
- Add crypto patch (5/5)
|
||||
|
||||
v4 -> v5:
|
||||
- Fix rare compression bug from upstream commit 308047eb5d (2/5)
|
||||
- Fix bug introduced in v3 when working around the gcc-7 bug (2/5)
|
||||
- Fix ZSTD_DStream initialization code in squashfs (4/5)
|
||||
- Fix patch documentation for patches written by Sean Purcell (4/5)
|
||||
|
||||
Nick Terrell (5):
|
||||
lib: Add xxhash module
|
||||
lib: Add zstd modules
|
||||
btrfs: Add zstd support
|
||||
squashfs: Add zstd support
|
||||
crypto: Add zstd support
|
||||
|
||||
crypto/Kconfig | 9 +
|
||||
crypto/Makefile | 1 +
|
||||
crypto/testmgr.c | 10 +
|
||||
crypto/testmgr.h | 71 +
|
||||
crypto/zstd.c | 265 ++++
|
||||
fs/btrfs/Kconfig | 2 +
|
||||
fs/btrfs/Makefile | 2 +-
|
||||
fs/btrfs/compression.c | 1 +
|
||||
fs/btrfs/compression.h | 6 +-
|
||||
fs/btrfs/ctree.h | 1 +
|
||||
fs/btrfs/disk-io.c | 2 +
|
||||
fs/btrfs/ioctl.c | 6 +-
|
||||
fs/btrfs/props.c | 6 +
|
||||
fs/btrfs/super.c | 12 +-
|
||||
fs/btrfs/sysfs.c | 2 +
|
||||
fs/btrfs/zstd.c | 432 ++++++
|
||||
fs/squashfs/Kconfig | 14 +
|
||||
fs/squashfs/Makefile | 1 +
|
||||
fs/squashfs/decompressor.c | 7 +
|
||||
fs/squashfs/decompressor.h | 4 +
|
||||
fs/squashfs/squashfs_fs.h | 1 +
|
||||
fs/squashfs/zstd_wrapper.c | 151 ++
|
||||
include/linux/xxhash.h | 236 +++
|
||||
include/linux/zstd.h | 1157 +++++++++++++++
|
||||
include/uapi/linux/btrfs.h | 8 +-
|
||||
lib/Kconfig | 11 +
|
||||
lib/Makefile | 3 +
|
||||
lib/xxhash.c | 500 +++++++
|
||||
lib/zstd/Makefile | 18 +
|
||||
lib/zstd/bitstream.h | 374 +++++
|
||||
lib/zstd/compress.c | 3484 ++++++++++++++++++++++++++++++++++++++++++++
|
||||
lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++
|
||||
lib/zstd/entropy_common.c | 243 +++
|
||||
lib/zstd/error_private.h | 53 +
|
||||
lib/zstd/fse.h | 575 ++++++++
|
||||
lib/zstd/fse_compress.c | 795 ++++++++++
|
||||
lib/zstd/fse_decompress.c | 332 +++++
|
||||
lib/zstd/huf.h | 212 +++
|
||||
lib/zstd/huf_compress.c | 770 ++++++++++
|
||||
lib/zstd/huf_decompress.c | 960 ++++++++++++
|
||||
lib/zstd/mem.h | 151 ++
|
||||
lib/zstd/zstd_common.c | 75 +
|
||||
lib/zstd/zstd_internal.h | 263 ++++
|
||||
lib/zstd/zstd_opt.h | 1014 +++++++++++++
|
||||
44 files changed, 14756 insertions(+), 12 deletions(-)
|
||||
create mode 100644 crypto/zstd.c
|
||||
create mode 100644 fs/btrfs/zstd.c
|
||||
create mode 100644 fs/squashfs/zstd_wrapper.c
|
||||
create mode 100644 include/linux/xxhash.h
|
||||
create mode 100644 include/linux/zstd.h
|
||||
create mode 100644 lib/xxhash.c
|
||||
create mode 100644 lib/zstd/Makefile
|
||||
create mode 100644 lib/zstd/bitstream.h
|
||||
create mode 100644 lib/zstd/compress.c
|
||||
create mode 100644 lib/zstd/decompress.c
|
||||
create mode 100644 lib/zstd/entropy_common.c
|
||||
create mode 100644 lib/zstd/error_private.h
|
||||
create mode 100644 lib/zstd/fse.h
|
||||
create mode 100644 lib/zstd/fse_compress.c
|
||||
create mode 100644 lib/zstd/fse_decompress.c
|
||||
create mode 100644 lib/zstd/huf.h
|
||||
create mode 100644 lib/zstd/huf_compress.c
|
||||
create mode 100644 lib/zstd/huf_decompress.c
|
||||
create mode 100644 lib/zstd/mem.h
|
||||
create mode 100644 lib/zstd/zstd_common.c
|
||||
create mode 100644 lib/zstd/zstd_internal.h
|
||||
create mode 100644 lib/zstd/zstd_opt.h
|
||||
|
||||
--
|
||||
2.9.3
|
||||
@@ -1,862 +0,0 @@
|
||||
From a4b1ffb6e89bbccd519f9afa0910635668436105 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Mon, 17 Jul 2017 17:07:18 -0700
|
||||
Subject: [PATCH v5 1/5] lib: Add xxhash module
|
||||
|
||||
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
|
||||
extremely fast non-cryptographic hash algorithm for checksumming.
|
||||
The zstd compression and decompression modules added in the next patch
|
||||
require xxhash. I extracted it out from zstd since it is useful on its
|
||||
own. I copied the code from the upstream XXHash source repository and
|
||||
translated it into kernel style. I ran benchmarks and tests in the kernel
|
||||
and tests in userland.
|
||||
|
||||
I benchmarked xxhash as a special character device. I ran in four modes,
|
||||
no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to
|
||||
kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute
|
||||
hashes on the copied data. I also ran it with four different buffer sizes.
|
||||
The benchmark file is located in the upstream zstd source repository under
|
||||
`contrib/linux-kernel/xxhash_test.c` [1].
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs`
|
||||
from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large.
|
||||
Run the following commands for the benchmark:
|
||||
|
||||
modprobe xxhash_test
|
||||
mknod xxhash_test c 245 0
|
||||
time cp filesystem.squashfs xxhash_test
|
||||
|
||||
The time is reported by the time of the userland `cp`.
|
||||
The GB/s is computed with
|
||||
|
||||
1,536,217,008 B / time(buffer size, hash)
|
||||
|
||||
which includes the time to copy from userland.
|
||||
The Normalized GB/s is computed with
|
||||
|
||||
1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)).
|
||||
|
||||
|
||||
| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s |
|
||||
|-----------------|-------|----------|------|---------------|
|
||||
| 1024 | none | 0.408 | 3.77 | - |
|
||||
| 1024 | xxh32 | 0.649 | 2.37 | 6.37 |
|
||||
| 1024 | xxh64 | 0.542 | 2.83 | 11.46 |
|
||||
| 1024 | crc32 | 1.290 | 1.19 | 1.74 |
|
||||
| 4096 | none | 0.380 | 4.04 | - |
|
||||
| 4096 | xxh32 | 0.645 | 2.38 | 5.79 |
|
||||
| 4096 | xxh64 | 0.500 | 3.07 | 12.80 |
|
||||
| 4096 | crc32 | 1.168 | 1.32 | 1.95 |
|
||||
| 8192 | none | 0.351 | 4.38 | - |
|
||||
| 8192 | xxh32 | 0.614 | 2.50 | 5.84 |
|
||||
| 8192 | xxh64 | 0.464 | 3.31 | 13.60 |
|
||||
| 8192 | crc32 | 1.163 | 1.32 | 1.89 |
|
||||
| 16384 | none | 0.346 | 4.43 | - |
|
||||
| 16384 | xxh32 | 0.590 | 2.60 | 6.30 |
|
||||
| 16384 | xxh64 | 0.466 | 3.30 | 12.80 |
|
||||
| 16384 | crc32 | 1.183 | 1.30 | 1.84 |
|
||||
|
||||
Tested in userland using the test-suite in the zstd repo under
|
||||
`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the
|
||||
kernel functions. A line in each branch of every function in `xxhash.c`
|
||||
was commented out to ensure that the test-suite fails. Additionally
|
||||
tested while testing zstd and with SMHasher [3].
|
||||
|
||||
[1] https://phabricator.intern.facebook.com/P57526246
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp
|
||||
[3] https://github.com/aappleby/smhasher
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
XXHash source repository: https://github.com/cyan4973/xxhash
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v1 -> v2:
|
||||
- Make pointer in lib/xxhash.c:394 non-const
|
||||
|
||||
include/linux/xxhash.h | 236 +++++++++++++++++++++++
|
||||
lib/Kconfig | 3 +
|
||||
lib/Makefile | 1 +
|
||||
lib/xxhash.c | 500 +++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
4 files changed, 740 insertions(+)
|
||||
create mode 100644 include/linux/xxhash.h
|
||||
create mode 100644 lib/xxhash.c
|
||||
|
||||
diff --git a/include/linux/xxhash.h b/include/linux/xxhash.h
|
||||
new file mode 100644
|
||||
index 0000000..9e1f42c
|
||||
--- /dev/null
|
||||
+++ b/include/linux/xxhash.h
|
||||
@@ -0,0 +1,236 @@
|
||||
+/*
|
||||
+ * xxHash - Extremely Fast Hash algorithm
|
||||
+ * Copyright (C) 2012-2016, Yann Collet.
|
||||
+ *
|
||||
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
+ *
|
||||
+ * Redistribution and use in source and binary forms, with or without
|
||||
+ * modification, are permitted provided that the following conditions are
|
||||
+ * met:
|
||||
+ *
|
||||
+ * * Redistributions of source code must retain the above copyright
|
||||
+ * notice, this list of conditions and the following disclaimer.
|
||||
+ * * Redistributions in binary form must reproduce the above
|
||||
+ * copyright notice, this list of conditions and the following disclaimer
|
||||
+ * in the documentation and/or other materials provided with the
|
||||
+ * distribution.
|
||||
+ *
|
||||
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it under
|
||||
+ * the terms of the GNU General Public License version 2 as published by the
|
||||
+ * Free Software Foundation. This program is dual-licensed; you may select
|
||||
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
+ * ("BSD").
|
||||
+ *
|
||||
+ * You can contact the author at:
|
||||
+ * - xxHash homepage: http://cyan4973.github.io/xxHash/
|
||||
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
+ */
|
||||
+
|
||||
+/*
|
||||
+ * Notice extracted from xxHash homepage:
|
||||
+ *
|
||||
+ * xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
|
||||
+ * It also successfully passes all tests from the SMHasher suite.
|
||||
+ *
|
||||
+ * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2
|
||||
+ * Duo @3GHz)
|
||||
+ *
|
||||
+ * Name Speed Q.Score Author
|
||||
+ * xxHash 5.4 GB/s 10
|
||||
+ * CrapWow 3.2 GB/s 2 Andrew
|
||||
+ * MumurHash 3a 2.7 GB/s 10 Austin Appleby
|
||||
+ * SpookyHash 2.0 GB/s 10 Bob Jenkins
|
||||
+ * SBox 1.4 GB/s 9 Bret Mulvey
|
||||
+ * Lookup3 1.2 GB/s 9 Bob Jenkins
|
||||
+ * SuperFastHash 1.2 GB/s 1 Paul Hsieh
|
||||
+ * CityHash64 1.05 GB/s 10 Pike & Alakuijala
|
||||
+ * FNV 0.55 GB/s 5 Fowler, Noll, Vo
|
||||
+ * CRC32 0.43 GB/s 9
|
||||
+ * MD5-32 0.33 GB/s 10 Ronald L. Rivest
|
||||
+ * SHA1-32 0.28 GB/s 10
|
||||
+ *
|
||||
+ * Q.Score is a measure of quality of the hash function.
|
||||
+ * It depends on successfully passing SMHasher test set.
|
||||
+ * 10 is a perfect score.
|
||||
+ *
|
||||
+ * A 64-bits version, named xxh64 offers much better speed,
|
||||
+ * but for 64-bits applications only.
|
||||
+ * Name Speed on 64 bits Speed on 32 bits
|
||||
+ * xxh64 13.8 GB/s 1.9 GB/s
|
||||
+ * xxh32 6.8 GB/s 6.0 GB/s
|
||||
+ */
|
||||
+
|
||||
+#ifndef XXHASH_H
|
||||
+#define XXHASH_H
|
||||
+
|
||||
+#include <linux/types.h>
|
||||
+
|
||||
+/*-****************************
|
||||
+ * Simple Hash Functions
|
||||
+ *****************************/
|
||||
+
|
||||
+/**
|
||||
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
|
||||
+ *
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ * @seed: The seed can be used to alter the result predictably.
|
||||
+ *
|
||||
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
|
||||
+ *
|
||||
+ * Return: The 32-bit hash of the data.
|
||||
+ */
|
||||
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64() - calculate the 64-bit hash of the input with a given seed.
|
||||
+ *
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ * @seed: The seed can be used to alter the result predictably.
|
||||
+ *
|
||||
+ * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems.
|
||||
+ *
|
||||
+ * Return: The 64-bit hash of the data.
|
||||
+ */
|
||||
+uint64_t xxh64(const void *input, size_t length, uint64_t seed);
|
||||
+
|
||||
+/*-****************************
|
||||
+ * Streaming Hash Functions
|
||||
+ *****************************/
|
||||
+
|
||||
+/*
|
||||
+ * These definitions are only meant to allow allocation of XXH state
|
||||
+ * statically, on stack, or in a struct for example.
|
||||
+ * Do not use members directly.
|
||||
+ */
|
||||
+
|
||||
+/**
|
||||
+ * struct xxh32_state - private xxh32 state, do not use members directly
|
||||
+ */
|
||||
+struct xxh32_state {
|
||||
+ uint32_t total_len_32;
|
||||
+ uint32_t large_len;
|
||||
+ uint32_t v1;
|
||||
+ uint32_t v2;
|
||||
+ uint32_t v3;
|
||||
+ uint32_t v4;
|
||||
+ uint32_t mem32[4];
|
||||
+ uint32_t memsize;
|
||||
+};
|
||||
+
|
||||
+/**
|
||||
+ * struct xxh32_state - private xxh64 state, do not use members directly
|
||||
+ */
|
||||
+struct xxh64_state {
|
||||
+ uint64_t total_len;
|
||||
+ uint64_t v1;
|
||||
+ uint64_t v2;
|
||||
+ uint64_t v3;
|
||||
+ uint64_t v4;
|
||||
+ uint64_t mem64[4];
|
||||
+ uint32_t memsize;
|
||||
+};
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_reset() - reset the xxh32 state to start a new hashing operation
|
||||
+ *
|
||||
+ * @state: The xxh32 state to reset.
|
||||
+ * @seed: Initialize the hash state with this seed.
|
||||
+ *
|
||||
+ * Call this function on any xxh32_state to prepare for a new hashing operation.
|
||||
+ */
|
||||
+void xxh32_reset(struct xxh32_state *state, uint32_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_update() - hash the data given and update the xxh32 state
|
||||
+ *
|
||||
+ * @state: The xxh32 state to update.
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ *
|
||||
+ * After calling xxh32_reset() call xxh32_update() as many times as necessary.
|
||||
+ *
|
||||
+ * Return: Zero on success, otherwise an error code.
|
||||
+ */
|
||||
+int xxh32_update(struct xxh32_state *state, const void *input, size_t length);
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_digest() - produce the current xxh32 hash
|
||||
+ *
|
||||
+ * @state: Produce the current xxh32 hash of this state.
|
||||
+ *
|
||||
+ * A hash value can be produced at any time. It is still possible to continue
|
||||
+ * inserting input into the hash state after a call to xxh32_digest(), and
|
||||
+ * generate new hashes later on, by calling xxh32_digest() again.
|
||||
+ *
|
||||
+ * Return: The xxh32 hash stored in the state.
|
||||
+ */
|
||||
+uint32_t xxh32_digest(const struct xxh32_state *state);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_reset() - reset the xxh64 state to start a new hashing operation
|
||||
+ *
|
||||
+ * @state: The xxh64 state to reset.
|
||||
+ * @seed: Initialize the hash state with this seed.
|
||||
+ */
|
||||
+void xxh64_reset(struct xxh64_state *state, uint64_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_update() - hash the data given and update the xxh64 state
|
||||
+ * @state: The xxh64 state to update.
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ *
|
||||
+ * After calling xxh64_reset() call xxh64_update() as many times as necessary.
|
||||
+ *
|
||||
+ * Return: Zero on success, otherwise an error code.
|
||||
+ */
|
||||
+int xxh64_update(struct xxh64_state *state, const void *input, size_t length);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_digest() - produce the current xxh64 hash
|
||||
+ *
|
||||
+ * @state: Produce the current xxh64 hash of this state.
|
||||
+ *
|
||||
+ * A hash value can be produced at any time. It is still possible to continue
|
||||
+ * inserting input into the hash state after a call to xxh64_digest(), and
|
||||
+ * generate new hashes later on, by calling xxh64_digest() again.
|
||||
+ *
|
||||
+ * Return: The xxh64 hash stored in the state.
|
||||
+ */
|
||||
+uint64_t xxh64_digest(const struct xxh64_state *state);
|
||||
+
|
||||
+/*-**************************
|
||||
+ * Utils
|
||||
+ ***************************/
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_copy_state() - copy the source state into the destination state
|
||||
+ *
|
||||
+ * @src: The source xxh32 state.
|
||||
+ * @dst: The destination xxh32 state.
|
||||
+ */
|
||||
+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_copy_state() - copy the source state into the destination state
|
||||
+ *
|
||||
+ * @src: The source xxh64 state.
|
||||
+ * @dst: The destination xxh64 state.
|
||||
+ */
|
||||
+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src);
|
||||
+
|
||||
+#endif /* XXHASH_H */
|
||||
diff --git a/lib/Kconfig b/lib/Kconfig
|
||||
index 6762529..5e7541f 100644
|
||||
--- a/lib/Kconfig
|
||||
+++ b/lib/Kconfig
|
||||
@@ -192,6 +192,9 @@ config CRC8
|
||||
when they need to do cyclic redundancy check according CRC8
|
||||
algorithm. Module will be called crc8.
|
||||
|
||||
+config XXHASH
|
||||
+ tristate
|
||||
+
|
||||
config AUDIT_GENERIC
|
||||
bool
|
||||
depends on AUDIT && !AUDIT_ARCH
|
||||
diff --git a/lib/Makefile b/lib/Makefile
|
||||
index 40c1837..d06b68a 100644
|
||||
--- a/lib/Makefile
|
||||
+++ b/lib/Makefile
|
||||
@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4) += crc4.o
|
||||
obj-$(CONFIG_CRC7) += crc7.o
|
||||
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
|
||||
obj-$(CONFIG_CRC8) += crc8.o
|
||||
+obj-$(CONFIG_XXHASH) += xxhash.o
|
||||
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
|
||||
|
||||
obj-$(CONFIG_842_COMPRESS) += 842/
|
||||
diff --git a/lib/xxhash.c b/lib/xxhash.c
|
||||
new file mode 100644
|
||||
index 0000000..aa61e2a
|
||||
--- /dev/null
|
||||
+++ b/lib/xxhash.c
|
||||
@@ -0,0 +1,500 @@
|
||||
+/*
|
||||
+ * xxHash - Extremely Fast Hash algorithm
|
||||
+ * Copyright (C) 2012-2016, Yann Collet.
|
||||
+ *
|
||||
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
+ *
|
||||
+ * Redistribution and use in source and binary forms, with or without
|
||||
+ * modification, are permitted provided that the following conditions are
|
||||
+ * met:
|
||||
+ *
|
||||
+ * * Redistributions of source code must retain the above copyright
|
||||
+ * notice, this list of conditions and the following disclaimer.
|
||||
+ * * Redistributions in binary form must reproduce the above
|
||||
+ * copyright notice, this list of conditions and the following disclaimer
|
||||
+ * in the documentation and/or other materials provided with the
|
||||
+ * distribution.
|
||||
+ *
|
||||
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it under
|
||||
+ * the terms of the GNU General Public License version 2 as published by the
|
||||
+ * Free Software Foundation. This program is dual-licensed; you may select
|
||||
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
+ * ("BSD").
|
||||
+ *
|
||||
+ * You can contact the author at:
|
||||
+ * - xxHash homepage: http://cyan4973.github.io/xxHash/
|
||||
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
+ */
|
||||
+
|
||||
+#include <asm/unaligned.h>
|
||||
+#include <linux/errno.h>
|
||||
+#include <linux/compiler.h>
|
||||
+#include <linux/kernel.h>
|
||||
+#include <linux/module.h>
|
||||
+#include <linux/string.h>
|
||||
+#include <linux/xxhash.h>
|
||||
+
|
||||
+/*-*************************************
|
||||
+ * Macros
|
||||
+ **************************************/
|
||||
+#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r)))
|
||||
+#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r)))
|
||||
+
|
||||
+#ifdef __LITTLE_ENDIAN
|
||||
+# define XXH_CPU_LITTLE_ENDIAN 1
|
||||
+#else
|
||||
+# define XXH_CPU_LITTLE_ENDIAN 0
|
||||
+#endif
|
||||
+
|
||||
+/*-*************************************
|
||||
+ * Constants
|
||||
+ **************************************/
|
||||
+static const uint32_t PRIME32_1 = 2654435761U;
|
||||
+static const uint32_t PRIME32_2 = 2246822519U;
|
||||
+static const uint32_t PRIME32_3 = 3266489917U;
|
||||
+static const uint32_t PRIME32_4 = 668265263U;
|
||||
+static const uint32_t PRIME32_5 = 374761393U;
|
||||
+
|
||||
+static const uint64_t PRIME64_1 = 11400714785074694791ULL;
|
||||
+static const uint64_t PRIME64_2 = 14029467366897019727ULL;
|
||||
+static const uint64_t PRIME64_3 = 1609587929392839161ULL;
|
||||
+static const uint64_t PRIME64_4 = 9650029242287828579ULL;
|
||||
+static const uint64_t PRIME64_5 = 2870177450012600261ULL;
|
||||
+
|
||||
+/*-**************************
|
||||
+ * Utils
|
||||
+ ***************************/
|
||||
+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
|
||||
+{
|
||||
+ memcpy(dst, src, sizeof(*dst));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_copy_state);
|
||||
+
|
||||
+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
|
||||
+{
|
||||
+ memcpy(dst, src, sizeof(*dst));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_copy_state);
|
||||
+
|
||||
+/*-***************************
|
||||
+ * Simple Hash Functions
|
||||
+ ****************************/
|
||||
+static uint32_t xxh32_round(uint32_t seed, const uint32_t input)
|
||||
+{
|
||||
+ seed += input * PRIME32_2;
|
||||
+ seed = xxh_rotl32(seed, 13);
|
||||
+ seed *= PRIME32_1;
|
||||
+ return seed;
|
||||
+}
|
||||
+
|
||||
+uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *b_end = p + len;
|
||||
+ uint32_t h32;
|
||||
+
|
||||
+ if (len >= 16) {
|
||||
+ const uint8_t *const limit = b_end - 16;
|
||||
+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
+ uint32_t v2 = seed + PRIME32_2;
|
||||
+ uint32_t v3 = seed + 0;
|
||||
+ uint32_t v4 = seed - PRIME32_1;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) +
|
||||
+ xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18);
|
||||
+ } else {
|
||||
+ h32 = seed + PRIME32_5;
|
||||
+ }
|
||||
+
|
||||
+ h32 += (uint32_t)len;
|
||||
+
|
||||
+ while (p + 4 <= b_end) {
|
||||
+ h32 += get_unaligned_le32(p) * PRIME32_3;
|
||||
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h32 += (*p) * PRIME32_5;
|
||||
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h32 ^= h32 >> 15;
|
||||
+ h32 *= PRIME32_2;
|
||||
+ h32 ^= h32 >> 13;
|
||||
+ h32 *= PRIME32_3;
|
||||
+ h32 ^= h32 >> 16;
|
||||
+
|
||||
+ return h32;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32);
|
||||
+
|
||||
+static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
|
||||
+{
|
||||
+ acc += input * PRIME64_2;
|
||||
+ acc = xxh_rotl64(acc, 31);
|
||||
+ acc *= PRIME64_1;
|
||||
+ return acc;
|
||||
+}
|
||||
+
|
||||
+static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val)
|
||||
+{
|
||||
+ val = xxh64_round(0, val);
|
||||
+ acc ^= val;
|
||||
+ acc = acc * PRIME64_1 + PRIME64_4;
|
||||
+ return acc;
|
||||
+}
|
||||
+
|
||||
+uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+ uint64_t h64;
|
||||
+
|
||||
+ if (len >= 32) {
|
||||
+ const uint8_t *const limit = b_end - 32;
|
||||
+ uint64_t v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
+ uint64_t v2 = seed + PRIME64_2;
|
||||
+ uint64_t v3 = seed + 0;
|
||||
+ uint64_t v4 = seed - PRIME64_1;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
|
||||
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
|
||||
+ h64 = xxh64_merge_round(h64, v1);
|
||||
+ h64 = xxh64_merge_round(h64, v2);
|
||||
+ h64 = xxh64_merge_round(h64, v3);
|
||||
+ h64 = xxh64_merge_round(h64, v4);
|
||||
+
|
||||
+ } else {
|
||||
+ h64 = seed + PRIME64_5;
|
||||
+ }
|
||||
+
|
||||
+ h64 += (uint64_t)len;
|
||||
+
|
||||
+ while (p + 8 <= b_end) {
|
||||
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
|
||||
+
|
||||
+ h64 ^= k1;
|
||||
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
|
||||
+ p += 8;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 4 <= b_end) {
|
||||
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
|
||||
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h64 ^= (*p) * PRIME64_5;
|
||||
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h64 ^= h64 >> 33;
|
||||
+ h64 *= PRIME64_2;
|
||||
+ h64 ^= h64 >> 29;
|
||||
+ h64 *= PRIME64_3;
|
||||
+ h64 ^= h64 >> 32;
|
||||
+
|
||||
+ return h64;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64);
|
||||
+
|
||||
+/*-**************************************************
|
||||
+ * Advanced Hash Functions
|
||||
+ ***************************************************/
|
||||
+void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
|
||||
+{
|
||||
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
|
||||
+ struct xxh32_state state;
|
||||
+
|
||||
+ memset(&state, 0, sizeof(state));
|
||||
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
+ state.v2 = seed + PRIME32_2;
|
||||
+ state.v3 = seed + 0;
|
||||
+ state.v4 = seed - PRIME32_1;
|
||||
+ memcpy(statePtr, &state, sizeof(state));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_reset);
|
||||
+
|
||||
+void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
|
||||
+{
|
||||
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
|
||||
+ struct xxh64_state state;
|
||||
+
|
||||
+ memset(&state, 0, sizeof(state));
|
||||
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
+ state.v2 = seed + PRIME64_2;
|
||||
+ state.v3 = seed + 0;
|
||||
+ state.v4 = seed - PRIME64_1;
|
||||
+ memcpy(statePtr, &state, sizeof(state));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_reset);
|
||||
+
|
||||
+int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+
|
||||
+ if (input == NULL)
|
||||
+ return -EINVAL;
|
||||
+
|
||||
+ state->total_len_32 += (uint32_t)len;
|
||||
+ state->large_len |= (len >= 16) | (state->total_len_32 >= 16);
|
||||
+
|
||||
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
|
||||
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input, len);
|
||||
+ state->memsize += (uint32_t)len;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ if (state->memsize) { /* some data left from previous update */
|
||||
+ const uint32_t *p32 = state->mem32;
|
||||
+
|
||||
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input,
|
||||
+ 16 - state->memsize);
|
||||
+
|
||||
+ state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+
|
||||
+ p += 16-state->memsize;
|
||||
+ state->memsize = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (p <= b_end - 16) {
|
||||
+ const uint8_t *const limit = b_end - 16;
|
||||
+ uint32_t v1 = state->v1;
|
||||
+ uint32_t v2 = state->v2;
|
||||
+ uint32_t v3 = state->v3;
|
||||
+ uint32_t v4 = state->v4;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ state->v1 = v1;
|
||||
+ state->v2 = v2;
|
||||
+ state->v3 = v3;
|
||||
+ state->v4 = v4;
|
||||
+ }
|
||||
+
|
||||
+ if (p < b_end) {
|
||||
+ memcpy(state->mem32, p, (size_t)(b_end-p));
|
||||
+ state->memsize = (uint32_t)(b_end-p);
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_update);
|
||||
+
|
||||
+uint32_t xxh32_digest(const struct xxh32_state *state)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)state->mem32;
|
||||
+ const uint8_t *const b_end = (const uint8_t *)(state->mem32) +
|
||||
+ state->memsize;
|
||||
+ uint32_t h32;
|
||||
+
|
||||
+ if (state->large_len) {
|
||||
+ h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) +
|
||||
+ xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18);
|
||||
+ } else {
|
||||
+ h32 = state->v3 /* == seed */ + PRIME32_5;
|
||||
+ }
|
||||
+
|
||||
+ h32 += state->total_len_32;
|
||||
+
|
||||
+ while (p + 4 <= b_end) {
|
||||
+ h32 += get_unaligned_le32(p) * PRIME32_3;
|
||||
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h32 += (*p) * PRIME32_5;
|
||||
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h32 ^= h32 >> 15;
|
||||
+ h32 *= PRIME32_2;
|
||||
+ h32 ^= h32 >> 13;
|
||||
+ h32 *= PRIME32_3;
|
||||
+ h32 ^= h32 >> 16;
|
||||
+
|
||||
+ return h32;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_digest);
|
||||
+
|
||||
+int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+
|
||||
+ if (input == NULL)
|
||||
+ return -EINVAL;
|
||||
+
|
||||
+ state->total_len += len;
|
||||
+
|
||||
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
|
||||
+ memcpy(((uint8_t *)state->mem64) + state->memsize, input, len);
|
||||
+ state->memsize += (uint32_t)len;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ if (state->memsize) { /* tmp buffer is full */
|
||||
+ uint64_t *p64 = state->mem64;
|
||||
+
|
||||
+ memcpy(((uint8_t *)p64) + state->memsize, input,
|
||||
+ 32 - state->memsize);
|
||||
+
|
||||
+ state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64));
|
||||
+
|
||||
+ p += 32 - state->memsize;
|
||||
+ state->memsize = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 32 <= b_end) {
|
||||
+ const uint8_t *const limit = b_end - 32;
|
||||
+ uint64_t v1 = state->v1;
|
||||
+ uint64_t v2 = state->v2;
|
||||
+ uint64_t v3 = state->v3;
|
||||
+ uint64_t v4 = state->v4;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ state->v1 = v1;
|
||||
+ state->v2 = v2;
|
||||
+ state->v3 = v3;
|
||||
+ state->v4 = v4;
|
||||
+ }
|
||||
+
|
||||
+ if (p < b_end) {
|
||||
+ memcpy(state->mem64, p, (size_t)(b_end-p));
|
||||
+ state->memsize = (uint32_t)(b_end - p);
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_update);
|
||||
+
|
||||
+uint64_t xxh64_digest(const struct xxh64_state *state)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)state->mem64;
|
||||
+ const uint8_t *const b_end = (const uint8_t *)state->mem64 +
|
||||
+ state->memsize;
|
||||
+ uint64_t h64;
|
||||
+
|
||||
+ if (state->total_len >= 32) {
|
||||
+ const uint64_t v1 = state->v1;
|
||||
+ const uint64_t v2 = state->v2;
|
||||
+ const uint64_t v3 = state->v3;
|
||||
+ const uint64_t v4 = state->v4;
|
||||
+
|
||||
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
|
||||
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
|
||||
+ h64 = xxh64_merge_round(h64, v1);
|
||||
+ h64 = xxh64_merge_round(h64, v2);
|
||||
+ h64 = xxh64_merge_round(h64, v3);
|
||||
+ h64 = xxh64_merge_round(h64, v4);
|
||||
+ } else {
|
||||
+ h64 = state->v3 + PRIME64_5;
|
||||
+ }
|
||||
+
|
||||
+ h64 += (uint64_t)state->total_len;
|
||||
+
|
||||
+ while (p + 8 <= b_end) {
|
||||
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
|
||||
+
|
||||
+ h64 ^= k1;
|
||||
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
|
||||
+ p += 8;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 4 <= b_end) {
|
||||
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
|
||||
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h64 ^= (*p) * PRIME64_5;
|
||||
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h64 ^= h64 >> 33;
|
||||
+ h64 *= PRIME64_2;
|
||||
+ h64 ^= h64 >> 29;
|
||||
+ h64 *= PRIME64_3;
|
||||
+ h64 ^= h64 >> 32;
|
||||
+
|
||||
+ return h64;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_digest);
|
||||
+
|
||||
+MODULE_LICENSE("Dual BSD/GPL");
|
||||
+MODULE_DESCRIPTION("xxHash");
|
||||
--
|
||||
2.9.3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,740 +0,0 @@
|
||||
From 8a9dddfbf6551afea73911e367dd4be64d62b9fd Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Mon, 17 Jul 2017 17:08:39 -0700
|
||||
Subject: [PATCH v5 3/5] btrfs: Add zstd support
|
||||
|
||||
Add zstd compression and decompression support to BtrFS. zstd at its
|
||||
fastest level compresses almost as well as zlib, while offering much
|
||||
faster compression and decompression, approaching lzo speeds.
|
||||
|
||||
I benchmarked btrfs with zstd compression against no compression, lzo
|
||||
compression, and zlib compression. I benchmarked two scenarios. Copying
|
||||
a set of files to btrfs, and then reading the files. Copying a tarball
|
||||
to btrfs, extracting it to btrfs, and then reading the extracted files.
|
||||
After every operation, I call `sync` and include the sync time.
|
||||
Between every pair of operations I unmount and remount the filesystem
|
||||
to avoid caching. The benchmark files can be found in the upstream
|
||||
zstd source repository under
|
||||
`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
|
||||
[1] [2].
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD.
|
||||
|
||||
The first compression benchmark is copying 10 copies of the unzipped
|
||||
Silesia corpus [3] into a BtrFS filesystem mounted with
|
||||
`-o compress-force=Method`. The decompression benchmark times how long
|
||||
it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
|
||||
measured by comparing the output of `df` and `du`. See the benchmark file
|
||||
[1] for details. I benchmarked multiple zstd compression levels, although
|
||||
the patch uses zstd level 1.
|
||||
|
||||
| Method | Ratio | Compression MB/s | Decompression speed |
|
||||
|---------|-------|------------------|---------------------|
|
||||
| None | 0.99 | 504 | 686 |
|
||||
| lzo | 1.66 | 398 | 442 |
|
||||
| zlib | 2.58 | 65 | 241 |
|
||||
| zstd 1 | 2.57 | 260 | 383 |
|
||||
| zstd 3 | 2.71 | 174 | 408 |
|
||||
| zstd 6 | 2.87 | 70 | 398 |
|
||||
| zstd 9 | 2.92 | 43 | 406 |
|
||||
| zstd 12 | 2.93 | 21 | 408 |
|
||||
| zstd 15 | 3.01 | 11 | 354 |
|
||||
|
||||
The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
|
||||
measures the compression ratio, extracts the tar, and deletes the tar.
|
||||
Then it measures the compression ratio again, and `tar`s the extracted
|
||||
files into `/dev/null`. See the benchmark file [2] for details.
|
||||
|
||||
| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
|
||||
|--------|-----------|---------------|----------|------------|----------|
|
||||
| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 |
|
||||
| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 |
|
||||
| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 |
|
||||
| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 |
|
||||
|
||||
[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
|
||||
[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||
[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v2 -> v3:
|
||||
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff
|
||||
- Change default compression level for BtrFS to 3
|
||||
|
||||
v3 -> v4:
|
||||
- Add missing includes, which fixes the aarch64 build
|
||||
- Fix minor linter warnings
|
||||
|
||||
fs/btrfs/Kconfig | 2 +
|
||||
fs/btrfs/Makefile | 2 +-
|
||||
fs/btrfs/compression.c | 1 +
|
||||
fs/btrfs/compression.h | 6 +-
|
||||
fs/btrfs/ctree.h | 1 +
|
||||
fs/btrfs/disk-io.c | 2 +
|
||||
fs/btrfs/ioctl.c | 6 +-
|
||||
fs/btrfs/props.c | 6 +
|
||||
fs/btrfs/super.c | 12 +-
|
||||
fs/btrfs/sysfs.c | 2 +
|
||||
fs/btrfs/zstd.c | 432 +++++++++++++++++++++++++++++++++++++++++++++
|
||||
include/uapi/linux/btrfs.h | 8 +-
|
||||
12 files changed, 468 insertions(+), 12 deletions(-)
|
||||
create mode 100644 fs/btrfs/zstd.c
|
||||
|
||||
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
|
||||
index 80e9c18..a26c63b 100644
|
||||
--- a/fs/btrfs/Kconfig
|
||||
+++ b/fs/btrfs/Kconfig
|
||||
@@ -6,6 +6,8 @@ config BTRFS_FS
|
||||
select ZLIB_DEFLATE
|
||||
select LZO_COMPRESS
|
||||
select LZO_DECOMPRESS
|
||||
+ select ZSTD_COMPRESS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
select RAID6_PQ
|
||||
select XOR_BLOCKS
|
||||
select SRCU
|
||||
diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
|
||||
index 128ce17..962a95a 100644
|
||||
--- a/fs/btrfs/Makefile
|
||||
+++ b/fs/btrfs/Makefile
|
||||
@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
|
||||
transaction.o inode.o file.o tree-defrag.o \
|
||||
extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \
|
||||
extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
|
||||
- export.o tree-log.o free-space-cache.o zlib.o lzo.o \
|
||||
+ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \
|
||||
compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
|
||||
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
|
||||
uuid-tree.o props.o hash.o free-space-tree.o
|
||||
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
|
||||
index d2ef9ac..4ff42d1 100644
|
||||
--- a/fs/btrfs/compression.c
|
||||
+++ b/fs/btrfs/compression.c
|
||||
@@ -704,6 +704,7 @@ static struct {
|
||||
static const struct btrfs_compress_op * const btrfs_compress_op[] = {
|
||||
&btrfs_zlib_compress,
|
||||
&btrfs_lzo_compress,
|
||||
+ &btrfs_zstd_compress,
|
||||
};
|
||||
|
||||
void __init btrfs_init_compress(void)
|
||||
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
|
||||
index 87f6d33..2269e00 100644
|
||||
--- a/fs/btrfs/compression.h
|
||||
+++ b/fs/btrfs/compression.h
|
||||
@@ -99,8 +99,9 @@ enum btrfs_compression_type {
|
||||
BTRFS_COMPRESS_NONE = 0,
|
||||
BTRFS_COMPRESS_ZLIB = 1,
|
||||
BTRFS_COMPRESS_LZO = 2,
|
||||
- BTRFS_COMPRESS_TYPES = 2,
|
||||
- BTRFS_COMPRESS_LAST = 3,
|
||||
+ BTRFS_COMPRESS_ZSTD = 3,
|
||||
+ BTRFS_COMPRESS_TYPES = 3,
|
||||
+ BTRFS_COMPRESS_LAST = 4,
|
||||
};
|
||||
|
||||
struct btrfs_compress_op {
|
||||
@@ -128,5 +129,6 @@ struct btrfs_compress_op {
|
||||
|
||||
extern const struct btrfs_compress_op btrfs_zlib_compress;
|
||||
extern const struct btrfs_compress_op btrfs_lzo_compress;
|
||||
+extern const struct btrfs_compress_op btrfs_zstd_compress;
|
||||
|
||||
#endif
|
||||
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
|
||||
index 3f3eb7b..845d77c 100644
|
||||
--- a/fs/btrfs/ctree.h
|
||||
+++ b/fs/btrfs/ctree.h
|
||||
@@ -270,6 +270,7 @@ struct btrfs_super_block {
|
||||
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
|
||||
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
|
||||
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
|
||||
+ BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \
|
||||
BTRFS_FEATURE_INCOMPAT_RAID56 | \
|
||||
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
|
||||
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
|
||||
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
|
||||
index 080e2eb..04632f4 100644
|
||||
--- a/fs/btrfs/disk-io.c
|
||||
+++ b/fs/btrfs/disk-io.c
|
||||
@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
|
||||
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
|
||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
|
||||
+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
|
||||
+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
|
||||
|
||||
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
|
||||
btrfs_info(fs_info, "has skinny extents");
|
||||
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
|
||||
index fa1b78c..b9963d9 100644
|
||||
--- a/fs/btrfs/ioctl.c
|
||||
+++ b/fs/btrfs/ioctl.c
|
||||
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
|
||||
|
||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
comp = "lzo";
|
||||
- else
|
||||
+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZLIB)
|
||||
comp = "zlib";
|
||||
+ else
|
||||
+ comp = "zstd";
|
||||
ret = btrfs_set_prop(inode, "btrfs.compression",
|
||||
comp, strlen(comp), 0);
|
||||
if (ret)
|
||||
@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
|
||||
|
||||
if (range->compress_type == BTRFS_COMPRESS_LZO) {
|
||||
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
|
||||
+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
|
||||
+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
|
||||
}
|
||||
|
||||
ret = defrag_count;
|
||||
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
|
||||
index 4b23ae5..20631e9 100644
|
||||
--- a/fs/btrfs/props.c
|
||||
+++ b/fs/btrfs/props.c
|
||||
@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
|
||||
return 0;
|
||||
else if (!strncmp("zlib", value, len))
|
||||
return 0;
|
||||
+ else if (!strncmp("zstd", value, len))
|
||||
+ return 0;
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
|
||||
type = BTRFS_COMPRESS_LZO;
|
||||
else if (!strncmp("zlib", value, len))
|
||||
type = BTRFS_COMPRESS_ZLIB;
|
||||
+ else if (!strncmp("zstd", value, len))
|
||||
+ type = BTRFS_COMPRESS_ZSTD;
|
||||
else
|
||||
return -EINVAL;
|
||||
|
||||
@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
|
||||
return "zlib";
|
||||
case BTRFS_COMPRESS_LZO:
|
||||
return "lzo";
|
||||
+ case BTRFS_COMPRESS_ZSTD:
|
||||
+ return "zstd";
|
||||
}
|
||||
|
||||
return NULL;
|
||||
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
|
||||
index 12540b6..c370dea 100644
|
||||
--- a/fs/btrfs/super.c
|
||||
+++ b/fs/btrfs/super.c
|
||||
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
|
||||
btrfs_clear_opt(info->mount_opt, NODATASUM);
|
||||
btrfs_set_fs_incompat(info, COMPRESS_LZO);
|
||||
no_compress = 0;
|
||||
+ } else if (strcmp(args[0].from, "zstd") == 0) {
|
||||
+ compress_type = "zstd";
|
||||
+ info->compress_type = BTRFS_COMPRESS_ZSTD;
|
||||
+ btrfs_set_opt(info->mount_opt, COMPRESS);
|
||||
+ btrfs_clear_opt(info->mount_opt, NODATACOW);
|
||||
+ btrfs_clear_opt(info->mount_opt, NODATASUM);
|
||||
+ btrfs_set_fs_incompat(info, COMPRESS_ZSTD);
|
||||
+ no_compress = 0;
|
||||
} else if (strncmp(args[0].from, "no", 2) == 0) {
|
||||
compress_type = "no";
|
||||
btrfs_clear_opt(info->mount_opt, COMPRESS);
|
||||
@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
|
||||
if (btrfs_test_opt(info, COMPRESS)) {
|
||||
if (info->compress_type == BTRFS_COMPRESS_ZLIB)
|
||||
compress_type = "zlib";
|
||||
- else
|
||||
+ else if (info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
compress_type = "lzo";
|
||||
+ else
|
||||
+ compress_type = "zstd";
|
||||
if (btrfs_test_opt(info, FORCE_COMPRESS))
|
||||
seq_printf(seq, ",compress-force=%s", compress_type);
|
||||
else
|
||||
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
|
||||
index c2d5f35..2b6d37c 100644
|
||||
--- a/fs/btrfs/sysfs.c
|
||||
+++ b/fs/btrfs/sysfs.c
|
||||
@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
|
||||
+BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
|
||||
@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = {
|
||||
BTRFS_FEAT_ATTR_PTR(default_subvol),
|
||||
BTRFS_FEAT_ATTR_PTR(mixed_groups),
|
||||
BTRFS_FEAT_ATTR_PTR(compress_lzo),
|
||||
+ BTRFS_FEAT_ATTR_PTR(compress_zstd),
|
||||
BTRFS_FEAT_ATTR_PTR(big_metadata),
|
||||
BTRFS_FEAT_ATTR_PTR(extended_iref),
|
||||
BTRFS_FEAT_ATTR_PTR(raid56),
|
||||
diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
|
||||
new file mode 100644
|
||||
index 0000000..607ce47
|
||||
--- /dev/null
|
||||
+++ b/fs/btrfs/zstd.c
|
||||
@@ -0,0 +1,432 @@
|
||||
+/*
|
||||
+ * Copyright (c) 2016-present, Facebook, Inc.
|
||||
+ * All rights reserved.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public
|
||||
+ * License v2 as published by the Free Software Foundation.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ * General Public License for more details.
|
||||
+ */
|
||||
+#include <linux/bio.h>
|
||||
+#include <linux/err.h>
|
||||
+#include <linux/init.h>
|
||||
+#include <linux/kernel.h>
|
||||
+#include <linux/mm.h>
|
||||
+#include <linux/pagemap.h>
|
||||
+#include <linux/refcount.h>
|
||||
+#include <linux/sched.h>
|
||||
+#include <linux/slab.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include "compression.h"
|
||||
+
|
||||
+#define ZSTD_BTRFS_MAX_WINDOWLOG 17
|
||||
+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
|
||||
+#define ZSTD_BTRFS_DEFAULT_LEVEL 3
|
||||
+
|
||||
+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
|
||||
+{
|
||||
+ ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
|
||||
+ src_len, 0);
|
||||
+
|
||||
+ if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
|
||||
+ params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
|
||||
+ WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
|
||||
+ return params;
|
||||
+}
|
||||
+
|
||||
+struct workspace {
|
||||
+ void *mem;
|
||||
+ size_t size;
|
||||
+ char *buf;
|
||||
+ struct list_head list;
|
||||
+};
|
||||
+
|
||||
+static void zstd_free_workspace(struct list_head *ws)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+
|
||||
+ kvfree(workspace->mem);
|
||||
+ kfree(workspace->buf);
|
||||
+ kfree(workspace);
|
||||
+}
|
||||
+
|
||||
+static struct list_head *zstd_alloc_workspace(void)
|
||||
+{
|
||||
+ ZSTD_parameters params =
|
||||
+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
|
||||
+ struct workspace *workspace;
|
||||
+
|
||||
+ workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
|
||||
+ if (!workspace)
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+
|
||||
+ workspace->size = max_t(size_t,
|
||||
+ ZSTD_CStreamWorkspaceBound(params.cParams),
|
||||
+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
|
||||
+ workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
|
||||
+ workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
|
||||
+ if (!workspace->mem || !workspace->buf)
|
||||
+ goto fail;
|
||||
+
|
||||
+ INIT_LIST_HEAD(&workspace->list);
|
||||
+
|
||||
+ return &workspace->list;
|
||||
+fail:
|
||||
+ zstd_free_workspace(&workspace->list);
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress_pages(struct list_head *ws,
|
||||
+ struct address_space *mapping,
|
||||
+ u64 start,
|
||||
+ struct page **pages,
|
||||
+ unsigned long *out_pages,
|
||||
+ unsigned long *total_in,
|
||||
+ unsigned long *total_out)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ ZSTD_CStream *stream;
|
||||
+ int ret = 0;
|
||||
+ int nr_pages = 0;
|
||||
+ struct page *in_page = NULL; /* The current page to read */
|
||||
+ struct page *out_page = NULL; /* The current page to write to */
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+ unsigned long tot_in = 0;
|
||||
+ unsigned long tot_out = 0;
|
||||
+ unsigned long len = *total_out;
|
||||
+ const unsigned long nr_dest_pages = *out_pages;
|
||||
+ unsigned long max_out = nr_dest_pages * PAGE_SIZE;
|
||||
+ ZSTD_parameters params = zstd_get_btrfs_parameters(len);
|
||||
+
|
||||
+ *out_pages = 0;
|
||||
+ *total_out = 0;
|
||||
+ *total_in = 0;
|
||||
+
|
||||
+ /* Initialize the stream */
|
||||
+ stream = ZSTD_initCStream(params, len, workspace->mem,
|
||||
+ workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_warn("BTRFS: ZSTD_initCStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* map in the first page of input data */
|
||||
+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
|
||||
+ in_buf.src = kmap(in_page);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
|
||||
+
|
||||
+
|
||||
+ /* Allocate and map in the output buffer */
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_compressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* Check to see if we are making it bigger */
|
||||
+ if (tot_in + in_buf.pos > 8192 &&
|
||||
+ tot_in + in_buf.pos <
|
||||
+ tot_out + out_buf.pos) {
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* We've reached the end of our output range */
|
||||
+ if (out_buf.pos >= max_out) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* Check if we need more output space */
|
||||
+ if (out_buf.pos == out_buf.size) {
|
||||
+ tot_out += PAGE_SIZE;
|
||||
+ max_out -= PAGE_SIZE;
|
||||
+ kunmap(out_page);
|
||||
+ if (nr_pages == nr_dest_pages) {
|
||||
+ out_page = NULL;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+ }
|
||||
+
|
||||
+ /* We've reached the end of the input */
|
||||
+ if (in_buf.pos >= len) {
|
||||
+ tot_in += in_buf.pos;
|
||||
+ break;
|
||||
+ }
|
||||
+
|
||||
+ /* Check if we need more input */
|
||||
+ if (in_buf.pos == in_buf.size) {
|
||||
+ tot_in += PAGE_SIZE;
|
||||
+ kunmap(in_page);
|
||||
+ put_page(in_page);
|
||||
+
|
||||
+ start += PAGE_SIZE;
|
||||
+ len -= PAGE_SIZE;
|
||||
+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
|
||||
+ in_buf.src = kmap(in_page);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
|
||||
+ }
|
||||
+ }
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_endStream(stream, &out_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_endStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ if (ret2 == 0) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ break;
|
||||
+ }
|
||||
+ if (out_buf.pos >= max_out) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ tot_out += PAGE_SIZE;
|
||||
+ max_out -= PAGE_SIZE;
|
||||
+ kunmap(out_page);
|
||||
+ if (nr_pages == nr_dest_pages) {
|
||||
+ out_page = NULL;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+ }
|
||||
+
|
||||
+ if (tot_out >= tot_in) {
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ret = 0;
|
||||
+ *total_in = tot_in;
|
||||
+ *total_out = tot_out;
|
||||
+out:
|
||||
+ *out_pages = nr_pages;
|
||||
+ /* Cleanup */
|
||||
+ if (in_page) {
|
||||
+ kunmap(in_page);
|
||||
+ put_page(in_page);
|
||||
+ }
|
||||
+ if (out_page)
|
||||
+ kunmap(out_page);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ struct page **pages_in = cb->compressed_pages;
|
||||
+ u64 disk_start = cb->start;
|
||||
+ struct bio *orig_bio = cb->orig_bio;
|
||||
+ size_t srclen = cb->compressed_len;
|
||||
+ ZSTD_DStream *stream;
|
||||
+ int ret = 0;
|
||||
+ unsigned long page_in_index = 0;
|
||||
+ unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
|
||||
+ unsigned long buf_start;
|
||||
+ unsigned long total_out = 0;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+
|
||||
+ stream = ZSTD_initDStream(
|
||||
+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_debug("BTRFS: ZSTD_initDStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+
|
||||
+ in_buf.src = kmap(pages_in[page_in_index]);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
|
||||
+
|
||||
+ out_buf.dst = workspace->buf;
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+ buf_start = total_out;
|
||||
+ total_out += out_buf.pos;
|
||||
+ out_buf.pos = 0;
|
||||
+
|
||||
+ ret = btrfs_decompress_buf2page(out_buf.dst, buf_start,
|
||||
+ total_out, disk_start, orig_bio);
|
||||
+ if (ret == 0)
|
||||
+ break;
|
||||
+
|
||||
+ if (in_buf.pos >= srclen)
|
||||
+ break;
|
||||
+
|
||||
+ /* Check if we've hit the end of a frame */
|
||||
+ if (ret2 == 0)
|
||||
+ break;
|
||||
+
|
||||
+ if (in_buf.pos == in_buf.size) {
|
||||
+ kunmap(pages_in[page_in_index++]);
|
||||
+ if (page_in_index >= total_pages_in) {
|
||||
+ in_buf.src = NULL;
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+ srclen -= PAGE_SIZE;
|
||||
+ in_buf.src = kmap(pages_in[page_in_index]);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
|
||||
+ }
|
||||
+ }
|
||||
+ ret = 0;
|
||||
+ zero_fill_bio(orig_bio);
|
||||
+done:
|
||||
+ if (in_buf.src)
|
||||
+ kunmap(pages_in[page_in_index]);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
|
||||
+ struct page *dest_page,
|
||||
+ unsigned long start_byte,
|
||||
+ size_t srclen, size_t destlen)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ ZSTD_DStream *stream;
|
||||
+ int ret = 0;
|
||||
+ size_t ret2;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+ unsigned long total_out = 0;
|
||||
+ unsigned long pg_offset = 0;
|
||||
+ char *kaddr;
|
||||
+
|
||||
+ stream = ZSTD_initDStream(
|
||||
+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_warn("BTRFS: ZSTD_initDStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+
|
||||
+ destlen = min_t(size_t, destlen, PAGE_SIZE);
|
||||
+
|
||||
+ in_buf.src = data_in;
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = srclen;
|
||||
+
|
||||
+ out_buf.dst = workspace->buf;
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+
|
||||
+ ret2 = 1;
|
||||
+ while (pg_offset < destlen && in_buf.pos < in_buf.size) {
|
||||
+ unsigned long buf_start;
|
||||
+ unsigned long buf_offset;
|
||||
+ unsigned long bytes;
|
||||
+
|
||||
+ /* Check if the frame is over and we still need more input */
|
||||
+ if (ret2 == 0) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+
|
||||
+ buf_start = total_out;
|
||||
+ total_out += out_buf.pos;
|
||||
+ out_buf.pos = 0;
|
||||
+
|
||||
+ if (total_out <= start_byte)
|
||||
+ continue;
|
||||
+
|
||||
+ if (total_out > start_byte && buf_start < start_byte)
|
||||
+ buf_offset = start_byte - buf_start;
|
||||
+ else
|
||||
+ buf_offset = 0;
|
||||
+
|
||||
+ bytes = min_t(unsigned long, destlen - pg_offset,
|
||||
+ out_buf.size - buf_offset);
|
||||
+
|
||||
+ kaddr = kmap_atomic(dest_page);
|
||||
+ memcpy(kaddr + pg_offset, out_buf.dst + buf_offset, bytes);
|
||||
+ kunmap_atomic(kaddr);
|
||||
+
|
||||
+ pg_offset += bytes;
|
||||
+ }
|
||||
+ ret = 0;
|
||||
+finish:
|
||||
+ if (pg_offset < destlen) {
|
||||
+ kaddr = kmap_atomic(dest_page);
|
||||
+ memset(kaddr + pg_offset, 0, destlen - pg_offset);
|
||||
+ kunmap_atomic(kaddr);
|
||||
+ }
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+const struct btrfs_compress_op btrfs_zstd_compress = {
|
||||
+ .alloc_workspace = zstd_alloc_workspace,
|
||||
+ .free_workspace = zstd_free_workspace,
|
||||
+ .compress_pages = zstd_compress_pages,
|
||||
+ .decompress_bio = zstd_decompress_bio,
|
||||
+ .decompress = zstd_decompress,
|
||||
+};
|
||||
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
|
||||
index 9aa74f3..378230c 100644
|
||||
--- a/include/uapi/linux/btrfs.h
|
||||
+++ b/include/uapi/linux/btrfs.h
|
||||
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
|
||||
#define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1)
|
||||
#define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2)
|
||||
#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3)
|
||||
-/*
|
||||
- * some patches floated around with a second compression method
|
||||
- * lets save that incompat here for when they do get in
|
||||
- * Note we don't actually support it, we're just reserving the
|
||||
- * number
|
||||
- */
|
||||
-#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
|
||||
+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
|
||||
|
||||
/*
|
||||
* older kernels tried to do bigger metadata blocks, but the
|
||||
--
|
||||
2.9.3
|
||||
@@ -1,306 +0,0 @@
|
||||
From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001
|
||||
From: Sean Purcell <me@seanp.xyz>
|
||||
Date: Mon, 17 Jul 2017 17:08:59 -0700
|
||||
Subject: [PATCH v5 4/5] squashfs: Add zstd support
|
||||
|
||||
Add zstd compression and decompression support to SquashFS. zstd is a
|
||||
great fit for SquashFS because it can compress at ratios approaching xz,
|
||||
while decompressing twice as fast as zlib. For SquashFS in particular,
|
||||
it can decompress as fast as lzo and lz4. It also has the flexibility
|
||||
to turn down the compression ratio for faster compression times.
|
||||
|
||||
The compression benchmark is run on the file tree from the SquashFS archive
|
||||
found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
|
||||
default block size (128 KB) and and various compression algorithms/levels.
|
||||
xz and zstd are also benchmarked with 256 KB blocks. The decompression
|
||||
benchmark times how long it takes to `tar` the file tree into `/dev/null`.
|
||||
See the benchmark file in the upstream zstd source repository located under
|
||||
`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD.
|
||||
|
||||
| Method | Ratio | Compression MB/s | Decompression MB/s |
|
||||
|----------------|-------|------------------|--------------------|
|
||||
| gzip | 2.92 | 15 | 128 |
|
||||
| lzo | 2.64 | 9.5 | 217 |
|
||||
| lz4 | 2.12 | 94 | 218 |
|
||||
| xz | 3.43 | 5.5 | 35 |
|
||||
| xz 256 KB | 3.53 | 5.4 | 40 |
|
||||
| zstd 1 | 2.71 | 96 | 210 |
|
||||
| zstd 5 | 2.93 | 69 | 198 |
|
||||
| zstd 10 | 3.01 | 41 | 225 |
|
||||
| zstd 15 | 3.13 | 11.4 | 224 |
|
||||
| zstd 16 256 KB | 3.24 | 8.1 | 210 |
|
||||
|
||||
This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
|
||||
taking over the submission process.
|
||||
|
||||
[1] http://releases.ubuntu.com/16.10/
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
|
||||
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v3 -> v4:
|
||||
- Fix minor linter warnings
|
||||
|
||||
v4 -> v5:
|
||||
- Fix ZSTD_DStream initialization code in squashfs
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
|
||||
fs/squashfs/Kconfig | 14 +++++
|
||||
fs/squashfs/Makefile | 1 +
|
||||
fs/squashfs/decompressor.c | 7 +++
|
||||
fs/squashfs/decompressor.h | 4 ++
|
||||
fs/squashfs/squashfs_fs.h | 1 +
|
||||
fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++
|
||||
6 files changed, 178 insertions(+)
|
||||
create mode 100644 fs/squashfs/zstd_wrapper.c
|
||||
|
||||
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
|
||||
index ffb093e..1adb334 100644
|
||||
--- a/fs/squashfs/Kconfig
|
||||
+++ b/fs/squashfs/Kconfig
|
||||
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
|
||||
|
||||
If unsure, say N.
|
||||
|
||||
+config SQUASHFS_ZSTD
|
||||
+ bool "Include support for ZSTD compressed file systems"
|
||||
+ depends on SQUASHFS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
+ help
|
||||
+ Saying Y here includes support for reading Squashfs file systems
|
||||
+ compressed with ZSTD compression. ZSTD gives better compression than
|
||||
+ the default ZLIB compression, while using less CPU.
|
||||
+
|
||||
+ ZSTD is not the standard compression used in Squashfs and so most
|
||||
+ file systems will be readable without selecting this option.
|
||||
+
|
||||
+ If unsure, say N.
|
||||
+
|
||||
config SQUASHFS_4K_DEVBLK_SIZE
|
||||
bool "Use 4K device block size?"
|
||||
depends on SQUASHFS
|
||||
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
|
||||
index 246a6f3..6655631 100644
|
||||
--- a/fs/squashfs/Makefile
|
||||
+++ b/fs/squashfs/Makefile
|
||||
@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o
|
||||
+squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o
|
||||
diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
|
||||
index d2bc136..8366398 100644
|
||||
--- a/fs/squashfs/decompressor.c
|
||||
+++ b/fs/squashfs/decompressor.c
|
||||
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
|
||||
};
|
||||
#endif
|
||||
|
||||
+#ifndef CONFIG_SQUASHFS_ZSTD
|
||||
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
||||
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
|
||||
+};
|
||||
+#endif
|
||||
+
|
||||
static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
|
||||
NULL, NULL, NULL, NULL, 0, "unknown", 0
|
||||
};
|
||||
@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = {
|
||||
&squashfs_lzo_comp_ops,
|
||||
&squashfs_xz_comp_ops,
|
||||
&squashfs_lzma_unsupported_comp_ops,
|
||||
+ &squashfs_zstd_comp_ops,
|
||||
&squashfs_unknown_comp_ops
|
||||
};
|
||||
|
||||
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
|
||||
index a25713c..0f5a8e4 100644
|
||||
--- a/fs/squashfs/decompressor.h
|
||||
+++ b/fs/squashfs/decompressor.h
|
||||
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
|
||||
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
|
||||
#endif
|
||||
|
||||
+#ifdef CONFIG_SQUASHFS_ZSTD
|
||||
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
|
||||
+#endif
|
||||
+
|
||||
#endif
|
||||
diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
|
||||
index 506f4ba..24d12fd 100644
|
||||
--- a/fs/squashfs/squashfs_fs.h
|
||||
+++ b/fs/squashfs/squashfs_fs.h
|
||||
@@ -241,6 +241,7 @@ struct meta_index {
|
||||
#define LZO_COMPRESSION 3
|
||||
#define XZ_COMPRESSION 4
|
||||
#define LZ4_COMPRESSION 5
|
||||
+#define ZSTD_COMPRESSION 6
|
||||
|
||||
struct squashfs_super_block {
|
||||
__le32 s_magic;
|
||||
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
|
||||
new file mode 100644
|
||||
index 0000000..eeaabf8
|
||||
--- /dev/null
|
||||
+++ b/fs/squashfs/zstd_wrapper.c
|
||||
@@ -0,0 +1,151 @@
|
||||
+/*
|
||||
+ * Squashfs - a compressed read only filesystem for Linux
|
||||
+ *
|
||||
+ * Copyright (c) 2016-present, Facebook, Inc.
|
||||
+ * All rights reserved.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.c
|
||||
+ */
|
||||
+
|
||||
+#include <linux/mutex.h>
|
||||
+#include <linux/buffer_head.h>
|
||||
+#include <linux/slab.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include <linux/vmalloc.h>
|
||||
+
|
||||
+#include "squashfs_fs.h"
|
||||
+#include "squashfs_fs_sb.h"
|
||||
+#include "squashfs.h"
|
||||
+#include "decompressor.h"
|
||||
+#include "page_actor.h"
|
||||
+
|
||||
+struct workspace {
|
||||
+ void *mem;
|
||||
+ size_t mem_size;
|
||||
+ size_t window_size;
|
||||
+};
|
||||
+
|
||||
+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
|
||||
+{
|
||||
+ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL);
|
||||
+
|
||||
+ if (wksp == NULL)
|
||||
+ goto failed;
|
||||
+ wksp->window_size = max_t(size_t,
|
||||
+ msblk->block_size, SQUASHFS_METADATA_SIZE);
|
||||
+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
|
||||
+ wksp->mem = vmalloc(wksp->mem_size);
|
||||
+ if (wksp->mem == NULL)
|
||||
+ goto failed;
|
||||
+
|
||||
+ return wksp;
|
||||
+
|
||||
+failed:
|
||||
+ ERROR("Failed to allocate zstd workspace\n");
|
||||
+ kfree(wksp);
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+}
|
||||
+
|
||||
+
|
||||
+static void zstd_free(void *strm)
|
||||
+{
|
||||
+ struct workspace *wksp = strm;
|
||||
+
|
||||
+ if (wksp)
|
||||
+ vfree(wksp->mem);
|
||||
+ kfree(wksp);
|
||||
+}
|
||||
+
|
||||
+
|
||||
+static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,
|
||||
+ struct buffer_head **bh, int b, int offset, int length,
|
||||
+ struct squashfs_page_actor *output)
|
||||
+{
|
||||
+ struct workspace *wksp = strm;
|
||||
+ ZSTD_DStream *stream;
|
||||
+ size_t total_out = 0;
|
||||
+ size_t zstd_err;
|
||||
+ int k = 0;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+
|
||||
+ stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
|
||||
+
|
||||
+ if (!stream) {
|
||||
+ ERROR("Failed to initialize zstd decompressor\n");
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+ out_buf.dst = squashfs_first_page(output);
|
||||
+
|
||||
+ do {
|
||||
+ if (in_buf.pos == in_buf.size && k < b) {
|
||||
+ int avail = min(length, msblk->devblksize - offset);
|
||||
+
|
||||
+ length -= avail;
|
||||
+ in_buf.src = bh[k]->b_data + offset;
|
||||
+ in_buf.size = avail;
|
||||
+ in_buf.pos = 0;
|
||||
+ offset = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (out_buf.pos == out_buf.size) {
|
||||
+ out_buf.dst = squashfs_next_page(output);
|
||||
+ if (out_buf.dst == NULL) {
|
||||
+ /* Shouldn't run out of pages
|
||||
+ * before stream is done.
|
||||
+ */
|
||||
+ squashfs_finish_page(output);
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+ }
|
||||
+
|
||||
+ total_out -= out_buf.pos;
|
||||
+ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ total_out += out_buf.pos; /* add the additional data produced */
|
||||
+
|
||||
+ if (in_buf.pos == in_buf.size && k < b)
|
||||
+ put_bh(bh[k++]);
|
||||
+ } while (zstd_err != 0 && !ZSTD_isError(zstd_err));
|
||||
+
|
||||
+ squashfs_finish_page(output);
|
||||
+
|
||||
+ if (ZSTD_isError(zstd_err)) {
|
||||
+ ERROR("zstd decompression error: %d\n",
|
||||
+ (int)ZSTD_getErrorCode(zstd_err));
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ if (k < b)
|
||||
+ goto out;
|
||||
+
|
||||
+ return (int)total_out;
|
||||
+
|
||||
+out:
|
||||
+ for (; k < b; k++)
|
||||
+ put_bh(bh[k]);
|
||||
+
|
||||
+ return -EIO;
|
||||
+}
|
||||
+
|
||||
+const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
||||
+ .init = zstd_init,
|
||||
+ .free = zstd_free,
|
||||
+ .decompress = zstd_uncompress,
|
||||
+ .id = ZSTD_COMPRESSION,
|
||||
+ .name = "zstd",
|
||||
+ .supported = 1
|
||||
+};
|
||||
--
|
||||
2.9.3
|
||||
@@ -1,424 +0,0 @@
|
||||
From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Wed, 2 Aug 2017 18:02:13 -0700
|
||||
Subject: [PATCH v5 5/5] crypto: Add zstd support
|
||||
|
||||
Adds zstd support to crypto and scompress. Only supports the default
|
||||
level.
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
crypto/Kconfig | 9 ++
|
||||
crypto/Makefile | 1 +
|
||||
crypto/testmgr.c | 10 +++
|
||||
crypto/testmgr.h | 71 +++++++++++++++
|
||||
crypto/zstd.c | 265 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
5 files changed, 356 insertions(+)
|
||||
create mode 100644 crypto/zstd.c
|
||||
|
||||
diff --git a/crypto/Kconfig b/crypto/Kconfig
|
||||
index caa770e..4fc3936 100644
|
||||
--- a/crypto/Kconfig
|
||||
+++ b/crypto/Kconfig
|
||||
@@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC
|
||||
help
|
||||
This is the LZ4 high compression mode algorithm.
|
||||
|
||||
+config CRYPTO_ZSTD
|
||||
+ tristate "Zstd compression algorithm"
|
||||
+ select CRYPTO_ALGAPI
|
||||
+ select CRYPTO_ACOMP2
|
||||
+ select ZSTD_COMPRESS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
+ help
|
||||
+ This is the zstd algorithm.
|
||||
+
|
||||
comment "Random Number Generation"
|
||||
|
||||
config CRYPTO_ANSI_CPRNG
|
||||
diff --git a/crypto/Makefile b/crypto/Makefile
|
||||
index d41f033..b22e1e8 100644
|
||||
--- a/crypto/Makefile
|
||||
+++ b/crypto/Makefile
|
||||
@@ -133,6 +133,7 @@ obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
|
||||
+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
|
||||
|
||||
ecdh_generic-y := ecc.o
|
||||
ecdh_generic-y += ecdh.o
|
||||
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
|
||||
index 7125ba3..8a124d3 100644
|
||||
--- a/crypto/testmgr.c
|
||||
+++ b/crypto/testmgr.c
|
||||
@@ -3603,6 +3603,16 @@ static const struct alg_test_desc alg_test_descs[] = {
|
||||
.decomp = __VECS(zlib_deflate_decomp_tv_template)
|
||||
}
|
||||
}
|
||||
+ }, {
|
||||
+ .alg = "zstd",
|
||||
+ .test = alg_test_comp,
|
||||
+ .fips_allowed = 1,
|
||||
+ .suite = {
|
||||
+ .comp = {
|
||||
+ .comp = __VECS(zstd_comp_tv_template),
|
||||
+ .decomp = __VECS(zstd_decomp_tv_template)
|
||||
+ }
|
||||
+ }
|
||||
}
|
||||
};
|
||||
|
||||
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
|
||||
index 6ceb0e2..e6b5920 100644
|
||||
--- a/crypto/testmgr.h
|
||||
+++ b/crypto/testmgr.h
|
||||
@@ -34631,4 +34631,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = {
|
||||
},
|
||||
};
|
||||
|
||||
+static const struct comp_testvec zstd_comp_tv_template[] = {
|
||||
+ {
|
||||
+ .inlen = 68,
|
||||
+ .outlen = 39,
|
||||
+ .input = "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd.",
|
||||
+ .output = "\x28\xb5\x2f\xfd\x00\x50\xf5\x00\x00\xb8\x54\x68\x65"
|
||||
+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
|
||||
+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
|
||||
+ ,
|
||||
+ },
|
||||
+ {
|
||||
+ .inlen = 244,
|
||||
+ .outlen = 151,
|
||||
+ .input = "zstd, short for Zstandard, is a fast lossless "
|
||||
+ "compression algorithm, targeting real-time "
|
||||
+ "compression scenarios at zlib-level and better "
|
||||
+ "compression ratios. The zstd compression library "
|
||||
+ "provides in-memory compression and decompression "
|
||||
+ "functions.",
|
||||
+ .output = "\x28\xb5\x2f\xfd\x00\x50\x75\x04\x00\x42\x4b\x1e\x17"
|
||||
+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
|
||||
+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
|
||||
+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
|
||||
+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
|
||||
+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
|
||||
+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
|
||||
+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
|
||||
+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
|
||||
+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
|
||||
+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
|
||||
+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01",
|
||||
+ },
|
||||
+};
|
||||
+
|
||||
+static const struct comp_testvec zstd_decomp_tv_template[] = {
|
||||
+ {
|
||||
+ .inlen = 43,
|
||||
+ .outlen = 68,
|
||||
+ .input = "\x28\xb5\x2f\xfd\x04\x50\xf5\x00\x00\xb8\x54\x68\x65"
|
||||
+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
|
||||
+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
|
||||
+ "\x6b\xf4\x13\x35",
|
||||
+ .output = "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd.",
|
||||
+ },
|
||||
+ {
|
||||
+ .inlen = 155,
|
||||
+ .outlen = 244,
|
||||
+ .input = "\x28\xb5\x2f\xfd\x04\x50\x75\x04\x00\x42\x4b\x1e\x17"
|
||||
+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
|
||||
+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
|
||||
+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
|
||||
+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
|
||||
+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
|
||||
+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
|
||||
+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
|
||||
+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
|
||||
+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
|
||||
+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
|
||||
+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01\xaa\x6d\xda\x0d",
|
||||
+ .output = "zstd, short for Zstandard, is a fast lossless "
|
||||
+ "compression algorithm, targeting real-time "
|
||||
+ "compression scenarios at zlib-level and better "
|
||||
+ "compression ratios. The zstd compression library "
|
||||
+ "provides in-memory compression and decompression "
|
||||
+ "functions.",
|
||||
+ },
|
||||
+};
|
||||
#endif /* _CRYPTO_TESTMGR_H */
|
||||
diff --git a/crypto/zstd.c b/crypto/zstd.c
|
||||
new file mode 100644
|
||||
index 0000000..9a76b3e
|
||||
--- /dev/null
|
||||
+++ b/crypto/zstd.c
|
||||
@@ -0,0 +1,265 @@
|
||||
+/*
|
||||
+ * Cryptographic API.
|
||||
+ *
|
||||
+ * Copyright (c) 2017-present, Facebook, Inc.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it
|
||||
+ * under the terms of the GNU General Public License version 2 as published by
|
||||
+ * the Free Software Foundation.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
+ * more details.
|
||||
+ */
|
||||
+#include <linux/crypto.h>
|
||||
+#include <linux/init.h>
|
||||
+#include <linux/interrupt.h>
|
||||
+#include <linux/mm.h>
|
||||
+#include <linux/module.h>
|
||||
+#include <linux/net.h>
|
||||
+#include <linux/vmalloc.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include <crypto/internal/scompress.h>
|
||||
+
|
||||
+
|
||||
+#define ZSTD_DEF_LEVEL 3
|
||||
+
|
||||
+struct zstd_ctx {
|
||||
+ ZSTD_CCtx *cctx;
|
||||
+ ZSTD_DCtx *dctx;
|
||||
+ void *cwksp;
|
||||
+ void *dwksp;
|
||||
+};
|
||||
+
|
||||
+static ZSTD_parameters zstd_params(void)
|
||||
+{
|
||||
+ return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0);
|
||||
+}
|
||||
+
|
||||
+static int zstd_comp_init(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ int ret = 0;
|
||||
+ const ZSTD_parameters params = zstd_params();
|
||||
+ const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams);
|
||||
+
|
||||
+ ctx->cwksp = vzalloc(wksp_size);
|
||||
+ if (!ctx->cwksp) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size);
|
||||
+ if (!ctx->cctx) {
|
||||
+ ret = -EINVAL;
|
||||
+ goto out_free;
|
||||
+ }
|
||||
+out:
|
||||
+ return ret;
|
||||
+out_free:
|
||||
+ vfree(ctx->cwksp);
|
||||
+ goto out;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decomp_init(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ int ret = 0;
|
||||
+ const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
|
||||
+
|
||||
+ ctx->dwksp = vzalloc(wksp_size);
|
||||
+ if (!ctx->dwksp) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size);
|
||||
+ if (!ctx->dctx) {
|
||||
+ ret = -EINVAL;
|
||||
+ goto out_free;
|
||||
+ }
|
||||
+out:
|
||||
+ return ret;
|
||||
+out_free:
|
||||
+ vfree(ctx->dwksp);
|
||||
+ goto out;
|
||||
+}
|
||||
+
|
||||
+static void zstd_comp_exit(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ vfree(ctx->cwksp);
|
||||
+ ctx->cwksp = NULL;
|
||||
+ ctx->cctx = NULL;
|
||||
+}
|
||||
+
|
||||
+static void zstd_decomp_exit(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ vfree(ctx->dwksp);
|
||||
+ ctx->dwksp = NULL;
|
||||
+ ctx->dctx = NULL;
|
||||
+}
|
||||
+
|
||||
+static int __zstd_init(void *ctx)
|
||||
+{
|
||||
+ int ret;
|
||||
+
|
||||
+ ret = zstd_comp_init(ctx);
|
||||
+ if (ret)
|
||||
+ return ret;
|
||||
+ ret = zstd_decomp_init(ctx);
|
||||
+ if (ret)
|
||||
+ zstd_comp_exit(ctx);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static void *zstd_alloc_ctx(struct crypto_scomp *tfm)
|
||||
+{
|
||||
+ int ret;
|
||||
+ struct zstd_ctx *ctx;
|
||||
+
|
||||
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
|
||||
+ if (!ctx)
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+
|
||||
+ ret = __zstd_init(ctx);
|
||||
+ if (ret) {
|
||||
+ kfree(ctx);
|
||||
+ return ERR_PTR(ret);
|
||||
+ }
|
||||
+
|
||||
+ return ctx;
|
||||
+}
|
||||
+
|
||||
+static int zstd_init(struct crypto_tfm *tfm)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_init(ctx);
|
||||
+}
|
||||
+
|
||||
+static void __zstd_exit(void *ctx)
|
||||
+{
|
||||
+ zstd_comp_exit(ctx);
|
||||
+ zstd_decomp_exit(ctx);
|
||||
+}
|
||||
+
|
||||
+static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx)
|
||||
+{
|
||||
+ __zstd_exit(ctx);
|
||||
+ kzfree(ctx);
|
||||
+}
|
||||
+
|
||||
+static void zstd_exit(struct crypto_tfm *tfm)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ __zstd_exit(ctx);
|
||||
+}
|
||||
+
|
||||
+static int __zstd_compress(const u8 *src, unsigned int slen,
|
||||
+ u8 *dst, unsigned int *dlen, void *ctx)
|
||||
+{
|
||||
+ size_t out_len;
|
||||
+ struct zstd_ctx *zctx = ctx;
|
||||
+ const ZSTD_parameters params = zstd_params();
|
||||
+
|
||||
+ out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params);
|
||||
+ if (ZSTD_isError(out_len))
|
||||
+ return -EINVAL;
|
||||
+ *dlen = out_len;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress(struct crypto_tfm *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_compress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen,
|
||||
+ void *ctx)
|
||||
+{
|
||||
+ return __zstd_compress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int __zstd_decompress(const u8 *src, unsigned int slen,
|
||||
+ u8 *dst, unsigned int *dlen, void *ctx)
|
||||
+{
|
||||
+ size_t out_len;
|
||||
+ struct zstd_ctx *zctx = ctx;
|
||||
+
|
||||
+ out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen);
|
||||
+ if (ZSTD_isError(out_len))
|
||||
+ return -EINVAL;
|
||||
+ *dlen = out_len;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_decompress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen,
|
||||
+ void *ctx)
|
||||
+{
|
||||
+ return __zstd_decompress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static struct crypto_alg alg = {
|
||||
+ .cra_name = "zstd",
|
||||
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
|
||||
+ .cra_ctxsize = sizeof(struct zstd_ctx),
|
||||
+ .cra_module = THIS_MODULE,
|
||||
+ .cra_init = zstd_init,
|
||||
+ .cra_exit = zstd_exit,
|
||||
+ .cra_u = { .compress = {
|
||||
+ .coa_compress = zstd_compress,
|
||||
+ .coa_decompress = zstd_decompress } }
|
||||
+};
|
||||
+
|
||||
+static struct scomp_alg scomp = {
|
||||
+ .alloc_ctx = zstd_alloc_ctx,
|
||||
+ .free_ctx = zstd_free_ctx,
|
||||
+ .compress = zstd_scompress,
|
||||
+ .decompress = zstd_sdecompress,
|
||||
+ .base = {
|
||||
+ .cra_name = "zstd",
|
||||
+ .cra_driver_name = "zstd-scomp",
|
||||
+ .cra_module = THIS_MODULE,
|
||||
+ }
|
||||
+};
|
||||
+
|
||||
+static int __init zstd_mod_init(void)
|
||||
+{
|
||||
+ int ret;
|
||||
+
|
||||
+ ret = crypto_register_alg(&alg);
|
||||
+ if (ret)
|
||||
+ return ret;
|
||||
+
|
||||
+ ret = crypto_register_scomp(&scomp);
|
||||
+ if (ret)
|
||||
+ crypto_unregister_alg(&alg);
|
||||
+
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static void __exit zstd_mod_fini(void)
|
||||
+{
|
||||
+ crypto_unregister_alg(&alg);
|
||||
+ crypto_unregister_scomp(&scomp);
|
||||
+}
|
||||
+
|
||||
+module_init(zstd_mod_init);
|
||||
+module_exit(zstd_mod_fini);
|
||||
+
|
||||
+MODULE_LICENSE("GPL");
|
||||
+MODULE_DESCRIPTION("Zstd Compression Algorithm");
|
||||
+MODULE_ALIAS_CRYPTO("zstd");
|
||||
--
|
||||
2.9.3
|
||||
@@ -1,420 +0,0 @@
|
||||
From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001
|
||||
From: Sean Purcell <me@seanp.xyz>
|
||||
Date: Thu, 3 Aug 2017 17:47:03 -0700
|
||||
Subject: [PATCH v6] squashfs-tools: Add zstd support
|
||||
|
||||
This patch adds zstd support to squashfs-tools. It works with zstd
|
||||
versions >= 1.0.0. It was originally written by Sean Purcell.
|
||||
|
||||
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v4 -> v5:
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
- Don't strip trailing whitespace of unrelated code
|
||||
- Make zstd_display_options() static
|
||||
|
||||
v5 -> v6:
|
||||
- Fix build instructions in Makefile
|
||||
|
||||
squashfs-tools/Makefile | 20 ++++
|
||||
squashfs-tools/compressor.c | 8 ++
|
||||
squashfs-tools/squashfs_fs.h | 1 +
|
||||
squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++
|
||||
squashfs-tools/zstd_wrapper.h | 48 ++++++++
|
||||
5 files changed, 331 insertions(+)
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.c
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.h
|
||||
|
||||
diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile
|
||||
index 52d2582..22fc559 100644
|
||||
--- a/squashfs-tools/Makefile
|
||||
+++ b/squashfs-tools/Makefile
|
||||
@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1
|
||||
#LZMA_SUPPORT = 1
|
||||
#LZMA_DIR = ../../../../LZMA/lzma465
|
||||
|
||||
+
|
||||
+########### Building ZSTD support ############
|
||||
+#
|
||||
+# The ZSTD library is supported
|
||||
+# ZSTD homepage: http://zstd.net
|
||||
+# ZSTD source repository: https://github.com/facebook/zstd
|
||||
+#
|
||||
+# To build using the ZSTD library - install the library and uncomment the
|
||||
+# ZSTD_SUPPORT line below.
|
||||
+#
|
||||
+#ZSTD_SUPPORT = 1
|
||||
+
|
||||
######## Specifying default compression ########
|
||||
#
|
||||
# The next line specifies which compression algorithm is used by default
|
||||
@@ -177,6 +189,14 @@ LIBS += -llz4
|
||||
COMPRESSORS += lz4
|
||||
endif
|
||||
|
||||
+ifeq ($(ZSTD_SUPPORT),1)
|
||||
+CFLAGS += -DZSTD_SUPPORT
|
||||
+MKSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+UNSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+LIBS += -lzstd
|
||||
+COMPRESSORS += zstd
|
||||
+endif
|
||||
+
|
||||
ifeq ($(XATTR_SUPPORT),1)
|
||||
ifeq ($(XATTR_DEFAULT),1)
|
||||
CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
|
||||
diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c
|
||||
index 525e316..02b5e90 100644
|
||||
--- a/squashfs-tools/compressor.c
|
||||
+++ b/squashfs-tools/compressor.c
|
||||
@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
|
||||
extern struct compressor xz_comp_ops;
|
||||
#endif
|
||||
|
||||
+#ifndef ZSTD_SUPPORT
|
||||
+static struct compressor zstd_comp_ops = {
|
||||
+ ZSTD_COMPRESSION, "zstd"
|
||||
+};
|
||||
+#else
|
||||
+extern struct compressor zstd_comp_ops;
|
||||
+#endif
|
||||
|
||||
static struct compressor unknown_comp_ops = {
|
||||
0, "unknown"
|
||||
@@ -77,6 +84,7 @@ struct compressor *compressor[] = {
|
||||
&lzo_comp_ops,
|
||||
&lz4_comp_ops,
|
||||
&xz_comp_ops,
|
||||
+ &zstd_comp_ops,
|
||||
&unknown_comp_ops
|
||||
};
|
||||
|
||||
diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
|
||||
index 791fe12..afca918 100644
|
||||
--- a/squashfs-tools/squashfs_fs.h
|
||||
+++ b/squashfs-tools/squashfs_fs.h
|
||||
@@ -277,6 +277,7 @@ typedef long long squashfs_inode;
|
||||
#define LZO_COMPRESSION 3
|
||||
#define XZ_COMPRESSION 4
|
||||
#define LZ4_COMPRESSION 5
|
||||
+#define ZSTD_COMPRESSION 6
|
||||
|
||||
struct squashfs_super_block {
|
||||
unsigned int s_magic;
|
||||
diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
|
||||
new file mode 100644
|
||||
index 0000000..dcab75a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.c
|
||||
@@ -0,0 +1,254 @@
|
||||
+/*
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.c
|
||||
+ *
|
||||
+ * Support for ZSTD compression http://zstd.net
|
||||
+ */
|
||||
+
|
||||
+#include <stdio.h>
|
||||
+#include <string.h>
|
||||
+#include <stdlib.h>
|
||||
+#include <zstd.h>
|
||||
+#include <zstd_errors.h>
|
||||
+
|
||||
+#include "squashfs_fs.h"
|
||||
+#include "zstd_wrapper.h"
|
||||
+#include "compressor.h"
|
||||
+
|
||||
+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by the options parsing code in mksquashfs.c
|
||||
+ * to parse any -X compressor option.
|
||||
+ *
|
||||
+ * This function returns:
|
||||
+ * >=0 (number of additional args parsed) on success
|
||||
+ * -1 if the option was unrecognised, or
|
||||
+ * -2 if the option was recognised, but otherwise bad in
|
||||
+ * some way (e.g. invalid parameter)
|
||||
+ *
|
||||
+ * Note: this function sets internal compressor state, but does not
|
||||
+ * pass back the results of the parsing other than success/failure.
|
||||
+ * The zstd_dump_options() function is called later to get the options in
|
||||
+ * a format suitable for writing to the filesystem.
|
||||
+ */
|
||||
+static int zstd_options(char *argv[], int argc)
|
||||
+{
|
||||
+ if (strcmp(argv[0], "-Xcompression-level") == 0) {
|
||||
+ if (argc < 2) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level missing "
|
||||
+ "compression level\n");
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level it should "
|
||||
+ "be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = atoi(argv[1]);
|
||||
+ if (compression_level < 1 ||
|
||||
+ compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level invalid, it "
|
||||
+ "should be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ return 1;
|
||||
+ }
|
||||
+
|
||||
+ return -1;
|
||||
+failed:
|
||||
+ return -2;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to dump the parsed
|
||||
+ * compressor options in a format suitable for writing to the
|
||||
+ * compressor options field in the filesystem (stored immediately
|
||||
+ * after the superblock).
|
||||
+ *
|
||||
+ * This function returns a pointer to the compression options structure
|
||||
+ * to be stored (and the size), or NULL if there are no compression
|
||||
+ * options.
|
||||
+ */
|
||||
+static void *zstd_dump_options(int block_size, int *size)
|
||||
+{
|
||||
+ static struct zstd_comp_opts comp_opts;
|
||||
+
|
||||
+ /* don't return anything if the options are all default */
|
||||
+ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
|
||||
+ return NULL;
|
||||
+
|
||||
+ comp_opts.compression_level = compression_level;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
|
||||
+
|
||||
+ *size = sizeof(comp_opts);
|
||||
+ return &comp_opts;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is a helper specifically for the append mode of
|
||||
+ * mksquashfs. Its purpose is to set the internal compressor state
|
||||
+ * to the stored compressor options in the passed compressor options
|
||||
+ * structure.
|
||||
+ *
|
||||
+ * In effect this function sets up the compressor options
|
||||
+ * to the same state they were when the filesystem was originally
|
||||
+ * generated, this is to ensure on appending, the compressor uses
|
||||
+ * the same compression options that were used to generate the
|
||||
+ * original filesystem.
|
||||
+ *
|
||||
+ * Note, even if there are no compressor options, this function is still
|
||||
+ * called with an empty compressor structure (size == 0), to explicitly
|
||||
+ * set the default options, this is to ensure any user supplied
|
||||
+ * -X options on the appending mksquashfs command line are over-ridden.
|
||||
+ *
|
||||
+ * This function returns 0 on successful extraction of options, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_extract_options(int block_size, void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ if (size == 0) {
|
||||
+ /* Set default values */
|
||||
+ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = comp_opts->compression_level;
|
||||
+
|
||||
+ return 0;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+
|
||||
+ return -1;
|
||||
+}
|
||||
+
|
||||
+static void zstd_display_options(void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ printf("\tcompression-level %d\n", comp_opts->compression_level);
|
||||
+
|
||||
+ return;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to initialise the
|
||||
+ * compressor, before compress() is called.
|
||||
+ *
|
||||
+ * This function returns 0 on success, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_init(void **strm, int block_size, int datablock)
|
||||
+{
|
||||
+ ZSTD_CCtx *cctx = ZSTD_createCCtx();
|
||||
+
|
||||
+ if (!cctx) {
|
||||
+ fprintf(stderr, "zstd: failed to allocate compression "
|
||||
+ "context!\n");
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ *strm = cctx;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress(void *strm, void *dest, void *src, int size,
|
||||
+ int block_size, int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
|
||||
+ src, size, compression_level);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ /* FIXME:
|
||||
+ * zstd does not expose stable error codes. The error enum may
|
||||
+ * change between versions. Until upstream zstd stablizes the
|
||||
+ * error codes, we have no way of knowing why the error occurs.
|
||||
+ * zstd shouldn't fail to compress any input unless there isn't
|
||||
+ * enough output space. We assume that is the cause and return
|
||||
+ * the special error code for not enough output space.
|
||||
+ */
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static int zstd_uncompress(void *dest, void *src, int size, int outsize,
|
||||
+ int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_decompress(dest, outsize, src, size);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ fprintf(stderr, "\t%d %d\n", outsize, size);
|
||||
+
|
||||
+ *error = (int)ZSTD_getErrorCode(res);
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static void zstd_usage(void)
|
||||
+{
|
||||
+ fprintf(stderr, "\t -Xcompression-level <compression-level>\n");
|
||||
+ fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "
|
||||
+ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);
|
||||
+}
|
||||
+
|
||||
+struct compressor zstd_comp_ops = {
|
||||
+ .init = zstd_init,
|
||||
+ .compress = zstd_compress,
|
||||
+ .uncompress = zstd_uncompress,
|
||||
+ .options = zstd_options,
|
||||
+ .dump_options = zstd_dump_options,
|
||||
+ .extract_options = zstd_extract_options,
|
||||
+ .display_options = zstd_display_options,
|
||||
+ .usage = zstd_usage,
|
||||
+ .id = ZSTD_COMPRESSION,
|
||||
+ .name = "zstd",
|
||||
+ .supported = 1
|
||||
+};
|
||||
diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h
|
||||
new file mode 100644
|
||||
index 0000000..4fbef0a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.h
|
||||
@@ -0,0 +1,48 @@
|
||||
+#ifndef ZSTD_WRAPPER_H
|
||||
+#define ZSTD_WRAPPER_H
|
||||
+/*
|
||||
+ * Squashfs
|
||||
+ *
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.h
|
||||
+ *
|
||||
+ */
|
||||
+
|
||||
+#ifndef linux
|
||||
+#define __BYTE_ORDER BYTE_ORDER
|
||||
+#define __BIG_ENDIAN BIG_ENDIAN
|
||||
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
||||
+#else
|
||||
+#include <endian.h>
|
||||
+#endif
|
||||
+
|
||||
+#if __BYTE_ORDER == __BIG_ENDIAN
|
||||
+extern unsigned int inswap_le16(unsigned short);
|
||||
+extern unsigned int inswap_le32(unsigned int);
|
||||
+
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
|
||||
+ (s)->compression_level = inswap_le32((s)->compression_level); \
|
||||
+}
|
||||
+#else
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s)
|
||||
+#endif
|
||||
+
|
||||
+/* Default compression */
|
||||
+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
|
||||
+
|
||||
+struct zstd_comp_opts {
|
||||
+ int compression_level;
|
||||
+};
|
||||
+#endif
|
||||
--
|
||||
2.9.5
|
||||
@@ -1,339 +0,0 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
@@ -1,101 +0,0 @@
|
||||
# Linux Kernel Patch
|
||||
|
||||
There are four pieces, the `xxhash` kernel module, the `zstd_compress` and `zstd_decompress` kernel modules, the BtrFS patch, and the SquashFS patch.
|
||||
The patches are based off of the linux kernel master branch.
|
||||
|
||||
## xxHash kernel module
|
||||
|
||||
* The patch is located in `xxhash.diff`.
|
||||
* The header is in `include/linux/xxhash.h`.
|
||||
* The source is in `lib/xxhash.c`.
|
||||
* `test/XXHashUserLandTest.cpp` contains tests for the patch in userland by mocking the kernel headers.
|
||||
I tested the tests by commenting a line of of each branch in `xxhash.c` one line at a time, and made sure the tests failed.
|
||||
It can be run with the following commands:
|
||||
```
|
||||
cd test && make googletest && make XXHashUserLandTest && ./XXHashUserLandTest
|
||||
```
|
||||
* I also benchmarked the `xxhash` module against upstream xxHash, and made sure that they ran at the same speed.
|
||||
|
||||
## Zstd Kernel modules
|
||||
|
||||
* The (large) patch is located in `zstd.diff`, which depends on `xxhash.diff`.
|
||||
* The header is in `include/linux/zstd.h`.
|
||||
* It is split up into `zstd_compress` and `zstd_decompress`, which can be loaded independently.
|
||||
* Source files are in `lib/zstd/`.
|
||||
* `lib/Kconfig` and `lib/Makefile` need to be modified by applying `lib/Kconfig.diff` and `lib/Makefile.diff` respectively.
|
||||
These changes are also included in the `zstd.diff`.
|
||||
* `test/UserlandTest.cpp` contains tests for the patch in userland by mocking the kernel headers.
|
||||
It can be run with the following commands:
|
||||
```
|
||||
cd test && make googletest && make UserlandTest && ./UserlandTest
|
||||
```
|
||||
|
||||
## BtrFS
|
||||
|
||||
* The patch is located in `btrfs.diff`.
|
||||
* Additionally `fs/btrfs/zstd.c` is provided as a source for convenience.
|
||||
* The patch seems to be working, it doesn't crash the kernel, and compresses at speeds and ratios that are expected.
|
||||
It could still use some more testing for fringe features, like printing options.
|
||||
|
||||
### Benchmarks
|
||||
|
||||
Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of ram, and a SSD.
|
||||
The kernel running was built from the master branch with the patch.
|
||||
|
||||
The compression benchmark is copying 10 copies of the
|
||||
unzipped [silesia corpus](http://mattmahoney.net/dc/silesia.html) into a BtrFS
|
||||
filesystem mounted with `-o compress-force={none, lzo, zlib, zstd}`.
|
||||
The decompression benchmark is timing how long it takes to `tar` all 10 copies
|
||||
into `/dev/null`.
|
||||
The compression ratio is measured by comparing the output of `df` and `du`.
|
||||
See `btrfs-benchmark.sh` for details.
|
||||
|
||||
| Algorithm | Compression ratio | Compression speed | Decompression speed |
|
||||
|-----------|-------------------|-------------------|---------------------|
|
||||
| None | 0.99 | 504 MB/s | 686 MB/s |
|
||||
| lzo | 1.66 | 398 MB/s | 442 MB/s |
|
||||
| zlib | 2.58 | 65 MB/s | 241 MB/s |
|
||||
| zstd 1 | 2.57 | 260 MB/s | 383 MB/s |
|
||||
| zstd 3 | 2.71 | 174 MB/s | 408 MB/s |
|
||||
| zstd 6 | 2.87 | 70 MB/s | 398 MB/s |
|
||||
| zstd 9 | 2.92 | 43 MB/s | 406 MB/s |
|
||||
| zstd 12 | 2.93 | 21 MB/s | 408 MB/s |
|
||||
| zstd 15 | 3.01 | 11 MB/s | 354 MB/s |
|
||||
|
||||
|
||||
## SquashFS
|
||||
|
||||
* The patch is located in `squashfs.diff`
|
||||
* Additionally `fs/squashfs/zstd_wrapper.c` is provided as a source for convenience.
|
||||
* The patch has been tested on the master branch of the kernel.
|
||||
|
||||
### Benchmarks
|
||||
|
||||
Benchmarks run on a Ubuntu 14.04 with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of ram, and a SSD.
|
||||
The kernel running was built from the master branch with the patch.
|
||||
|
||||
The compression benchmark is the file tree from the SquashFS archive found in the
|
||||
Ubuntu 16.10 desktop image (ubuntu-16.10-desktop-amd64.iso).
|
||||
The compression benchmark uses mksquashfs with the default block size (128 KB)
|
||||
and various compression algorithms/compression levels.
|
||||
`xz` and `zstd` are also benchmarked with 256 KB blocks.
|
||||
The decompression benchmark is timing how long it takes to `tar` the file tree
|
||||
into `/dev/null`.
|
||||
See `squashfs-benchmark.sh` for details.
|
||||
|
||||
| Algorithm | Compression ratio | Compression speed | Decompression speed |
|
||||
|----------------|-------------------|-------------------|---------------------|
|
||||
| gzip | 2.92 | 15 MB/s | 128 MB/s |
|
||||
| lzo | 2.64 | 9.5 MB/s | 217 MB/s |
|
||||
| lz4 | 2.12 | 94 MB/s | 218 MB/s |
|
||||
| xz | 3.43 | 5.5 MB/s | 35 MB/s |
|
||||
| xz 256 KB | 3.53 | 5.4 MB/s | 40 MB/s |
|
||||
| zstd 1 | 2.71 | 96 MB/s | 210 MB/s |
|
||||
| zstd 5 | 2.93 | 69 MB/s | 198 MB/s |
|
||||
| zstd 10 | 3.01 | 41 MB/s | 225 MB/s |
|
||||
| zstd 15 | 3.13 | 11.4 MB/s | 224 MB/s |
|
||||
| zstd 16 256 KB | 3.24 | 8.1 MB/s | 210 MB/s |
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user