Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yann Collet
|
f4340f46b2 | ||
|
Yann Collet
|
d29bf941b9 | ||
|
Yann Collet
|
969e6ef44f | ||
|
Yann Collet
|
2b2f3c6f64 | ||
|
Yann Collet
|
b0a3ad96d3 | ||
|
Yann Collet
|
aecf3b479c | ||
|
Yann Collet
|
b72808a49b | ||
|
Yann Collet
|
fe007ba8bd | ||
|
Yann Collet
|
7d30795931 | ||
|
Yann Collet
|
da0c157fc4 | ||
|
Yann Collet
|
39d0c6cde7 | ||
|
Sean Purcell
|
02c44a7257 | ||
|
Yann Collet
|
585bb51413 | ||
|
Sean Purcell
|
bb8a7601bc | ||
|
Yann Collet
|
ebf275912a | ||
|
Yann Collet
|
f2ea8932fc | ||
|
Yann Collet
|
2c2ee2b4b0 | ||
|
Sean Purcell
|
c4f2490050 | ||
|
Yann Collet
|
cbc5225d38 | ||
|
Yann Collet
|
817995bb7a | ||
|
Yann Collet
|
99b6d9f1a3 | ||
|
Yann Collet
|
b9edd2f9fb |
@@ -1,169 +0,0 @@
|
||||
version: 2
|
||||
|
||||
jobs:
|
||||
# the first half of the jobs are in this test
|
||||
short-tests-0:
|
||||
# TODO: Create a small custom docker image with all the dependencies we need
|
||||
# preinstalled to reduce installation time.
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
./tests/test-license.py
|
||||
cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
|
||||
make c99build ; make clean
|
||||
make c11build ; make clean
|
||||
make aarch64build ; make clean
|
||||
make -j regressiontest; make clean
|
||||
make shortest ; make clean
|
||||
make cxxtest ; make clean
|
||||
# the second half of the jobs are in this test
|
||||
short-tests-1:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
make gnu90build; make clean
|
||||
make gnu99build; make clean
|
||||
make ppc64build; make clean
|
||||
make ppcbuild ; make clean
|
||||
make armbuild ; make clean
|
||||
make -C tests test-legacy test-longmatch; make clean
|
||||
make -C lib libzstd-nomt; make clean
|
||||
# This step is only run on release tags.
|
||||
# It publishes the source tarball as artifacts and if the GITHUB_TOKEN
|
||||
# environment variable is set it will publish the source tarball to the
|
||||
# tagged release.
|
||||
publish-github-release:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
environment:
|
||||
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Publish
|
||||
command: |
|
||||
export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
|
||||
export ZSTD_VERSION=zstd-$VERSION
|
||||
git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
|
||||
-o $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar > $ZSTD_VERSION.tar.sha256
|
||||
zstd -19 $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
|
||||
gzip -k -9 $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
|
||||
mkdir -p $CIRCLE_ARTIFACTS
|
||||
cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS
|
||||
- store_artifacts:
|
||||
path: /tmp/circleci-artifacts
|
||||
# This step should only be run in a cron job
|
||||
regression-test:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
environment:
|
||||
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||
steps:
|
||||
- checkout
|
||||
# Restore the cached resources.
|
||||
- restore_cache:
|
||||
# We try our best to bust the cache when the data changes by hashing
|
||||
# data.c. If that doesn't work, simply update the version number here
|
||||
# and below. If we fail to bust the cache, the regression testing will
|
||||
# still work, since it has its own stamp, but will need to redownload
|
||||
# everything.
|
||||
keys:
|
||||
- regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
- run:
|
||||
name: Regression Test
|
||||
command: |
|
||||
make -C programs zstd
|
||||
make -C tests/regression test
|
||||
mkdir -p $CIRCLE_ARTIFACTS
|
||||
./tests/regression/test \
|
||||
--cache tests/regression/cache \
|
||||
--output $CIRCLE_ARTIFACTS/results.csv \
|
||||
--zstd programs/zstd
|
||||
echo "NOTE: The new results.csv is uploaded as an artifact to this job"
|
||||
echo " If this fails, go to the Artifacts pane in CircleCI, "
|
||||
echo " download /tmp/circleci-artifacts/results.csv, and if they "
|
||||
echo " are still good, copy it into the repo and commit it."
|
||||
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
|
||||
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
|
||||
# Only save the cache on success (default), since if the failure happened
|
||||
# before we stamp the data cache, we will have a bad cache for this key.
|
||||
- save_cache:
|
||||
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
paths:
|
||||
- tests/regression/cache
|
||||
- store_artifacts:
|
||||
path: /tmp/circleci-artifacts
|
||||
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
commit:
|
||||
jobs:
|
||||
# Run the tests in parallel
|
||||
- short-tests-0:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
- short-tests-1:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
# Create a branch called regression and set it to dev to force a
|
||||
# regression test run
|
||||
- regression-test:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- regression
|
||||
# Only run on release tags.
|
||||
- publish-github-release:
|
||||
requires:
|
||||
- short-tests-0
|
||||
- short-tests-1
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
nightly:
|
||||
triggers:
|
||||
- schedule:
|
||||
cron: "0 0 * * *"
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- dev
|
||||
jobs:
|
||||
# Run daily long regression tests
|
||||
- regression-test
|
||||
|
||||
|
||||
|
||||
# Longer tests
|
||||
#- make -C tests test-zstd-nolegacy && make clean
|
||||
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
||||
#- make zlibwrapper && make clean
|
||||
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
||||
#- make uasan && make clean
|
||||
#- make asan32 && make clean
|
||||
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
||||
# Valgrind tests
|
||||
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
||||
#- make -C tests valgrindTest && make clean
|
||||
# ARM, AArch64, PowerPC, PowerPC64 tests
|
||||
#- make ppctest && make clean
|
||||
#- make ppc64test && make clean
|
||||
#- make armtest && make clean
|
||||
#- make aarch64test && make clean
|
||||
@@ -1,9 +0,0 @@
|
||||
FROM circleci/buildpack-deps:bionic
|
||||
|
||||
RUN sudo dpkg --add-architecture i386
|
||||
RUN sudo apt-get -y -qq update
|
||||
RUN sudo apt-get -y install \
|
||||
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
|
||||
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
|
||||
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
|
||||
libcurl4-openssl-dev
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
task:
|
||||
name: FreeBSD (shortest)
|
||||
freebsd_instance:
|
||||
matrix:
|
||||
image_family: freebsd-12-1
|
||||
# The stable 11.3 image causes "Agent is not responding" so use a snapshot
|
||||
image_family: freebsd-11-3-snap
|
||||
install_script: pkg install -y gmake coreutils
|
||||
script: |
|
||||
MOREFLAGS="-Werror" gmake -j all
|
||||
gmake shortest
|
||||
@@ -19,3 +19,6 @@
|
||||
# Windows
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
# .travis.yml merging
|
||||
.travis.yml merge=ours
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Downloads data '...'
|
||||
2. Run '...' with flags '...'
|
||||
3. Scroll up on the log to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots and charts**
|
||||
If applicable, add screenshots and charts to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. Mac]
|
||||
- Version [e.g. 22]
|
||||
- Compiler [e.g gcc]
|
||||
- Flags [e.g O2]
|
||||
- Other relevant hardware specs [e.g. Dual-core]
|
||||
- Build system [e.g. Makefile]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -1,184 +0,0 @@
|
||||
name: generic-dev
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ dev, master, actionsTest ]
|
||||
|
||||
jobs:
|
||||
|
||||
# Dev PR jobs that still have to be migrated from travis
|
||||
#
|
||||
# icc (need self-hosted)
|
||||
# versionTag
|
||||
# valgrindTest (keeps failing for some reason. need investigation)
|
||||
# staticAnalyze (need trusty so need self-hosted)
|
||||
# pcc-fuzz: (need trusty so need self-hosted)
|
||||
# min-decomp-macros (flakey)
|
||||
#
|
||||
# setting up self-hosted is pretty straightforward, but
|
||||
# I need admins permissions to the repo for that it looks like
|
||||
# So I'm tabling that for now
|
||||
#
|
||||
# The master branch exclusive jobs will be in a separate
|
||||
# workflow file (the osx tests and meson build that is)
|
||||
|
||||
benchmarking:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make benchmarking
|
||||
run: make benchmarking
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make test
|
||||
run: make test
|
||||
|
||||
gcc-6-7-libzstd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + gcc-7 + libzstdmt compilation
|
||||
run: |
|
||||
make gcc6install gcc7install
|
||||
CC=gcc-6 CFLAGS=-Werror make -j all
|
||||
make clean
|
||||
CC=gcc-7 CFLAGS=-Werror make -j all
|
||||
make clean
|
||||
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||
make -C tests zbufftest-dll
|
||||
|
||||
gcc-8-asan-ubsan-testzstd:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Test Zstd
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 CFLAGS="-Werror" make -j all
|
||||
make clean
|
||||
CC=gcc-8 make -j uasan-test-zstd </dev/null
|
||||
|
||||
gcc-6-asan-ubsan-testzstd-32bit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
|
||||
run: |
|
||||
make gcc6install libc6install
|
||||
CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
|
||||
make clean
|
||||
CC=gcc-6 make -j uasan-test-zstd32
|
||||
|
||||
clang-38-msan-testzstd:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: clang-3.8 + MSan + Test Zstd
|
||||
run: |
|
||||
# make clang38install (doesn't work)
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||
|
||||
cmake-build-and-test-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: cmake build and test check
|
||||
run: make cmakebuild
|
||||
|
||||
gcc-8-asan-ubsan-fuzz:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Fuzz Test
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 make clean uasan-fuzztest
|
||||
|
||||
gcc-6-asan-ubsan-fuzz32:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit
|
||||
run: |
|
||||
make gcc6install libc6install
|
||||
CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest
|
||||
|
||||
clang-38-msan-fuzz:
|
||||
runs-on: ubuntu-16.04 # fails on 18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: clang-3.8 + MSan + Fuzz Test
|
||||
run: |
|
||||
# make clang38install (doesn't work)
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make clean msan-fuzztest
|
||||
|
||||
asan-ubsan-msan-regression:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ASan + UBSan + MSan + Regression Test
|
||||
run: |
|
||||
make -j uasanregressiontest
|
||||
make clean
|
||||
make -j msanregressiontest
|
||||
|
||||
cpp-gnu90-c99-compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: C++, gnu90 and c99 compatibility
|
||||
run: |
|
||||
make cxxtest
|
||||
make clean
|
||||
make gnu90build
|
||||
make clean
|
||||
make c99build
|
||||
make clean
|
||||
make travis-install
|
||||
|
||||
mingw-cross-compilation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: mingw cross-compilation
|
||||
run: |
|
||||
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
|
||||
sudo apt-get install gcc-mingw-w64
|
||||
CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
||||
|
||||
armbuild:
|
||||
runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ARM Build Test
|
||||
run: |
|
||||
make arminstall
|
||||
make armbuild
|
||||
|
||||
armfuzz:
|
||||
runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Qemu ARM emulation + Fuzz Test
|
||||
run: |
|
||||
make arminstall
|
||||
make armfuzz
|
||||
|
||||
bourne-shell:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Bourne shell compatibility (shellcheck)
|
||||
run: |
|
||||
wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
|
||||
@@ -1,68 +0,0 @@
|
||||
name: generic-release
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# This will eventually only be for pushes to master
|
||||
# but for dogfooding purposes, I'm running it even
|
||||
# on dev pushes
|
||||
branches: [ dev, master, actionsTest ]
|
||||
|
||||
jobs:
|
||||
# missing jobs
|
||||
#
|
||||
# ppc64le + fuzz test
|
||||
# Qemu PPC64 + Fuzz test
|
||||
# Qemu aarch64 + Fuzz Test (on Xenial)
|
||||
# versions comp
|
||||
# meson test
|
||||
|
||||
osx:
|
||||
runs-on: macos-10.15
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: OS-X
|
||||
run: |
|
||||
make test
|
||||
# make -c lib all (need to fix. not working right now)
|
||||
|
||||
zbuff:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zbuff test
|
||||
run: |
|
||||
make -C tests test-zbuff
|
||||
|
||||
tsan:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: thread sanitizer
|
||||
run: |
|
||||
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang-3.8
|
||||
CC=clang-3.8 make tsan-test-zstream
|
||||
CC=clang-3.8 make tsan-fuzztest
|
||||
zlib-wrapper:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zlib wrapper test
|
||||
run: |
|
||||
make gpp6install valgrindinstall
|
||||
make -C zlibWrapper test
|
||||
make -C zlibWrapper valgrindTest
|
||||
|
||||
lz4-threadpool-partial-libs:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: LZ4, thread pool, and partial libs testslib wrapper test
|
||||
run: |
|
||||
make lz4install
|
||||
make -C tests test-lz4
|
||||
make clean
|
||||
make -C tests test-pool
|
||||
make clean
|
||||
bash tests/libzstd_partial_builds.sh
|
||||
@@ -1,30 +0,0 @@
|
||||
name: CIFuzz
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
Fuzzing:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
sanitizer: [address, undefined, memory]
|
||||
steps:
|
||||
- name: Build Fuzzers (${{ matrix.sanitizer }})
|
||||
id: build
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Run Fuzzers (${{ matrix.sanitizer }})
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
fuzz-seconds: 600
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Upload Crash
|
||||
uses: actions/upload-artifact@v1
|
||||
if: failure() && steps.build.outcome == 'success'
|
||||
with:
|
||||
name: ${{ matrix.sanitizer }}-artifacts
|
||||
path: ./out/artifacts
|
||||
+2
-11
@@ -14,7 +14,7 @@
|
||||
*.dylib
|
||||
|
||||
# Executables
|
||||
/zstd
|
||||
zstd
|
||||
zstdmt
|
||||
*.exe
|
||||
*.out
|
||||
@@ -22,18 +22,13 @@ zstdmt
|
||||
|
||||
# Test artefacts
|
||||
tmp*
|
||||
*.zst
|
||||
dictionary.
|
||||
dictionary
|
||||
NUL
|
||||
dictionary*
|
||||
|
||||
# Build artefacts
|
||||
projects/
|
||||
bin/
|
||||
.buckd/
|
||||
buck-out/
|
||||
build-*
|
||||
*.gcda
|
||||
|
||||
# Other files
|
||||
.directory
|
||||
@@ -45,7 +40,3 @@ _zstdbench/
|
||||
.DS_Store
|
||||
googletest/
|
||||
*.d
|
||||
*.vscode
|
||||
*.code-workspace
|
||||
compile_commands.json
|
||||
.clangd
|
||||
|
||||
+48
-269
@@ -1,6 +1,42 @@
|
||||
# Medium Tests: Run on all commits/PRs to dev branch
|
||||
# Long tests: run on commits to master branch/cron builds
|
||||
|
||||
language: c
|
||||
sudo: required
|
||||
dist: trusty
|
||||
matrix:
|
||||
include:
|
||||
# Ubuntu 14.04
|
||||
- env: Cmd="make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install"
|
||||
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test'
|
||||
- env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
||||
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
|
||||
|
||||
- env: Cmd='make arminstall && make armtest'
|
||||
- env: Cmd='make arminstall && make aarch64test'
|
||||
- env: Cmd='make ppcinstall && make ppctest'
|
||||
- env: Cmd='make ppcinstall && make ppc64test'
|
||||
|
||||
- env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy'
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff'
|
||||
|
||||
- env: Cmd='make -j uasanregressiontest'
|
||||
- env: Cmd='make -j msanregressiontest'
|
||||
|
||||
- env: Cmd='make -C tests versionsTest && make lz4install && make clean && make -C tests test-lz4'
|
||||
|
||||
# OS X Mavericks
|
||||
- env: Cmd="make test"
|
||||
os: osx
|
||||
|
||||
before_install:
|
||||
- if [ `uname` = "Darwin" ]; then brew update; fi
|
||||
|
||||
install:
|
||||
- if [ `uname` = "Darwin" ]; then brew install xz; fi
|
||||
|
||||
git:
|
||||
depth: 1
|
||||
@@ -9,272 +45,15 @@ branches:
|
||||
only:
|
||||
- dev
|
||||
- master
|
||||
- travisTest
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
|
||||
env:
|
||||
global:
|
||||
- FUZZERTEST=-T2mn
|
||||
ZSTREAM_TESTTIME=-T2mn
|
||||
DECODECORPUS_TESTTIME=-T1mn
|
||||
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- name: icc
|
||||
compiler: icc
|
||||
env:
|
||||
- C_COMPILER=icc
|
||||
- CXX_COMPILER=icpc
|
||||
install:
|
||||
- source /opt/intel/inteloneapi/compiler/latest/env/vars.sh
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- sourceline: 'deb https://apt.repos.intel.com/oneapi all main'
|
||||
key_url: 'https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB'
|
||||
packages:
|
||||
- intel-oneapi-icc
|
||||
script: make check
|
||||
|
||||
- name: arm64 # ~2.5 mn
|
||||
os: linux
|
||||
arch: arm64
|
||||
script:
|
||||
- make check
|
||||
|
||||
- name: make benchmarking
|
||||
script:
|
||||
- make benchmarking
|
||||
|
||||
- name: make test (complete)
|
||||
script:
|
||||
# DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
|
||||
- DEVNULLRIGHTS=test make test
|
||||
|
||||
- name: gcc-6 + gcc-7 + libzstdmt compilation # ~ 6mn
|
||||
script:
|
||||
- make gcc6install gcc7install
|
||||
- CC=gcc-6 CFLAGS=-Werror make -j all
|
||||
- make clean
|
||||
- CC=gcc-7 CFLAGS=-Werror make -j all
|
||||
- make clean
|
||||
- LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||
- make -C tests zbufftest-dll
|
||||
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
|
||||
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
|
||||
|
||||
- name: gcc-8 + ASan + UBSan + Test Zstd # ~6.5mn
|
||||
script:
|
||||
- make gcc8install
|
||||
- CC=gcc-8 CFLAGS="-Werror" make -j all
|
||||
- make clean
|
||||
- CC=gcc-8 make -j uasan-test-zstd </dev/null # test when stdin is not a tty
|
||||
|
||||
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode # ~4mn
|
||||
script:
|
||||
- make gcc6install libc6install
|
||||
- CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
|
||||
- make clean
|
||||
- CC=gcc-6 make -j uasan-test-zstd32 # note : can complain about pointer overflow
|
||||
|
||||
- name: clang-3.8 + MSan + Test Zstd # ~3.5mn
|
||||
script:
|
||||
- make clang38install
|
||||
# External libraries must be turned off when using MSAN tests,
|
||||
# because they are not msan-instrumented,
|
||||
# so any data coming from these libraries is always considered "uninitialized"
|
||||
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||
|
||||
- name: Minimal Decompressor Macros # ~5mn
|
||||
script:
|
||||
- make clean
|
||||
- make -j all check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
|
||||
- make clean
|
||||
- make -j all check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||
|
||||
- name: cmake build and test check # ~6mn
|
||||
script:
|
||||
- make cmakebuild
|
||||
|
||||
- name: static analyzer scanbuild # ~26mn
|
||||
dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives
|
||||
script:
|
||||
- make staticAnalyze
|
||||
|
||||
- name: gcc-8 + ASan + UBSan + Fuzz Test # ~19mn
|
||||
script:
|
||||
- make gcc8install
|
||||
- CC=gcc-8 make clean uasan-fuzztest
|
||||
|
||||
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit # ~15.5mn
|
||||
script:
|
||||
- make gcc6install libc6install
|
||||
- CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest # can complain about pointer overflow
|
||||
|
||||
- name: clang-3.8 + MSan + Fuzz Test # ~14.5mn
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make clean msan-fuzztest
|
||||
|
||||
- name: ASan + UBSan + MSan + Regression Test # ~ 4.5mn
|
||||
script:
|
||||
- make -j uasanregressiontest
|
||||
- make clean
|
||||
- make -j msanregressiontest
|
||||
|
||||
- name: C++, gnu90 and c99 compatibility # ~3mn
|
||||
script:
|
||||
- make cxxtest
|
||||
- make clean
|
||||
- make gnu90build
|
||||
- make clean
|
||||
- make c99build
|
||||
- make clean
|
||||
- make travis-install # just ensures `make install` works
|
||||
|
||||
- name: mingw cross-compilation
|
||||
script :
|
||||
- sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
|
||||
- CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
||||
|
||||
- name: Valgrind + Fuzz Test Stack Mode # ~ 7mn
|
||||
script:
|
||||
- make valgrindinstall
|
||||
- make -C tests clean valgrindTest
|
||||
- make clean
|
||||
- make -C tests test-fuzzer-stackmode
|
||||
|
||||
- name: Qemu ARM emulation + Fuzz Test # ~13.5mn
|
||||
script:
|
||||
- make arminstall
|
||||
- make armfuzz
|
||||
|
||||
# Introduced to check compat with old toolchains, to prevent e.g. #1872
|
||||
- name: ARM Build Test (on Trusty)
|
||||
dist: trusty
|
||||
script:
|
||||
- make arminstall
|
||||
- make armbuild
|
||||
|
||||
- name: Qemu PPC + Fuzz Test # ~13mn
|
||||
dist: trusty # it seems ppc cross-compilation fails on "current"
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppcfuzz
|
||||
|
||||
# check release number (release only)
|
||||
- name: Tag-Specific Test
|
||||
if: tag =~ ^v[0-9]\.[0-9]
|
||||
script:
|
||||
- make -C tests checkTag
|
||||
- tests/checkTag "$TRAVIS_BRANCH"
|
||||
|
||||
# tests for master branch and cron job only
|
||||
- name: OS-X # ~13mn
|
||||
if: branch = master
|
||||
os: osx
|
||||
script:
|
||||
- make test
|
||||
- make -C lib all
|
||||
|
||||
- name: zbuff test
|
||||
if: branch = master
|
||||
script:
|
||||
- make -C tests test-zbuff
|
||||
|
||||
- name: Versions Compatibility Test # 11.5mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make -C tests versionsTest
|
||||
|
||||
- name: thread sanitizer # ~29mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make tsan-test-zstream
|
||||
- CC=clang-3.8 make tsan-fuzztest
|
||||
|
||||
- name: PPC64LE + Fuzz test # ~13mn
|
||||
if: branch = master
|
||||
arch: ppc64le
|
||||
script:
|
||||
- cat /proc/cpuinfo
|
||||
- make test
|
||||
|
||||
- name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?)
|
||||
dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
|
||||
if: branch = master
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppc64fuzz
|
||||
|
||||
# note : we already have aarch64 tests on hardware
|
||||
- name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn
|
||||
if: branch = master
|
||||
dist: xenial
|
||||
script:
|
||||
- make arminstall
|
||||
- make aarch64fuzz
|
||||
|
||||
- name: zlib wrapper test # ~7.5mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make gpp6install valgrindinstall
|
||||
- make -C zlibWrapper test
|
||||
- make -C zlibWrapper valgrindTest
|
||||
|
||||
- name: LZ4, thread pool, and partial libs tests # ~2mn
|
||||
if: branch = master
|
||||
script:
|
||||
- make lz4install
|
||||
- make -C tests test-lz4
|
||||
- make clean
|
||||
- make -C tests test-pool
|
||||
- make clean
|
||||
- bash tests/libzstd_partial_builds.sh
|
||||
|
||||
# meson dedicated test
|
||||
- name: Xenial (Meson + clang) # ~15mn
|
||||
if: branch = master
|
||||
dist: xenial
|
||||
language: cpp
|
||||
compiler: clang
|
||||
install:
|
||||
- sudo apt-get install -qq liblz4-dev valgrind tree
|
||||
- |
|
||||
travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
|
||||
unzip ~/ninja.zip -d ~/.local/bin
|
||||
- |
|
||||
travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
|
||||
python3 ~/get-pip.py --user &&
|
||||
pip3 install --user meson
|
||||
script:
|
||||
- |
|
||||
meson setup \
|
||||
--buildtype=debugoptimized \
|
||||
-Db_lundef=false \
|
||||
-Dauto_features=enabled \
|
||||
-Dbin_programs=true \
|
||||
-Dbin_tests=true \
|
||||
-Dbin_contrib=true \
|
||||
-Ddefault_library=both \
|
||||
build/meson builddir
|
||||
- pushd builddir
|
||||
- ninja
|
||||
- meson test --verbose --no-rebuild
|
||||
- DESTDIR=./staging ninja install
|
||||
- tree ./staging
|
||||
after_failure:
|
||||
- cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
|
||||
|
||||
allow_failures:
|
||||
- env: ALLOW_FAILURES=true
|
||||
script:
|
||||
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
||||
- echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
|
||||
- export FUZZERTEST=-T5mn;
|
||||
export ZSTREAM_TESTTIME=-T2mn;
|
||||
export DECODECORPUS_TESTTIME=-T1mn;
|
||||
if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
|
||||
git fetch origin dev;
|
||||
git checkout -f FETCH_HEAD;
|
||||
fi;
|
||||
sh -c "$Cmd" || travis_terminate 1;
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
# Code of Conduct
|
||||
|
||||
Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
|
||||
Please read the [full text](https://code.fb.com/codeofconduct/)
|
||||
so that you can understand what actions will and will not be tolerated.
|
||||
+1
-365
@@ -26,370 +26,6 @@ to do this once to work on any of Facebook's open source projects.
|
||||
|
||||
Complete your CLA here: <https://code.facebook.com/cla>
|
||||
|
||||
## Workflow
|
||||
Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
|
||||
will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
|
||||
related changes into a branch.
|
||||
|
||||
Our contribution process works in three main stages:
|
||||
1. Local development
|
||||
* Update:
|
||||
* Checkout your fork of zstd if you have not already
|
||||
```
|
||||
git checkout https://github.com/<username>/zstd
|
||||
cd zstd
|
||||
```
|
||||
* Update your local dev branch
|
||||
```
|
||||
git pull https://github.com/facebook/zstd dev
|
||||
git push origin dev
|
||||
```
|
||||
* Topic and development:
|
||||
* Make a new branch on your fork about the topic you're developing for
|
||||
```
|
||||
# branch names should be consise but sufficiently informative
|
||||
git checkout -b <branch-name>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Make commits and push
|
||||
```
|
||||
# make some changes =
|
||||
git add -u && git commit -m <message>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Note: run local tests to ensure that your changes didn't break existing functionality
|
||||
* Quick check
|
||||
```
|
||||
make shortest
|
||||
```
|
||||
* Longer check
|
||||
```
|
||||
make test
|
||||
```
|
||||
2. Code Review and CI tests
|
||||
* Ensure CI tests pass:
|
||||
* Before sharing anything to the community, make sure that all CI tests pass on your local fork.
|
||||
See our section on setting up your CI environment for more information on how to do this.
|
||||
* Ensure that static analysis passes on your development machine. See the Static Analysis section
|
||||
below to see how to do this.
|
||||
* Create a pull request:
|
||||
* When you are ready to share you changes to the community, create a pull request from your branch
|
||||
to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
|
||||
page.
|
||||
* From there, select the branch where you made changes as your source branch and facebook:dev
|
||||
as the destination.
|
||||
* Examine the diff presented between the two branches to make sure there is nothing unexpected.
|
||||
* Write a good pull request description:
|
||||
* While there is no strict template that our contributors follow, we would like them to
|
||||
sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
|
||||
at least indirectly, address the following points.
|
||||
* Is this pull request important and why?
|
||||
* Is it addressing an issue? If so, what issue? (provide links for convenience please)
|
||||
* Is this a new feature? If so, why is it useful and/or necessary?
|
||||
* Are there background references and documents that reviewers should be aware of to properly assess this change?
|
||||
* Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
|
||||
* Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
|
||||
* Submit the pull request and iterate with feedback.
|
||||
3. Merge and Release
|
||||
* Getting approval:
|
||||
* You will have to iterate on your changes with feedback from other collaborators to reach a point
|
||||
where your pull request can be safely merged.
|
||||
* To avoid too many comments on style and convention, make sure that you have a
|
||||
look at our style section below before creating a pull request.
|
||||
* Eventually, someone from the zstd team will approve your pull request and not long after merge it into
|
||||
the dev branch.
|
||||
* Housekeeping:
|
||||
* Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
|
||||
the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
|
||||
issue at hand, then please indicate this by requesting that an issue be closed by commenting.
|
||||
* Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
|
||||
that the change must make it to an official zstd release for it to be meaningful. We recommend
|
||||
that contributers track the activity on their pull request and corresponding issue(s) page(s) until
|
||||
their change makes it to the next release of zstd. Users will often discover bugs in your code or
|
||||
suggest ways to refine and improve your initial changes even after the pull request is merged.
|
||||
|
||||
## Static Analysis
|
||||
Static analysis is a process for examining the correctness or validity of a program without actually
|
||||
executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
|
||||
static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
|
||||
|
||||
Once installed, you can ensure that our static analysis tests pass on your local development machine
|
||||
by running:
|
||||
```
|
||||
make staticAnalyze
|
||||
```
|
||||
|
||||
In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
|
||||
just `contrib/largeNbDicts` and nothing else, you can run:
|
||||
|
||||
```
|
||||
scan-build make -C contrib/largeNbDicts largeNbDicts
|
||||
```
|
||||
|
||||
### Pitfalls of static analysis
|
||||
`scan-build` is part of our regular CI suite. Other static analyzers are not.
|
||||
|
||||
It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
|
||||
|
||||
- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
|
||||
- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
|
||||
- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
|
||||
+ torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
|
||||
+ sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
|
||||
- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
|
||||
|
||||
This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.
|
||||
|
||||
## Performance
|
||||
Performance is extremely important for zstd and we only merge pull requests whose performance
|
||||
landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
|
||||
This high bar for performance means that every PR which has the potential to
|
||||
impact performance takes a very long time for us to properly review. That being said, we
|
||||
always welcome contributions to improve performance (or worsen performance for the trade-off of
|
||||
something else). Please keep the following in mind before submitting a performance related PR:
|
||||
|
||||
1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
|
||||
very well documented via past Github issues and pull requests. It may be the case that your
|
||||
particular performance optimization has already been considered in the past. Please take some
|
||||
time to search through old issues and pull requests using keywords specific to your
|
||||
would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
|
||||
on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
|
||||
it will be helpful for you to have context from that topic's history before contributing.
|
||||
2. The distinction between noise and actual performance gains can unfortunately be very subtle
|
||||
especially when microbenchmarking extremely small wins or losses. The only remedy to getting
|
||||
something subtle merged is extensive benchmarking. You will be doing us a great favor if you
|
||||
take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
|
||||
benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
|
||||
your changes on every single processor and os out there (and neither will we) but do that best
|
||||
you can:) We've adding some things to think about when benchmarking below in the Benchmarking
|
||||
Performance section which might be helpful for you.
|
||||
3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
|
||||
legitimate thing to do as long as it does not harm the overall performance health of Zstd.
|
||||
This is a hard balance to strike but please keep in mind other aspects of Zstd when
|
||||
submitting changes that are clang-specific, windows-specific, etc.
|
||||
|
||||
## Benchmarking Performance
|
||||
Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
|
||||
testing over theoretical speculation. This guide it not perfect but for most scenarios, it
|
||||
is a good place to start.
|
||||
|
||||
### Stability
|
||||
Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
|
||||
benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
|
||||
will typically not be stable enough to obtain reliable benchmark results. If you can get your
|
||||
hands on a desktop, this is usually a better scenario.
|
||||
|
||||
Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
|
||||
do a little more work to ensure that you are in fact measuring the changes you've made not and
|
||||
noise. Here are some things you can do to make your benchmarks more stable:
|
||||
|
||||
1. The most simple thing you can do to drastically improve the stability of your benchmark is
|
||||
to run it multiple times and then aggregate the results of those runs. As a general rule of
|
||||
thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
|
||||
you will have to aggregate over to get reliable results. Here are some additional things to keep in
|
||||
mind when running multiple trials:
|
||||
* How you aggregate your samples are important. You might be tempted to use the mean of your
|
||||
results. While this is certainly going to be a more stable number than a raw single sample
|
||||
benchmark number, you might have more luck by taking the median. The mean is not robust to
|
||||
outliers whereas the median is. Better still, you could simply take the fastest speed your
|
||||
benchmark achieved on each run since that is likely the fastest your process will be
|
||||
capable of running your code. In our experience, this (aggregating by just taking the sample
|
||||
with the fastest running time) has been the most stable approach.
|
||||
* The more samples you have, the more stable your benchmarks should be. You can verify
|
||||
your improved stability by looking at the size of your confidence intervals as you
|
||||
increase your sample count. These should get smaller and smaller. Eventually hopefully
|
||||
smaller than the performance win you are expecting.
|
||||
* Most processors will take some time to get `hot` when running anything. The observations
|
||||
you collect during that time period will very different from the true performance number. Having
|
||||
a very large number of sample will help alleviate this problem slightly but you can also
|
||||
address is directly by simply not including the first `n` iterations of your benchmark in
|
||||
your aggregations. You can determine `n` by simply looking at the results from each iteration
|
||||
and then hand picking a good threshold after which the variance in results seems to stabilize.
|
||||
2. You cannot really get reliable benchmarks if your host machine is simultaneously running
|
||||
another cpu/memory-intensive application in the background. If you are running benchmarks on your
|
||||
personal laptop for instance, you should close all applications (including your code editor and
|
||||
browser) before running your benchmarks. You might also have invisible background applications
|
||||
running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
|
||||
on Windows. You will get more stable benchmark results of you end those processes as well.
|
||||
* If you have multiple cores, you can even run your benchmark on a reserved core to prevent
|
||||
pollution from other OS and user processes. There are a number of ways to do this depending
|
||||
on your OS:
|
||||
* On linux boxes, you have use https://github.com/lpechacek/cpuset.
|
||||
* On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
|
||||
* On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
|
||||
3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
|
||||
Dynamically linking your library will introduce some added variation (not a large amount but
|
||||
definitely some). Statically linking libzstd will be more stable. Static libraries should
|
||||
be enabled by default when building zstd.
|
||||
4. Use a profiler with a good high resolution timer. See the section below on profiling for
|
||||
details on this.
|
||||
5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
|
||||
6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
|
||||
tmpfs avoids touching a real storage system, which can have a pretty big variability.
|
||||
|
||||
Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
|
||||
|
||||
### Zstd benchmark
|
||||
The fastest signal you can get regarding your performance changes is via the in-build zstd cli
|
||||
bench option. You can run Zstd as you typically would for your scenario using some set of options
|
||||
and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
|
||||
for that options you have just provided. If you want to look at the internals of how this
|
||||
benchmarking script works, you can check out programs/benchzstd.c
|
||||
|
||||
For example: say you have made a change that you believe improves the speed of zstd level 1. The
|
||||
very first thing you should use to asses whether you actually achieved any sort of improvement
|
||||
is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
|
||||
specify a running time for your benchmark in seconds (default is 3 seconds).
|
||||
Usually, the longer the running time, the more stable your results will be.
|
||||
|
||||
```
|
||||
$ git checkout <commit-before-your-change>
|
||||
$ make && cp zstd zstd-old
|
||||
$ git checkout <commit-after-your-change>
|
||||
$ make && cp zstd zstd-new
|
||||
$ zstd-old -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s
|
||||
$ zstd-new -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s
|
||||
```
|
||||
|
||||
Unless your performance win is large enough to be visible despite the intrinsic noise
|
||||
on your computer, benchzstd alone will likely not be enough to validate the impact of your
|
||||
changes. For example, the results of the example above indicate that effectively nothing
|
||||
changed but there could be a small <3% improvement that the noise on the host machine
|
||||
obscured. So unless you see a large performance win (10-15% consistently) using just
|
||||
this method of evaluation will not be sufficient.
|
||||
|
||||
### Profiling
|
||||
There are a number of great profilers out there. We're going to briefly mention how you can
|
||||
profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
|
||||
on windows.
|
||||
|
||||
Say you have an idea for a change that you think will provide some good performance gains
|
||||
for level 1 compression on Zstd. Typically this means, you have identified a section of
|
||||
code that you think can be made to run faster.
|
||||
|
||||
The first thing you will want to do is make sure that the piece of code is actually taking up
|
||||
a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
|
||||
0.0001% of the total running time. Luckily, there are tools to help with this.
|
||||
Profilers will let you see how much time your code spends inside a particular function.
|
||||
If your target code snippit is only part of a function, it might be worth trying to
|
||||
isolate that snippit by moving it to its own function (this is usually not necessary but
|
||||
might be).
|
||||
|
||||
Most profilers (including the profilers dicusssed below) will generate a call graph of
|
||||
functions for you. Your goal will be to find your function of interest in this call grapch
|
||||
and then inspect the time spent inside of it. You might also want to to look at the
|
||||
annotated assembly which most profilers will provide you with.
|
||||
|
||||
#### Instruments
|
||||
We will once again consider the scenario where you think you've identified a piece of code
|
||||
whose performance can be improved upon. Follow these steps to profile your code using
|
||||
Instruments.
|
||||
|
||||
1. Open Instruments
|
||||
2. Select `Time Profiler` from the list of standard templates
|
||||
3. Close all other applications except for your instruments window and your terminal
|
||||
4. Run your benchmarking script from your terminal window
|
||||
* You will want a benchmark that runs for at least a few seconds (5 seconds will
|
||||
usually be long enough). This way the profiler will have something to work with
|
||||
and you will have ample time to attach your profiler to this process:)
|
||||
* I will just use benchzstd as my bencharmking script for this example:
|
||||
```
|
||||
$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
|
||||
```
|
||||
5. Once you run your benchmarking script, switch back over to instruments and attach your
|
||||
process to the time profiler. You can do this by:
|
||||
* Clicking on the `All Processes` drop down in the top left of the toolbar.
|
||||
* Selecting your process from the dropdown. In my case, it is just going to be labled
|
||||
`zstd`
|
||||
* Hitting the bright red record circle button on the top left of the toolbar
|
||||
6. You profiler will now start collecting metrics from your bencharking script. Once
|
||||
you think you have collected enough samples (usually this is the case after 3 seconds of
|
||||
recording), stop your profiler.
|
||||
7. Make sure that in toolbar of the bottom window, `profile` is selected.
|
||||
8. You should be able to see your call graph.
|
||||
* If you don't see the call graph or an incomplete call graph, make sure you have compiled
|
||||
zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
|
||||
you will have to supply the `-g` flag alone with your build script. You might also
|
||||
have to provide the `-fno-omit-frame-pointer` flag
|
||||
9. Dig down the graph to find your function call and then inspect it by double clicking
|
||||
the list item. You will be able to see the annotated source code and the assembly side by
|
||||
side.
|
||||
|
||||
#### Perf
|
||||
|
||||
This wiki has a pretty detailed tutorial on getting started working with perf so we'll
|
||||
leave you to check that out of you're getting started:
|
||||
|
||||
https://perf.wiki.kernel.org/index.php/Tutorial
|
||||
|
||||
Some general notes on perf:
|
||||
* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
|
||||
counter statistics. Perf uses a high resolution timer and this is likely one
|
||||
of the first things your team will run when assessing your PR.
|
||||
* Perf has a long list of hardware counters that can be viewed with `perf --list`.
|
||||
When measuring optimizations, something worth trying is to make sure the handware
|
||||
counters you expect to be impacted by your change are in fact being so. For example,
|
||||
if you expect the L1 cache misses to decrease with your change, you can look at the
|
||||
counter `L1-dcache-load-misses`
|
||||
* Perf hardware counters will not work on a virtual machine.
|
||||
|
||||
#### Visual Studio
|
||||
|
||||
TODO
|
||||
|
||||
|
||||
## Setting up continuous integration (CI) on your fork
|
||||
Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
|
||||
are well tested before they make it to an official release. Specifically, we use the platforms
|
||||
travis-ci, circle-ci, and appveyor.
|
||||
|
||||
Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
|
||||
The easiest way to run these CI tests on your own before submitting a PR to our dev branch
|
||||
is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
|
||||
instructions for doing this.
|
||||
|
||||
### travis-ci
|
||||
Follow these steps to link travis-ci with your github fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://travis-ci.org/
|
||||
3. Click 'Sign in with Github' on the top right
|
||||
4. Click 'Authorize travis-ci'
|
||||
5. Click 'Activate all repositories using Github Apps'
|
||||
6. Select 'Only select repositories' and select your fork of zstd from the drop down
|
||||
7. Click 'Approve and Install'
|
||||
8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
|
||||
9. Click 'Authorize travis-pro'
|
||||
10. You should have travis set up on your fork now.
|
||||
|
||||
### circle-ci
|
||||
TODO
|
||||
|
||||
### appveyor
|
||||
Follow these steps to link circle-ci with your girhub fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://www.appveyor.com/
|
||||
3. Click 'Sign in' on the top right
|
||||
4. Select 'Github' on the left panel
|
||||
5. Click 'Authorize appveyor'
|
||||
6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
|
||||
7. You should have appveyor set up on your fork now.
|
||||
|
||||
### General notes on CI
|
||||
CI tests run every time a pull request (PR) is created or updated. The exact tests
|
||||
that get run will depend on the destination branch you specify. Some tests take
|
||||
longer to run than others. Currently, our CI is set up to run a short
|
||||
series of tests when creating a PR to the dev branch and a longer series of tests
|
||||
when creating a PR to the master branch. You can look in the configuration files
|
||||
of the respective CI platform for more information on what gets run when.
|
||||
|
||||
Most people will just want to create a PR with the destination set to their local dev
|
||||
branch of zstd. You can then find the status of the tests on the PR's page. You can also
|
||||
re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
|
||||
|
||||
## Issues
|
||||
We use GitHub issues to track public bugs. Please ensure your description is
|
||||
clear and has sufficient instructions to be able to reproduce the issue.
|
||||
@@ -398,7 +34,7 @@ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
|
||||
disclosure of security bugs. In those cases, please go through the process
|
||||
outlined on that page and do not file a public issue.
|
||||
|
||||
## Coding Style
|
||||
## Coding Style
|
||||
* 4 spaces for indentation rather than tabs
|
||||
|
||||
## License
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
|
||||
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
# verbose mode (print commands) on V=1 or VERBOSE=1
|
||||
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
BUILDIR = build
|
||||
@@ -21,177 +17,121 @@ FUZZDIR = $(TESTDIR)/fuzz
|
||||
# Define nul output
|
||||
VOID = /dev/null
|
||||
|
||||
# When cross-compiling from linux to windows, you might
|
||||
# need to specify this as "Windows." Fedora build fails
|
||||
# without it.
|
||||
#
|
||||
# Note: mingw-w64 build from linux to windows does not
|
||||
# fail on other tested distros (ubuntu, debian) even
|
||||
# without manually specifying the TARGET_SYSTEM.
|
||||
TARGET_SYSTEM ?= $(OS)
|
||||
|
||||
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
## default: Build lib-release and zstd-release
|
||||
.PHONY: default
|
||||
default: lib-release zstd-release
|
||||
|
||||
.PHONY: all
|
||||
all: allmost examples manual contrib
|
||||
all: | allmost examples manual
|
||||
|
||||
.PHONY: allmost
|
||||
allmost: allzstd zlibwrapper
|
||||
allmost: allzstd
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
|
||||
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||
#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||
.PHONY: allzstd
|
||||
allzstd: lib-all
|
||||
$(Q)$(MAKE) -C $(PRGDIR) all
|
||||
$(Q)$(MAKE) -C $(TESTDIR) all
|
||||
allzstd:
|
||||
$(MAKE) -C $(ZSTDDIR) all
|
||||
$(MAKE) -C $(PRGDIR) all
|
||||
$(MAKE) -C $(TESTDIR) all
|
||||
|
||||
.PHONY: all32
|
||||
all32:
|
||||
$(MAKE) -C $(PRGDIR) zstd32
|
||||
$(MAKE) -C $(TESTDIR) all32
|
||||
|
||||
.PHONY: lib lib-release libzstd.a
|
||||
lib-all : lib
|
||||
lib lib-release lib-all :
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
.PHONY: lib
|
||||
lib:
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
|
||||
.PHONY: zstd zstd-release
|
||||
zstd zstd-release:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)cp $(PRGDIR)/zstd$(EXT) .
|
||||
.PHONY: lib-release
|
||||
lib-release:
|
||||
@$(MAKE) -C $(ZSTDDIR)
|
||||
|
||||
.PHONY: zstd
|
||||
zstd:
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) .
|
||||
|
||||
.PHONY: zstd-release
|
||||
zstd-release:
|
||||
@$(MAKE) -C $(PRGDIR)
|
||||
cp $(PRGDIR)/zstd$(EXT) .
|
||||
|
||||
.PHONY: zstdmt
|
||||
zstdmt:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
|
||||
.PHONY: zlibwrapper
|
||||
zlibwrapper: lib
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
zlibwrapper:
|
||||
$(MAKE) -C $(ZWRAPDIR) test
|
||||
|
||||
## test: run long-duration tests
|
||||
.PHONY: test
|
||||
DEBUGLEVEL ?= 1
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
|
||||
test:
|
||||
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
|
||||
|
||||
## shortest: same as `make check`
|
||||
.PHONY: shortest
|
||||
shortest:
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
## check: run basic tests for `zstd` cli
|
||||
.PHONY: check
|
||||
check: shortest
|
||||
|
||||
.PHONY: automated_benchmarking
|
||||
automated_benchmarking:
|
||||
.PHONY: test shortest
|
||||
test shortest:
|
||||
$(MAKE) -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: benchmarking
|
||||
benchmarking: automated_benchmarking
|
||||
|
||||
## examples: build all examples in `/examples` directory
|
||||
.PHONY: examples
|
||||
examples: lib
|
||||
examples:
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
|
||||
|
||||
## manual: generate API documentation in html format
|
||||
.PHONY: manual
|
||||
manual:
|
||||
$(MAKE) -C contrib/gen_html $@
|
||||
|
||||
## man: generate man page
|
||||
.PHONY: man
|
||||
man:
|
||||
$(MAKE) -C programs $@
|
||||
|
||||
## contrib: build all supported projects in `/contrib` directory
|
||||
.PHONY: contrib
|
||||
contrib: lib
|
||||
$(MAKE) -C contrib/pzstd all
|
||||
$(MAKE) -C contrib/seekable_format/examples all
|
||||
$(MAKE) -C contrib/largeNbDicts all
|
||||
cd contrib/single_file_libs/ ; ./build_decoder_test.sh
|
||||
cd contrib/single_file_libs/ ; ./build_library_test.sh
|
||||
|
||||
.PHONY: cleanTabs
|
||||
cleanTabs:
|
||||
cd contrib; ./cleanTabs
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C examples/ $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
$(Q)$(RM) -r lz4
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C examples/ $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
@$(RM) -r lz4
|
||||
@echo Cleaning completed
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, macOS, Hurd and some BSD targets
|
||||
# make install is validated only for Linux, OSX, Hurd and some BSD targets
|
||||
#------------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT))
|
||||
|
||||
HOST_OS = POSIX
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
#
|
||||
## list: Print all targets and their descriptions (if provided)
|
||||
.PHONY: list
|
||||
list:
|
||||
$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
|
||||
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
|
||||
{ \
|
||||
printf "Target Name\tDescription\n"; \
|
||||
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
|
||||
for target in $$TARGETS; do \
|
||||
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
|
||||
description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
|
||||
printf "$$target\t$$description\n"; \
|
||||
done \
|
||||
} | column -t -s $$'\t'
|
||||
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
|
||||
|
||||
.PHONY: install armtest usan asan uasan
|
||||
.PHONY: install clangtest armtest usan asan uasan
|
||||
install:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: travis-install
|
||||
travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
.PHONY: gppbuild
|
||||
gppbuild: clean
|
||||
g++ -v
|
||||
CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
.PHONY: gcc5build
|
||||
gcc5build: clean
|
||||
gcc-5 -v
|
||||
@@ -210,7 +150,7 @@ gcc7build: clean
|
||||
.PHONY: clangbuild
|
||||
clangbuild: clean
|
||||
clang -v
|
||||
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
|
||||
CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
|
||||
|
||||
m32build: clean
|
||||
gcc -v
|
||||
@@ -223,7 +163,7 @@ aarch64build: clean
|
||||
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
ppcbuild: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
|
||||
ppc64build: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
|
||||
@@ -232,7 +172,6 @@ armfuzz: clean
|
||||
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
aarch64fuzz: clean
|
||||
ld -v
|
||||
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
ppcfuzz: clean
|
||||
@@ -254,6 +193,10 @@ gcc6test: clean
|
||||
gcc-6 -v
|
||||
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
|
||||
|
||||
clangtest: clean
|
||||
clang -v
|
||||
$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
|
||||
@@ -290,31 +233,31 @@ msanregressiontest:
|
||||
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
|
||||
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
asan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
msan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
|
||||
|
||||
msan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
|
||||
|
||||
asan32: clean
|
||||
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"
|
||||
|
||||
uasan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
tsan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
|
||||
|
||||
apt-install:
|
||||
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
|
||||
@@ -338,12 +281,6 @@ libc6install:
|
||||
gcc6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc7install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc8install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
|
||||
|
||||
gpp6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
|
||||
|
||||
@@ -359,7 +296,7 @@ endif
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
endif
|
||||
|
||||
|
||||
@@ -371,31 +308,27 @@ cmakebuild:
|
||||
cmake --version
|
||||
$(RM) -r $(BUILDIR)/cmake/build
|
||||
mkdir $(BUILDIR)/cmake/build
|
||||
cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build -j4;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build install;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
|
||||
cd $(BUILDIR)/cmake/build; ctest -V -L Medium
|
||||
cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||
|
||||
c89build: clean
|
||||
c90build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c89 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
|
||||
gnu90build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
|
||||
CFLAGS="-std=gnu90" $(MAKE) allmost
|
||||
|
||||
c99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c99 -Werror" $(MAKE) allmost
|
||||
CFLAGS="-std=c99" $(MAKE) allmost
|
||||
|
||||
gnu99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
|
||||
CFLAGS="-std=gnu99" $(MAKE) allmost
|
||||
|
||||
c11build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c11 -Werror" $(MAKE) allmost
|
||||
CFLAGS="-std=c11" $(MAKE) allmost
|
||||
|
||||
bmix64build: clean
|
||||
$(CC) -v
|
||||
@@ -409,10 +342,7 @@ bmi32build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
# static analyzer test uses clang's scan-build
|
||||
# does not analyze zlibWrapper, due to detected issues in zlib source code
|
||||
staticAnalyze: SCANBUILD ?= scan-build
|
||||
staticAnalyze:
|
||||
staticAnalyze: clean
|
||||
$(CC) -v
|
||||
CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
|
||||
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
|
||||
endif
|
||||
|
||||
+1
-220
@@ -1,222 +1,3 @@
|
||||
v1.4.5
|
||||
fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
|
||||
perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
|
||||
perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
|
||||
perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
|
||||
perf: Small level 1 compression speed gains (depending on compiler)
|
||||
cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
|
||||
cli : New --filelist= : Provide a list of files to operate upon from a file
|
||||
cli : -b -d command can now benchmark decompression on multiple files
|
||||
cli : New --no-content-size command
|
||||
cli : New --show-default-cparams information command
|
||||
api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
|
||||
api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
|
||||
build: Generate a single-file libzstd library (#2065, by @cwoffenden)
|
||||
build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
|
||||
build: zstd now compiles cleanly under -pedantic (#2099)
|
||||
build: zstd now compiles with make-4.3
|
||||
build: Support mingw cross-compilation from Linux, by @Ericson2314
|
||||
build: Meson multi-thread build fix on windows
|
||||
build: Some misc icc fixes backed by new ci test on travis
|
||||
misc: bitflip analyzer tool, by @felixhandte
|
||||
misc: Extend largeNbDicts benchmark to compression
|
||||
misc: Edit-distance match finder in contrib/
|
||||
doc : Improved beginner CONTRIBUTING.md docs
|
||||
doc : New issue templates for zstd
|
||||
|
||||
v1.4.4
|
||||
perf: Improved decompression speed, by > 10%, by @terrelln
|
||||
perf: Better compression speed when re-using a context, by @felixhandte
|
||||
perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
|
||||
perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
|
||||
perf: minor generic speed optimization, by @davidbolvansky
|
||||
api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
|
||||
api: fixed decoding of magic-less frames, by @terrelln
|
||||
api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
|
||||
cli: Named pipes support, by @bimbashrestha
|
||||
cli: short tar's extension support, by @stokito
|
||||
cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
|
||||
cli: commands --stream-size=# and --size-hint=#, by @nmagerko
|
||||
cli: command --exclude-compressed, by @shashank0791
|
||||
cli: faster `-t` test mode
|
||||
cli: improved some error messages, by @vangyzen
|
||||
cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
|
||||
cli: fix rare deadlock condition within dictionary builder, by @terrelln
|
||||
build: single-file decoder with emscripten compilation script, by @cwoffenden
|
||||
build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
|
||||
build: fixed deprecation warning for certain gcc version, reported by @jasonma163
|
||||
build: fix compilation on old gcc versions, by @cemeyer
|
||||
build: improved installation directories for cmake script, by Dmitri Shubin
|
||||
pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
|
||||
misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
|
||||
misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
|
||||
|
||||
v1.4.3
|
||||
bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
|
||||
bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
|
||||
build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
|
||||
|
||||
v1.4.2
|
||||
bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
|
||||
bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
|
||||
misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
|
||||
misc: Restructure source files by @ephiepark (#1679)
|
||||
|
||||
v1.4.1
|
||||
bug: Fix data corruption in niche use cases by @terrelln (#1659)
|
||||
bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
|
||||
bug: Fix out of bounds read by @terrelln (#1590)
|
||||
perf: Improve decode speed by ~7% @mgrice (#1668)
|
||||
perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
|
||||
perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
|
||||
perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
|
||||
perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
|
||||
api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
|
||||
cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
|
||||
cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
|
||||
cli: Restrict read permissions on destination files by @chungy (#1644)
|
||||
cli: zstdgrep: handle -f flag by @felixhandte (#1618)
|
||||
cli: zstdcat: follow symlinks by @vejnar (#1604)
|
||||
doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
|
||||
doc: Fix typo by @yk-tanigawa (#1633)
|
||||
doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
|
||||
build: CMake: support building with LZ4 @leeyoung624 (#1626)
|
||||
build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
|
||||
build: CMake: respect existing uninstall target by @j301scott (#1619)
|
||||
build: Make: skip multithread tests when built without support by @michaelforney (#1620)
|
||||
build: Make: Fix examples/ test target by @sjnam (#1603)
|
||||
build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
|
||||
build: Meson: fix build by @lzutao (#1602)
|
||||
build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
|
||||
build: Visual Studio: fix linking by @absotively (#1639)
|
||||
build: Fix MinGW-W64 build by @myzhang1029 (#1600)
|
||||
misc: Expand decodecorpus coverage by @ephiepark (#1664)
|
||||
|
||||
v1.4.0
|
||||
perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
|
||||
api: Move the advanced API, including all functions in the staging section, to the stable section
|
||||
api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
|
||||
api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
|
||||
api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
|
||||
api: Don't export ZSTDMT functions from the shared library by default
|
||||
api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
|
||||
api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
|
||||
api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
|
||||
api: Move ZSTD_getDictID_*() functions to the stable section
|
||||
api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
|
||||
api: Allow compression parameters to be set when a dictionary is used
|
||||
api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
|
||||
api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
|
||||
api: Setting ZSTD_d_maxWindowLog to 0 means use the default
|
||||
cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
|
||||
cli: Add --[no-]compress-literals flag to enable or disable literal compression
|
||||
doc: Update the examples to use the advanced API
|
||||
doc: Explain how to transition from old streaming functions to the advanced API in the header
|
||||
build: Improve the Windows release packages
|
||||
build: Improve CMake build by @hjmjohnson
|
||||
build: Build fixes for FreeBSD by @lwhsu
|
||||
build: Remove redundant warnings by @thatsafunnyname
|
||||
build: Fix tests on OpenBSD by @bket
|
||||
build: Extend fuzzer build system to work with the new clang engine
|
||||
build: CMake now creates the libzstd.so.1 symlink
|
||||
build: Improve Menson build by @lzutao
|
||||
misc: Fix symbolic link detection on FreeBSD
|
||||
misc: Use physical core count for -T0 on FreeBSD by @cemeyer
|
||||
misc: Fix zstd --list on truncated files by @kostmo
|
||||
misc: Improve logging in debug mode by @felixhandte
|
||||
misc: Add CirrusCI tests by @lwhsu
|
||||
misc: Optimize dictionary memory usage in corner cases
|
||||
misc: Improve the dictionary builder on small or homogeneous data
|
||||
misc: Fix spelling across the repo by @jsoref
|
||||
|
||||
v1.3.8
|
||||
perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
|
||||
perf: slightly better compression ratio at high compression modes
|
||||
api : finalized advanced API, last stage before "stable" status
|
||||
api : new --rsyncable mode, by @terrelln
|
||||
api : support decompression of empty frames into NULL (used to be an error) (#1385)
|
||||
build: new set of macros to build a minimal size decoder, by @felixhandte
|
||||
build: fix compilation on MIPS32, reported by @clbr (#1441)
|
||||
build: fix compilation with multiple -arch flags, by @ryandesign
|
||||
build: highly upgraded meson build, by @lzutao
|
||||
build: improved buck support, by @obelisk
|
||||
build: fix cmake script : can create debug build, by @pitrou
|
||||
build: Makefile : grep works on both colored consoles and systems without color support
|
||||
build: fixed zstd-pgo, by @bmwiedemann
|
||||
cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
|
||||
cli : --no-progress flag, preserving final summary (#1371), by @terrelln
|
||||
cli : ensure destination file is not source file (#1422)
|
||||
cli : clearer error messages, especially when input file not present
|
||||
doc : clarified zstd_compression_format.md, by @ulikunitz
|
||||
misc: fixed zstdgrep, returns 1 on failure, by @lzutao
|
||||
misc: NEWS renamed as CHANGELOG, in accordance with fboss
|
||||
|
||||
v1.3.7
|
||||
perf: slightly better decompression speed on clang (depending on hardware target)
|
||||
fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
|
||||
build: no longer build backtrace by default in release mode; restrict further automatic mode
|
||||
build: control backtrace support through build macro BACKTRACE
|
||||
misc: added man pages for zstdless and zstdgrep, by @samrussell
|
||||
|
||||
v1.3.6
|
||||
perf: much faster dictionary builder, by @jenniferliu
|
||||
perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
|
||||
perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
|
||||
cli : fix : does no longer overwrite destination when source does not exist (#1082)
|
||||
cli : new command --adapt, for automatic compression level adaptation
|
||||
api : fix : block api can be streamed with > 4 GB, reported by @catid
|
||||
api : reduced ZSTD_DDict size by 2 KB
|
||||
api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
|
||||
build: support Haiku target, by @korli
|
||||
build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
|
||||
doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
|
||||
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
|
||||
|
||||
v1.3.5
|
||||
perf: much faster dictionary compression, by @felixhandte
|
||||
perf: small quality improvement for dictionary generation, by @terrelln
|
||||
perf: slightly improved high compression levels (notably level 19)
|
||||
mem : automatic memory release for long duration contexts
|
||||
cli : fix : overlapLog can be manually set
|
||||
cli : fix : decoding invalid lz4 frames
|
||||
api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
|
||||
api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
|
||||
build: select custom libzstd scope through control macros, by @GeorgeLu97
|
||||
build: OpenBSD patch, by @bket
|
||||
build: make and make all are compatible with -j
|
||||
doc : clarify zstd_compression_format.md, updated for IETF RFC process
|
||||
misc: pzstd compatible with reproducible compilation, by @lamby
|
||||
|
||||
v1.3.4
|
||||
perf: faster speed (especially decoding speed) on recent cpus (haswell+)
|
||||
perf: much better performance associating --long with multi-threading, by @terrelln
|
||||
perf: better compression at levels 13-15
|
||||
cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
|
||||
cli : smoother status report in multi-threading mode
|
||||
cli : added command --fast=#, for faster compression modes
|
||||
cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
|
||||
api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
|
||||
api : compression levels can be negative, for even more speed
|
||||
api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
|
||||
api : ZSTDMT can accept new compression parameters during compression
|
||||
api : implemented all advanced dictionary decompression prototypes
|
||||
build: improved meson recipe, by Shawn Landden (@shawnl)
|
||||
build: VS2017 scripts, by @HaydnTrigg
|
||||
misc: all /contrib projects fixed
|
||||
misc: added /contrib/docker script by @gyscos
|
||||
|
||||
v1.3.3
|
||||
perf: faster zstd_opt strategy (levels 16-19)
|
||||
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
|
||||
cli : fix : content size written in header by default
|
||||
cli : fix : improved LZ4 format support, by @felixhandte
|
||||
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
|
||||
api : fix : support large skippable frames, by @terrelln
|
||||
api : fix : streaming interface was adding a useless 3-bytes null block to small frames
|
||||
api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
|
||||
build: fix : compilation under rhel6 and centos6, reported by @pixelb
|
||||
build: added `check` target
|
||||
|
||||
v1.3.2
|
||||
new : long range mode, using --long command, by Stella Lau (@stellamplau)
|
||||
new : ability to generate and decode magicless frames (#591)
|
||||
@@ -370,7 +151,7 @@ v1.0.0
|
||||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
@@ -1,21 +1,14 @@
|
||||
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
It is provided as an open-source BSD-licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst` and `.gz` files.
|
||||
For other programming languages,
|
||||
you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
||||
Should your project require another programming language,
|
||||
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
**Development branch status:**
|
||||
|
||||
[![Build Status][travisDevBadge]][travisLink]
|
||||
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
||||
[![Build status][CircleDevBadge]][CircleLink]
|
||||
[![Build status][CirrusDevBadge]][CirrusLink]
|
||||
[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
|
||||
| dev branch status |
|
||||
|-------------------|
|
||||
| [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
|
||||
|
||||
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
||||
[travisLink]: https://travis-ci.org/facebook/zstd
|
||||
@@ -23,67 +16,51 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
|
||||
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
|
||||
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
||||
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
||||
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
|
||||
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
|
||||
[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
|
||||
[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
|
||||
|
||||
## Benchmarks
|
||||
|
||||
For reference, several fast compression algorithms were tested and compared
|
||||
on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
|
||||
with a Core i9-9900K CPU @ 5.0GHz,
|
||||
As a reference, several fast compression algorithms were tested and compared
|
||||
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
|
||||
with a Core i7-6700K CPU @ 4.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
compiled with [gcc] 9.3.0,
|
||||
compiled with GCC 6.3.0,
|
||||
on the [Silesia compression corpus].
|
||||
|
||||
[lzbench]: https://github.com/inikep/lzbench
|
||||
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||
[gcc]: https://gcc.gnu.org/
|
||||
|
||||
| Compressor name | Ratio | Compression| Decompress.|
|
||||
| --------------- | ------| -----------| ---------- |
|
||||
| **zstd 1.4.5 -1** | 2.884 | 500 MB/s | 1660 MB/s |
|
||||
| zlib 1.2.11 -1 | 2.743 | 90 MB/s | 400 MB/s |
|
||||
| brotli 1.0.7 -0 | 2.703 | 400 MB/s | 450 MB/s |
|
||||
| **zstd 1.4.5 --fast=1** | 2.434 | 570 MB/s | 2200 MB/s |
|
||||
| **zstd 1.4.5 --fast=3** | 2.312 | 640 MB/s | 2300 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 560 MB/s | 710 MB/s |
|
||||
| **zstd 1.4.5 --fast=5** | 2.178 | 700 MB/s | 2420 MB/s |
|
||||
| lzo1x 2.10 -1 | 2.106 | 690 MB/s | 820 MB/s |
|
||||
| lz4 1.9.2 | 2.101 | 740 MB/s | 4530 MB/s |
|
||||
| **zstd 1.4.5 --fast=7** | 2.096 | 750 MB/s | 2480 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 410 MB/s | 860 MB/s |
|
||||
| snappy 1.1.8 | 2.073 | 560 MB/s | 1790 MB/s |
|
||||
| **zstd 1.1.3 -1** | 2.877 | 430 MB/s | 1110 MB/s |
|
||||
| zlib 1.2.8 -1 | 2.743 | 110 MB/s | 400 MB/s |
|
||||
| brotli 0.5.2 -0 | 2.708 | 400 MB/s | 430 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s |
|
||||
| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s |
|
||||
| lz4 1.7.5 | 2.101 | 720 MB/s | 3600 MB/s |
|
||||
| snappy 1.1.3 | 2.091 | 500 MB/s | 1650 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
|
||||
|
||||
[zlib]: http://www.zlib.net/
|
||||
[zlib]:http://www.zlib.net/
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
The negative compression levels, specified with `--fast=#`,
|
||||
offer faster compression and decompression speed in exchange for some loss in
|
||||
compression ratio compared to level 1, as seen in the table above.
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increments.
|
||||
Decompression speed is preserved and remains roughly the same at all settings,
|
||||
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
|
||||
The following tests were run
|
||||
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)
|
||||
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
|
||||
with a Core i7-6700K CPU @ 4.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
compiled with [gcc] 7.3.0,
|
||||
compiled with GCC 6.3.0,
|
||||
on the [Silesia compression corpus].
|
||||
|
||||
Compression Speed vs Ratio | Decompression Speed
|
||||
---------------------------|--------------------
|
||||
 | 
|
||||
 | 
|
||||
|
||||
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
|
||||
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
||||
Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
|
||||
For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
|
||||
|
||||
|
||||
## The case for Small Data compression
|
||||
### The case for Small Data compression
|
||||
|
||||
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
||||
|
||||
@@ -107,93 +84,70 @@ Training works if there is some correlation in a family of small data samples. T
|
||||
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
||||
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
### Dictionary compression How To:
|
||||
#### Dictionary compression How To :
|
||||
|
||||
1. Create the dictionary
|
||||
1) Create the dictionary
|
||||
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
|
||||
2. Compress with dictionary
|
||||
2) Compress with dictionary
|
||||
|
||||
`zstd -D dictionaryName FILE`
|
||||
`zstd -D dictionaryName FILE`
|
||||
|
||||
3. Decompress with dictionary
|
||||
3) Decompress with dictionary
|
||||
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
|
||||
|
||||
## Build instructions
|
||||
### Build
|
||||
|
||||
### Makefile
|
||||
Once you have the repository cloned, there are multiple ways provided to build Zstandard.
|
||||
|
||||
If your system is compatible with standard `make` (or `gmake`),
|
||||
invoking `make` in root directory will generate `zstd` cli in root directory.
|
||||
#### Makefile
|
||||
|
||||
Other available options include:
|
||||
- `make install` : create and install zstd cli, library and man pages
|
||||
- `make check` : create and run `zstd`, tests its behavior on local platform
|
||||
If your system is compatible with a standard `make` (or `gmake`) binary generator,
|
||||
you can simply run it at the root directory.
|
||||
It will generate `zstd` within root directory.
|
||||
|
||||
### cmake
|
||||
Other available options include :
|
||||
- `make install` : create and install zstd binary, library and man page
|
||||
- `make test` : create and run `zstd` and test tools on local platform
|
||||
|
||||
#### cmake
|
||||
|
||||
A `cmake` project generator is provided within `build/cmake`.
|
||||
It can generate Makefiles or other build scripts
|
||||
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||
|
||||
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||
#### Meson
|
||||
|
||||
### Meson
|
||||
A Meson project is provided within `contrib/meson`.
|
||||
|
||||
A Meson project is provided within [`build/meson`](build/meson). Follow
|
||||
build instructions in that directory.
|
||||
#### Visual Studio (Windows)
|
||||
|
||||
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
||||
example about how Meson is used to build this project.
|
||||
|
||||
Note that default build type is **release**.
|
||||
|
||||
### VCPKG
|
||||
You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
|
||||
|
||||
git clone https://github.com/Microsoft/vcpkg.git
|
||||
cd vcpkg
|
||||
./bootstrap-vcpkg.sh
|
||||
./vcpkg integrate install
|
||||
./vcpkg install zstd
|
||||
|
||||
The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
|
||||
If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
|
||||
|
||||
### Visual Studio (Windows)
|
||||
|
||||
Going into `build` directory, you will find additional possibilities:
|
||||
- Projects for Visual Studio 2005, 2008 and 2010.
|
||||
+ VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
|
||||
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
||||
Going into `build` directory, you will find additional possibilities :
|
||||
- Projects for Visual Studio 2005, 2008 and 2010
|
||||
+ VS2010 project is compatible with VS2012, VS2013 and VS2015
|
||||
- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
|
||||
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
||||
|
||||
### Buck
|
||||
|
||||
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
||||
The output binary will be in `buck-out/gen/programs/`.
|
||||
|
||||
## Testing
|
||||
|
||||
You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory.
|
||||
Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary.
|
||||
For information on CI testing, please refer to TESTING.md
|
||||
|
||||
## Status
|
||||
### Status
|
||||
|
||||
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
||||
Zstandard is considered safe for production environments.
|
||||
|
||||
## License
|
||||
### License
|
||||
|
||||
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
||||
|
||||
## Contributing
|
||||
### Contributing
|
||||
|
||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
||||
The "dev" branch is the one where all contributions will be merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
|
||||
Direct commit to "master" are not permitted.
|
||||
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
||||
|
||||
### Miscellaneous
|
||||
|
||||
Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@ They consist of the following tests:
|
||||
- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
|
||||
- Compilation on various versions of gcc, clang, and g++
|
||||
- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
|
||||
|
||||
Medium Tests
|
||||
------------
|
||||
@@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
|
||||
They consist of the following tests:
|
||||
- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
|
||||
Msan on x86_64
|
||||
- `tests/playTests.sh --test-large-data`
|
||||
- `tests/playTests.sh --test-long-data`
|
||||
- Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
|
||||
- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
|
||||
- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
|
||||
@@ -41,4 +41,4 @@ They consist of the following tests:
|
||||
- `pzstd` with asan and tsan, as well as in 32-bits mode
|
||||
- Testing `zstd` with legacy mode off
|
||||
- Testing `zbuff` (old streaming API)
|
||||
- Entire test suite and make install on macOS
|
||||
- Entire test suite and make install on OS X
|
||||
|
||||
+9
-58
@@ -1,20 +1,15 @@
|
||||
# Following tests are run _only_ on master branch
|
||||
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
|
||||
# or a branch `visual*`, they will intentionnally trigger `master` tests
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
branches:
|
||||
only:
|
||||
- dev
|
||||
- master
|
||||
- appveyorTest
|
||||
- /visual*/
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-dll fullbench-lib"
|
||||
ARTIFACT: "true"
|
||||
BUILD: "true"
|
||||
- COMPILER: "gcc"
|
||||
@@ -85,22 +80,12 @@
|
||||
SET "LDFLAGS=../../zlib/libz.a" &&
|
||||
sh -c "%SCRIPT%" &&
|
||||
( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
|
||||
ECHO Creating artifacts &&
|
||||
ECHO %cd% &&
|
||||
lib\dll\example\build_package.bat &&
|
||||
make -C programs DEBUGFLAGS= clean zstd &&
|
||||
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
||||
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
||||
cp zstd.exe ..\bin\zstd.exe &&
|
||||
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
||||
cd zstd &&
|
||||
git archive --format=tar master -o zstd-src.tar &&
|
||||
..\zstd -19 zstd-src.tar &&
|
||||
appveyor PushArtifact zstd-src.tar.zst &&
|
||||
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
||||
appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
|
||||
cd ..\..\bin\ &&
|
||||
7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
|
||||
cd ..\bin\ && 7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
|
||||
appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
|
||||
)
|
||||
)
|
||||
@@ -169,40 +154,33 @@
|
||||
- SET "FUZZERTEST=-T30s"
|
||||
- if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||
CD tests &&
|
||||
SET ZSTD_BIN=./zstd.exe&&
|
||||
SET DATAGEN_BIN=./datagen.exe&&
|
||||
SET ZSTD=./zstd.exe &&
|
||||
sh -e playTests.sh --test-large-data &&
|
||||
fullbench.exe -i1 &&
|
||||
fullbench.exe -i1 -P0 &&
|
||||
fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
)
|
||||
|
||||
|
||||
# The following tests are for regular pushes
|
||||
# into `dev` or some feature branch
|
||||
# There run less tests, for shorter feedback loop
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "cygwin"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
||||
SCRIPT: "make allzstd"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x86"
|
||||
SCRIPT: "CFLAGS=-Werror make -j allzstd"
|
||||
SCRIPT: "make allzstd"
|
||||
- COMPILER: "clang"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
||||
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
|
||||
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
@@ -224,14 +202,6 @@
|
||||
install:
|
||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET PATH_ORIGINAL=%PATH%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
ECHO Installing Cygwin Packages &&
|
||||
C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
|
||||
gcc-g++,^
|
||||
gcc,^
|
||||
cmake,^
|
||||
make
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
|
||||
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
|
||||
@@ -244,17 +214,6 @@
|
||||
|
||||
build_script:
|
||||
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
set CHERE_INVOKING=yes &&
|
||||
set CC=%COMPILER% &&
|
||||
C:\cygwin64\bin\bash --login -c "
|
||||
set -e;
|
||||
cd build/cmake;
|
||||
CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
|
||||
make -j4;
|
||||
ctest -V -L Medium;
|
||||
"
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
( if [%PLATFORM%]==[x64] (
|
||||
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
|
||||
@@ -279,11 +238,3 @@
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
|
||||
)
|
||||
|
||||
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[mingw] (
|
||||
set "CC=%COMPILER%" &&
|
||||
make check
|
||||
)
|
||||
|
||||
+5
-18
@@ -8,26 +8,13 @@
|
||||
*.user
|
||||
*.opendb
|
||||
|
||||
VS2008/bin/
|
||||
VS2005/
|
||||
VS2008/
|
||||
VS2010/bin/
|
||||
VS2010/zwrapbench/
|
||||
VS2012/
|
||||
VS2013/
|
||||
VS2015/
|
||||
Studio*
|
||||
VS2012/bin/
|
||||
VS2013/bin/
|
||||
VS2015/bin/
|
||||
|
||||
# CMake
|
||||
cmake/build/
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
CMakeScripts
|
||||
Testing
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
install_manifest.txt
|
||||
compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
build
|
||||
lib
|
||||
!cmake/lib
|
||||
!meson/lib
|
||||
|
||||
@@ -332,10 +332,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -364,18 +360,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
|
||||
>
|
||||
@@ -392,14 +376,6 @@
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
@@ -408,22 +384,10 @@
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\tests\fullbench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -506,22 +470,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -121,7 +121,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -195,7 +195,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -272,7 +272,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -328,14 +328,6 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
@@ -344,10 +336,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -364,10 +352,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -412,30 +396,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -514,10 +478,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -542,22 +502,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -122,7 +122,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -197,7 +197,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -275,7 +275,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -333,19 +333,7 @@
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchzstd.c"
|
||||
RelativePath="..\..\..\programs\bench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
@@ -360,10 +348,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -372,10 +356,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -424,30 +404,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -558,10 +518,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -622,22 +578,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -121,7 +121,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -195,7 +195,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -272,7 +272,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -332,10 +332,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
@@ -352,10 +348,6 @@
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\hist.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.c"
|
||||
>
|
||||
@@ -396,18 +388,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -432,14 +412,6 @@
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -526,10 +498,6 @@
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
@@ -554,22 +522,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -155,7 +155,6 @@
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\datagencli.c" />
|
||||
</ItemGroup>
|
||||
@@ -165,4 +164,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -166,16 +166,12 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -186,4 +182,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -156,20 +156,15 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
@@ -178,12 +173,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -196,21 +186,15 @@
|
||||
<ClInclude Include="..\..\..\lib\common\threading.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
|
||||
@@ -90,7 +90,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -105,7 +105,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -122,7 +122,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -142,7 +142,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -156,10 +156,8 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
@@ -167,9 +165,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -178,14 +173,9 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\fuzzer.c" />
|
||||
</ItemGroup>
|
||||
@@ -199,20 +189,14 @@
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
|
||||
@@ -20,10 +20,8 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
@@ -31,9 +29,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,13 +37,10 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -81,17 +73,12 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ResourceCompile Include="libzstd-dll.rc" />
|
||||
@@ -172,7 +159,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -192,7 +179,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -212,10 +199,11 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -233,13 +221,14 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -20,10 +20,8 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\threading.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
@@ -31,9 +29,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,13 +37,10 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -81,42 +73,42 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>libzstd</RootNamespace>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
|
||||
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
@@ -136,36 +128,35 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -185,7 +176,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -205,10 +196,11 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -226,13 +218,14 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\debug.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\error_private.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\pool.c" />
|
||||
@@ -27,14 +26,10 @@
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\hist.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,10 +37,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
@@ -55,10 +47,7 @@
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchzstd.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\dibio.c" />
|
||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||
@@ -70,7 +59,6 @@
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
@@ -78,16 +66,11 @@
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
|
||||
@@ -96,7 +79,7 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchzstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\dibio.h" />
|
||||
<ClInclude Include="..\..\..\programs\fileio.h" />
|
||||
@@ -182,7 +165,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -198,7 +181,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -216,7 +199,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -237,7 +220,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
|
||||
+17
-27
@@ -7,58 +7,48 @@ Here are a few command lines for reference :
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build.VS2013.cmd
|
||||
build\build.VS2013.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `bin\Release\{ARCH}\` folder.
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build.generic.cmd VS2013 Win32 Release v120`
|
||||
- x64: `build.generic.cmd VS2013 x64 Release v120`
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Release v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Release v120`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build.generic.cmd VS2013 Win32 Debug v120`
|
||||
- x64: `build.generic.cmd VS2013 x64 Debug v120`
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Debug v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Debug v120`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr140.dll
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build.VS2015.cmd
|
||||
build\build.VS2015.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `bin\Release\{ARCH}\` folder.
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build.generic.cmd VS2015 Win32 Release v140`
|
||||
- x64: `build.generic.cmd VS2015 x64 Release v140`
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Release v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Release v140`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build.generic.cmd VS2015 Win32 Debug v140`
|
||||
- x64: `build.generic.cmd VS2015 x64 Debug v140`
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Debug v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Debug v140`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr120.dll
|
||||
|
||||
This capability is offered through `build.generic.cmd` using proper arguments:
|
||||
You need to invoke `build\build.generic.cmd` with the proper arguments:
|
||||
|
||||
**For Win32**
|
||||
```batch
|
||||
build.generic.cmd VS2015 Win32 Release v120
|
||||
build\build.generic.cmd VS2015 Win32 Release v120
|
||||
```
|
||||
The result of the build will be in the `bin\Release\Win32\` folder.
|
||||
The result of the build will be in the `build\bin\Release\Win32\` folder.
|
||||
|
||||
**For x64**
|
||||
```batch
|
||||
build.generic.cmd VS2015 x64 Release v120
|
||||
build\build.generic.cmd VS2015 x64 Release v120
|
||||
```
|
||||
The result of the build will be in the `bin\Release\x64\` folder.
|
||||
The result of the build will be in the `build\bin\Release\x64\` folder.
|
||||
|
||||
If you want Debug builds, replace `Release` with `Debug`.
|
||||
|
||||
### Build with Visual Studio 2017
|
||||
|
||||
`build.VS2017.cmd`, contributed by [@HaydnTrigg](https://github.com/HaydnTrigg),
|
||||
will build both the `Release Win32` and `Release x64` versions
|
||||
of the first VS2017 variant it finds, in this priority order :
|
||||
Enterprise > Professional > Community
|
||||
|
||||
Alternatively, it's possible to target a specific version,
|
||||
using appropriate script, such as `build.VS2017Enterprise.cmd` for example.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017 Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017 x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Community Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Community x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Enterprise Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Enterprise x64 Release v141
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Professional Win32 Release v141
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2017Professional x64 Release v141
|
||||
@@ -19,29 +19,19 @@ GOTO build
|
||||
:display_help
|
||||
|
||||
echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
|
||||
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, VS2017, ...)
|
||||
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, ...)
|
||||
echo msbuild_platform: Platform (x64 or Win32)
|
||||
echo msbuild_configuration: VS configuration (Release or Debug)
|
||||
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140, v141)
|
||||
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140)
|
||||
|
||||
EXIT /B 1
|
||||
|
||||
:build
|
||||
|
||||
SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
|
||||
SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
|
||||
IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
|
||||
IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
|
||||
IF %msbuild_version% == VS2017 (
|
||||
IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
|
||||
IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
|
||||
IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
|
||||
)
|
||||
rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ?
|
||||
|
||||
SET project="%~p0\..\VS2010\zstd.sln"
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
# cmake working directory
|
||||
cmakeBuild
|
||||
|
||||
# cmake artefacts
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
|
||||
+40
-164
@@ -7,197 +7,73 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
cmake_minimum_required(VERSION 2.8.9 FATAL_ERROR)
|
||||
|
||||
# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
|
||||
# Set and use the newest cmake policies that are validated to work
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
|
||||
if("${CMAKE_MAJOR_VERSION}" LESS 3)
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
|
||||
"${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
else()
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
|
||||
endif()
|
||||
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
|
||||
|
||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
# Parse version
|
||||
include(GetZstdLibraryVersion)
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
|
||||
|
||||
if( CMAKE_MAJOR_VERSION LESS 3 )
|
||||
## Provide cmake 3+ behavior for older versions of cmake
|
||||
project(zstd)
|
||||
set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
|
||||
set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
|
||||
set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
|
||||
set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
|
||||
enable_language(C) # Main library is in C
|
||||
enable_language(CXX) # Testing contributed code also utilizes CXX
|
||||
else()
|
||||
project(zstd
|
||||
VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
|
||||
LANGUAGES C # Main library is in C
|
||||
CXX # Testing contributed code also utilizes CXX
|
||||
)
|
||||
endif()
|
||||
message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
|
||||
set(zstd_HOMEPAGE_URL "http://www.zstd.net")
|
||||
set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
|
||||
|
||||
# Set a default build type if none was specified
|
||||
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
||||
message(STATUS "Setting build type to 'Release' as none was specified.")
|
||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
|
||||
# Set the possible values of build type for cmake-gui
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
|
||||
endif()
|
||||
|
||||
include(GNUInstallDirs)
|
||||
PROJECT(zstd)
|
||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
|
||||
SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add extra compilation flags
|
||||
#-----------------------------------------------------------------------------
|
||||
include(AddZstdCompilationFlags)
|
||||
INCLUDE(AddZstdCompilationFlags)
|
||||
ADD_ZSTD_COMPILATION_FLAGS()
|
||||
|
||||
# Always hide XXHash symbols
|
||||
add_definitions(-DXXH_NAMESPACE=ZSTD_)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Installation variables
|
||||
#-----------------------------------------------------------------------------
|
||||
message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
|
||||
message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
|
||||
ADD_DEFINITIONS(-DXXH_NAMESPACE=ZSTD_)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Options
|
||||
#-----------------------------------------------------------------------------
|
||||
|
||||
# Legacy support
|
||||
option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=5)
|
||||
else ()
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=0)
|
||||
endif ()
|
||||
|
||||
# Multi-threading support
|
||||
option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
|
||||
else ()
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
|
||||
endif ()
|
||||
|
||||
option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
|
||||
option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
|
||||
|
||||
# Respect the conventional CMake option for enabling tests if it was specified on the first configure
|
||||
if (BUILD_TESTING)
|
||||
set(ZSTD_BUILD_TESTS_default ON)
|
||||
else()
|
||||
set(ZSTD_BUILD_TESTS_default OFF)
|
||||
endif()
|
||||
option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
|
||||
OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
IF (UNIX)
|
||||
OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
|
||||
ELSE (UNIX)
|
||||
OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" OFF)
|
||||
ENDIF (UNIX)
|
||||
OPTION(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
|
||||
OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
|
||||
OPTION(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
|
||||
if (MSVC)
|
||||
option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
|
||||
OPTION(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# External dependencies
|
||||
#-----------------------------------------------------------------------------
|
||||
if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if(CMAKE_USE_PTHREADS_INIT)
|
||||
set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
endif ()
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
|
||||
ELSE (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add source directories
|
||||
#-----------------------------------------------------------------------------
|
||||
add_subdirectory(lib)
|
||||
ADD_SUBDIRECTORY(lib)
|
||||
|
||||
option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
|
||||
IF (ZSTD_BUILD_PROGRAMS)
|
||||
IF (NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
ENDIF (NOT ZSTD_BUILD_STATIC)
|
||||
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build shared library to build zstd CLI")
|
||||
endif ()
|
||||
ADD_SUBDIRECTORY(programs)
|
||||
ENDIF (ZSTD_BUILD_PROGRAMS)
|
||||
|
||||
add_subdirectory(programs)
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_TESTS)
|
||||
IF (NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build static library to build tests")
|
||||
ENDIF (NOT ZSTD_BUILD_STATIC)
|
||||
|
||||
if (ZSTD_BUILD_TESTS)
|
||||
enable_testing()
|
||||
if (NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build static library to build tests")
|
||||
endif ()
|
||||
ADD_SUBDIRECTORY(tests)
|
||||
ENDIF (ZSTD_BUILD_TESTS)
|
||||
|
||||
add_subdirectory(tests)
|
||||
endif ()
|
||||
|
||||
if (ZSTD_BUILD_CONTRIB)
|
||||
add_subdirectory(contrib)
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_CONTRIB)
|
||||
ADD_SUBDIRECTORY(contrib)
|
||||
ENDIF (ZSTD_BUILD_CONTRIB)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add clean-all target
|
||||
#-----------------------------------------------------------------------------
|
||||
add_custom_target(clean-all
|
||||
ADD_CUSTOM_TARGET(clean-all
|
||||
COMMAND ${CMAKE_BUILD_TOOL} clean
|
||||
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
|
||||
)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Generate Package Config files
|
||||
#
|
||||
# This section is based on the boiler plate code from:
|
||||
# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
|
||||
#-----------------------------------------------------------------------------
|
||||
include(CMakePackageConfigHelpers)
|
||||
write_basic_package_version_file(
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
VERSION ${zstd_VERSION}
|
||||
COMPATIBILITY SameMajorVersion
|
||||
)
|
||||
|
||||
# A Package Config file that works from the build directory
|
||||
export(EXPORT zstdExports
|
||||
FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
|
||||
NAMESPACE zstd::
|
||||
)
|
||||
configure_file(zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
|
||||
COPYONLY
|
||||
)
|
||||
|
||||
# A Package Config file that works from the installation directory
|
||||
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
|
||||
install(EXPORT zstdExports
|
||||
FILE zstdTargets.cmake
|
||||
NAMESPACE zstd::
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
install(FILES
|
||||
zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
|
||||
@@ -20,7 +20,7 @@ function(EnableCompilerFlag _flag _C _CXX)
|
||||
endif ()
|
||||
endfunction()
|
||||
|
||||
macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
MACRO(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
|
||||
#Set c++11 by default
|
||||
EnableCompilerFlag("-std=c++11" false true)
|
||||
@@ -33,10 +33,6 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
EnableCompilerFlag("-Wcast-align" true true)
|
||||
EnableCompilerFlag("-Wcast-qual" true true)
|
||||
EnableCompilerFlag("-Wstrict-prototypes" true false)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
elseif (MSVC) # Add specific compilation flags for Windows Visual
|
||||
|
||||
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
|
||||
@@ -47,32 +43,25 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
# UNICODE SUPPORT
|
||||
EnableCompilerFlag("/D_UNICODE" true true)
|
||||
EnableCompilerFlag("/DUNICODE" true true)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Remove duplicates compilation flags
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if( ${flag_var} )
|
||||
separate_arguments(${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
separate_arguments(${flag_var})
|
||||
list(REMOVE_DUPLICATES ${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
ENDFOREACH (flag_var)
|
||||
|
||||
if (MSVC AND ZSTD_USE_STATIC_RUNTIME)
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if ( ${flag_var} )
|
||||
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
STRING(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
ENDFOREACH (flag_var)
|
||||
endif ()
|
||||
|
||||
endmacro()
|
||||
ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
# Find LibLZ4
|
||||
#
|
||||
# Find LibLZ4 headers and library
|
||||
#
|
||||
# Result Variables
|
||||
#
|
||||
# LIBLZ4_FOUND - True if lz4 is found
|
||||
# LIBLZ4_INCLUDE_DIRS - lz4 headers directories
|
||||
# LIBLZ4_LIBRARIES - lz4 libraries
|
||||
# LIBLZ4_VERSION_MAJOR - The major version of lz4
|
||||
# LIBLZ4_VERSION_MINOR - The minor version of lz4
|
||||
# LIBLZ4_VERSION_RELEASE - The release version of lz4
|
||||
# LIBLZ4_VERSION_STRING - version number string (e.g. 1.8.3)
|
||||
#
|
||||
# Hints
|
||||
#
|
||||
# Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library
|
||||
|
||||
set(_LIBLZ4_ROOT_HINTS
|
||||
ENV LZ4_ROOT_DIR)
|
||||
|
||||
find_path( LIBLZ4_INCLUDE_DIR lz4.h
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
find_library( LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
|
||||
if(LIBLZ4_INCLUDE_DIR)
|
||||
file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")
|
||||
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")
|
||||
|
||||
set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
|
||||
unset(LIBLZ4_HEADER_CONTENT)
|
||||
endif()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4 REQUIRED_VARS LIBLZ4_INCLUDE_DIR
|
||||
LIBLZ4_LIBRARY
|
||||
VERSION_VAR LIBLZ4_VERSION_STRING
|
||||
FAIL_MESSAGE "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
|
||||
set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )
|
||||
@@ -1,10 +1,9 @@
|
||||
function(GetZstdLibraryVersion _header _major _minor _patch)
|
||||
function(GetZstdLibraryVersion _header _major _minor _release)
|
||||
# Read file content
|
||||
file(READ ${_header} CONTENT)
|
||||
FILE(READ ${_header} CONTENT)
|
||||
|
||||
string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}")
|
||||
set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
SET(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
SET(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
SET(${_release} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# Cmake contributions
|
||||
|
||||
Contributions to the cmake build configurations are welcome. Please
|
||||
use case sensitivity that matches modern (ie. cmake version 2.6 and above)
|
||||
conventions of using lower-case for commands, and upper-case for
|
||||
variables.
|
||||
|
||||
## How to build
|
||||
|
||||
As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
|
||||
To do this, you can create a new directory and build in it:
|
||||
```sh
|
||||
cd build/cmake
|
||||
mkdir builddir
|
||||
cd builddir
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
Then you can clean all cmake caches by simply delete the new directory:
|
||||
```sh
|
||||
rm -rf build/cmake/builddir
|
||||
```
|
||||
|
||||
And of course, you can directly build in build/cmake:
|
||||
```sh
|
||||
cd build/cmake
|
||||
cmake
|
||||
make
|
||||
```
|
||||
|
||||
To show cmake build options, you can:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -LH ..
|
||||
```
|
||||
|
||||
Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
|
||||
make
|
||||
```
|
||||
|
||||
### referring
|
||||
[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
|
||||
|
||||
## CMake Style Recommendations
|
||||
|
||||
### Indent all code correctly, i.e. the body of
|
||||
|
||||
* if/else/endif
|
||||
* foreach/endforeach
|
||||
* while/endwhile
|
||||
* macro/endmacro
|
||||
* function/endfunction
|
||||
|
||||
Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
|
||||
spaces for indenting as is used in the rest of the file. Do not use tabs.
|
||||
|
||||
### Upper/lower casing
|
||||
|
||||
Most important: use consistent upper- or lowercasing within one file !
|
||||
|
||||
In general, the all-lowercase style is preferred.
|
||||
|
||||
So, this is recommended:
|
||||
|
||||
```
|
||||
add_executable(foo foo.c)
|
||||
```
|
||||
|
||||
These forms are discouraged
|
||||
|
||||
```
|
||||
ADD_EXECUTABLE(bar bar.c)
|
||||
Add_Executable(hello hello.c)
|
||||
aDd_ExEcUtAbLe(blub blub.c)
|
||||
```
|
||||
|
||||
### End commands
|
||||
To make the code easier to read, use empty commands for endforeach(), endif(),
|
||||
endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
|
||||
|
||||
For example, do this:
|
||||
|
||||
```
|
||||
if(FOOVAR)
|
||||
some_command(...)
|
||||
else()
|
||||
another_command(...)
|
||||
endif()
|
||||
```
|
||||
|
||||
and not this:
|
||||
|
||||
```
|
||||
if(BARVAR)
|
||||
some_other_command(...)
|
||||
endif(BARVAR)
|
||||
```
|
||||
|
||||
### Other resources for best practices
|
||||
|
||||
https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules
|
||||
@@ -7,7 +7,7 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(contrib)
|
||||
PROJECT(contrib)
|
||||
|
||||
add_subdirectory(pzstd)
|
||||
add_subdirectory(gen_html)
|
||||
ADD_SUBDIRECTORY(pzstd)
|
||||
ADD_SUBDIRECTORY(gen_html)
|
||||
|
||||
@@ -7,24 +7,24 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(gen_html)
|
||||
include(GetZstdLibraryVersion)
|
||||
PROJECT(gen_html)
|
||||
INCLUDE(GetZstdLibraryVersion)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
|
||||
set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
|
||||
SET(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
|
||||
|
||||
add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp)
|
||||
ADD_EXECUTABLE(gen_html ${GENHTML_DIR}/gen_html.cpp)
|
||||
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
|
||||
set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
|
||||
add_custom_target(zstd_manual.html ALL
|
||||
SET(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
|
||||
ADD_CUSTOM_TARGET(zstd_manual.html ALL
|
||||
${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
|
||||
DEPENDS gen_html COMMENT "Update zstd manual")
|
||||
|
||||
install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
|
||||
INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "share/doc")
|
||||
|
||||
@@ -7,26 +7,26 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(pzstd)
|
||||
PROJECT(pzstd)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
|
||||
|
||||
add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
|
||||
ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
|
||||
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
|
||||
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if (CMAKE_USE_PTHREADS_INIT)
|
||||
target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
SET(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
FIND_PACKAGE(Threads REQUIRED)
|
||||
IF (CMAKE_USE_PTHREADS_INIT)
|
||||
TARGET_LINK_LIBRARIES(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
|
||||
ELSE()
|
||||
MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
ENDIF()
|
||||
|
||||
install(TARGETS pzstd RUNTIME DESTINATION "bin")
|
||||
INSTALL(TARGETS pzstd RUNTIME DESTINATION "bin")
|
||||
|
||||
+119
-112
@@ -7,51 +7,77 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(libzstd)
|
||||
PROJECT(libzstd)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
||||
option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
||||
OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||
|
||||
if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build at least one flavor of libzstd")
|
||||
endif()
|
||||
IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build at least one flavor of libstd")
|
||||
ENDIF()
|
||||
|
||||
# Define library directory, where sources and header files are located
|
||||
include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
|
||||
file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
|
||||
file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
|
||||
file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
|
||||
file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
|
||||
file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
|
||||
# Parse version
|
||||
INCLUDE(GetZstdLibraryVersion)
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
|
||||
MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
|
||||
set(Sources
|
||||
${CommonSources}
|
||||
${CompressSources}
|
||||
${DecompressSources}
|
||||
${DictBuilderSources}
|
||||
${DeprecatedSources})
|
||||
SET(Sources
|
||||
${LIBRARY_DIR}/common/entropy_common.c
|
||||
${LIBRARY_DIR}/common/fse_decompress.c
|
||||
${LIBRARY_DIR}/common/threading.c
|
||||
${LIBRARY_DIR}/common/pool.c
|
||||
${LIBRARY_DIR}/common/zstd_common.c
|
||||
${LIBRARY_DIR}/common/error_private.c
|
||||
${LIBRARY_DIR}/common/xxhash.c
|
||||
${LIBRARY_DIR}/compress/fse_compress.c
|
||||
${LIBRARY_DIR}/compress/huf_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_compress.c
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_fast.c
|
||||
${LIBRARY_DIR}/compress/zstd_double_fast.c
|
||||
${LIBRARY_DIR}/compress/zstd_lazy.c
|
||||
${LIBRARY_DIR}/compress/zstd_opt.c
|
||||
${LIBRARY_DIR}/compress/zstd_ldm.c
|
||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/cover.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_compress.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
|
||||
|
||||
file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
|
||||
file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
|
||||
file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
|
||||
file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
|
||||
file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
|
||||
|
||||
set(Headers
|
||||
SET(Headers
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${CommonHeaders}
|
||||
${CompressHeaders}
|
||||
${DecompressHeaders}
|
||||
${DictBuilderHeaders}
|
||||
${DeprecatedHeaders})
|
||||
${LIBRARY_DIR}/common/pool.h
|
||||
${LIBRARY_DIR}/common/threading.h
|
||||
${LIBRARY_DIR}/common/bitstream.h
|
||||
${LIBRARY_DIR}/common/error_private.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
${LIBRARY_DIR}/common/fse.h
|
||||
${LIBRARY_DIR}/common/huf.h
|
||||
${LIBRARY_DIR}/common/mem.h
|
||||
${LIBRARY_DIR}/common/zstd_internal.h
|
||||
${LIBRARY_DIR}/compress/zstd_compress.h
|
||||
${LIBRARY_DIR}/compress/zstd_fast.h
|
||||
${LIBRARY_DIR}/compress/zstd_double_fast.h
|
||||
${LIBRARY_DIR}/compress/zstd_lazy.h
|
||||
${LIBRARY_DIR}/compress/zstd_opt.h
|
||||
${LIBRARY_DIR}/compress/zstd_ldm.h
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
include_directories(${LIBRARY_LEGACY_DIR})
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
|
||||
|
||||
set(Sources ${Sources}
|
||||
SET(Sources ${Sources}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
||||
@@ -60,7 +86,7 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.c)
|
||||
|
||||
set(Headers ${Headers}
|
||||
SET(Headers ${Headers}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.h
|
||||
@@ -69,108 +95,89 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v05.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.h)
|
||||
endif ()
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Split project to static and shared libraries build
|
||||
set(library_targets)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
list(APPEND library_targets libzstd_shared)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_shared ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif()
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
add_library(libzstd_static STATIC ${Sources} ${Headers})
|
||||
list(APPEND library_targets libzstd_static)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_static ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
# Add specific compile definitions for MSVC project
|
||||
if (MSVC)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# With MSVC static library needs to be renamed to avoid conflict with import library
|
||||
if (MSVC)
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd_static)
|
||||
else ()
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(STATIC_LIBRARY_BASE_NAME zstd_static)
|
||||
ELSE ()
|
||||
SET(STATIC_LIBRARY_BASE_NAME zstd)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Define static and shared library names
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_target_properties(
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_shared
|
||||
PROPERTIES
|
||||
OUTPUT_NAME zstd
|
||||
VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
|
||||
SOVERSION ${zstd_VERSION_MAJOR})
|
||||
endif ()
|
||||
SOVERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE})
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_target_properties(
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_static
|
||||
PROPERTIES
|
||||
POSITION_INDEPENDENT_CODE On
|
||||
OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
|
||||
endif ()
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
if (UNIX OR MINGW)
|
||||
IF (UNIX)
|
||||
# pkg-config
|
||||
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
set(LIBDIR "${CMAKE_INSTALL_LIBDIR}")
|
||||
set(INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
set(VERSION "${zstd_VERSION}")
|
||||
add_custom_target(libzstd.pc ALL
|
||||
SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/lib")
|
||||
SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
|
||||
SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
ADD_CUSTOM_TARGET(libzstd.pc ALL
|
||||
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
|
||||
-DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
|
||||
COMMENT "Creating pkg-config file")
|
||||
|
||||
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
|
||||
endif ()
|
||||
INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "share/pkgconfig")
|
||||
ENDIF (UNIX)
|
||||
|
||||
# install target
|
||||
install(FILES
|
||||
"${LIBRARY_DIR}/zstd.h"
|
||||
"${LIBRARY_DIR}/dictBuilder/zdict.h"
|
||||
"${LIBRARY_DIR}/common/zstd_errors.h"
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
INSTALL(FILES
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
DESTINATION "include")
|
||||
|
||||
install(TARGETS ${library_targets}
|
||||
EXPORT zstdExports
|
||||
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
||||
)
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
INSTALL(TARGETS libzstd_shared RUNTIME DESTINATION "bin" LIBRARY DESTINATION "lib" ARCHIVE DESTINATION "lib")
|
||||
ENDIF()
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "lib")
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
# uninstall target
|
||||
if (NOT TARGET uninstall)
|
||||
configure_file(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
IMMEDIATE @ONLY)
|
||||
CONFIGURE_FILE(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
IMMEDIATE @ONLY)
|
||||
|
||||
add_custom_target(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
endif ()
|
||||
ADD_CUSTOM_TARGET(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
endif()
|
||||
endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
|
||||
file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
|
||||
string(REGEX REPLACE "\n" ";" files "${files}")
|
||||
@@ -15,8 +15,8 @@ foreach(file ${files})
|
||||
)
|
||||
if(NOT "${rm_retval}" STREQUAL 0)
|
||||
message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
|
||||
endif()
|
||||
else()
|
||||
endif(NOT "${rm_retval}" STREQUAL 0)
|
||||
else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
|
||||
endif()
|
||||
endforeach()
|
||||
endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
endforeach(file)
|
||||
|
||||
@@ -1 +1 @@
|
||||
configure_file("${IN}" "${OUT}" @ONLY)
|
||||
CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)
|
||||
|
||||
@@ -7,129 +7,91 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(programs)
|
||||
PROJECT(programs)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
endif ()
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (ZSTD_PROGRAMS_LINK_SHARED)
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
|
||||
else ()
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
endif ()
|
||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
||||
IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
TARGET_LINK_LIBRARIES(zstd rt)
|
||||
ENDIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
|
||||
|
||||
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
target_link_libraries(zstd rt)
|
||||
endif ()
|
||||
install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
IF (UNIX)
|
||||
ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
|
||||
ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
|
||||
INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
|
||||
|
||||
if (UNIX)
|
||||
add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
|
||||
add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
|
||||
add_custom_target(zstd.1 ALL
|
||||
ADD_CUSTOM_TARGET(zstd.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
|
||||
COMMENT "Copying manpage zstd.1")
|
||||
add_custom_target(zstdgrep.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
|
||||
COMMENT "Copying manpage zstdgrep.1")
|
||||
add_custom_target(zstdless.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
|
||||
COMMENT "Copying manpage zstdless.1")
|
||||
add_custom_target(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
|
||||
add_custom_target(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
|
||||
ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
|
||||
ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "share/man/man1")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "share/man/man1")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "share/man/man1")
|
||||
|
||||
# Define MAN_INSTALL_DIR if necessary
|
||||
if (MAN_INSTALL_DIR)
|
||||
else ()
|
||||
set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1)
|
||||
endif ()
|
||||
ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
|
||||
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
||||
SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
ENDIF (UNIX)
|
||||
|
||||
install(FILES
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/unzstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
|
||||
DESTINATION "${MAN_INSTALL_DIR}")
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
|
||||
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
|
||||
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
endif ()
|
||||
SET(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
FIND_PACKAGE(Threads REQUIRED)
|
||||
IF (CMAKE_USE_PTHREADS_INIT)
|
||||
TARGET_LINK_LIBRARIES(zstd ${CMAKE_THREAD_LIBS_INIT})
|
||||
ELSE()
|
||||
MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
ENDIF()
|
||||
|
||||
# Add multi-threading support definitions
|
||||
ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
|
||||
ENDIF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
|
||||
OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
|
||||
|
||||
if (UNIX)
|
||||
target_link_libraries(zstd ${THREADS_LIBS})
|
||||
IF (ZSTD_ZLIB_SUPPORT)
|
||||
FIND_PACKAGE(ZLIB REQUIRED)
|
||||
|
||||
add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (ZLIB_FOUND)
|
||||
INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
|
||||
TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
|
||||
ELSE ()
|
||||
MESSAGE(SEND_ERROR "zlib library is missing")
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
option(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
|
||||
option(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
|
||||
option(ZSTD_LZ4_SUPPORT "LZ4 SUPPORT" OFF)
|
||||
IF (ZSTD_LZMA_SUPPORT)
|
||||
FIND_PACKAGE(LibLZMA REQUIRED)
|
||||
|
||||
# Add gzip support
|
||||
if (ZSTD_ZLIB_SUPPORT)
|
||||
find_package(ZLIB REQUIRED)
|
||||
|
||||
if (ZLIB_FOUND)
|
||||
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${ZLIB_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "zlib library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Add lzma support
|
||||
if (ZSTD_LZMA_SUPPORT)
|
||||
find_package(LibLZMA REQUIRED)
|
||||
|
||||
if (LIBLZMA_FOUND)
|
||||
include_directories(${LIBLZMA_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZMA_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lzma library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Add lz4 support
|
||||
if (ZSTD_LZ4_SUPPORT)
|
||||
find_package(LibLZ4 REQUIRED)
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
include_directories(${LIBLZ4_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZ4_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZ4COMPRESS;ZSTD_LZ4DECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lz4 library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (LIBLZMA_FOUND)
|
||||
INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
|
||||
TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
|
||||
ELSE ()
|
||||
MESSAGE(SEND_ERROR "lzma library is missing")
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ################################################################
|
||||
# zstd - Makefile
|
||||
# Copyright (C) Yann Collet 2014-present
|
||||
# Copyright (C) Yann Collet 2014-2016
|
||||
# All rights reserved.
|
||||
#
|
||||
# BSD license
|
||||
@@ -30,77 +30,26 @@
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
project(tests)
|
||||
PROJECT(tests)
|
||||
|
||||
# name: Cache variable name. The value is expected to be a semicolon-separated
|
||||
# list of command line flags
|
||||
# default_value: Value to initialize the option with. Can be space separated.
|
||||
function(AddTestFlagsOption name default_value doc)
|
||||
string(STRIP "${default_value}" default_value)
|
||||
string(REGEX REPLACE " +" ";" default_value "${default_value}")
|
||||
set(${name} ${default_value} CACHE STRING "${doc}")
|
||||
mark_as_advanced(${name})
|
||||
endfunction()
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
|
||||
include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
|
||||
INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
target_link_libraries(datagen libzstd_static)
|
||||
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c)
|
||||
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
||||
|
||||
#
|
||||
# fullbench
|
||||
#
|
||||
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
|
||||
target_link_libraries(fullbench libzstd_static)
|
||||
add_test(NAME fullbench COMMAND fullbench)
|
||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
|
||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||
|
||||
#
|
||||
# fuzzer
|
||||
#
|
||||
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
|
||||
target_link_libraries(fuzzer libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
|
||||
add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
|
||||
# Disable the timeout since the run time is too long for the default timeout of
|
||||
# 1500 seconds and varies considerably between low-end and high-end CPUs.
|
||||
# set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
|
||||
IF (UNIX)
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
|
||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||
|
||||
#
|
||||
# zstreamtest
|
||||
#
|
||||
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
|
||||
target_link_libraries(zstreamtest libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
|
||||
add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
|
||||
|
||||
#
|
||||
# playTests.sh
|
||||
#
|
||||
AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "--test-large-data"
|
||||
"Semicolon-separated list of flags to pass to the playTests.sh test")
|
||||
add_test(NAME playTests COMMAND sh -c "${TESTS_DIR}/playTests.sh" ${ZSTD_PLAYTESTS_FLAGS})
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
|
||||
"ZSTD_BIN=$<TARGET_FILE:zstd>"
|
||||
"DATAGEN_BIN=$<TARGET_FILE:datagen>"
|
||||
)
|
||||
else()
|
||||
message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
|
||||
set_tests_properties(playTests PROPERTIES DISABLED YES)
|
||||
endif()
|
||||
|
||||
# Label the "Medium" set of tests (see TESTING.md)
|
||||
set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
|
||||
|
||||
if (UNIX)
|
||||
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
|
||||
target_link_libraries(paramgrill libzstd_static m) #m is math library
|
||||
endif ()
|
||||
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
TARGET_LINK_LIBRARIES(datagen libzstd_static)
|
||||
ENDIF (UNIX)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import re
|
||||
|
||||
|
||||
def find_version_tuple(filepath):
|
||||
version_file_data = None
|
||||
with open(filepath) as fd:
|
||||
version_file_data = fd.read()
|
||||
|
||||
patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
|
||||
"""
|
||||
regex = re.compile(patterns, re.MULTILINE)
|
||||
version_match = regex.search(version_file_data)
|
||||
if version_match:
|
||||
return version_match.groups()
|
||||
raise Exception("Unable to find version string")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
|
||||
parser.add_argument('file', help='path to lib/zstd.h')
|
||||
args = parser.parse_args()
|
||||
version_tuple = find_version_tuple(args.file)
|
||||
print('.'.join(version_tuple))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
# This file should be synced with https://github.com/lzutao/meson-symlink
|
||||
|
||||
import os
|
||||
import pathlib # since Python 3.4
|
||||
|
||||
|
||||
def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
|
||||
if not install_dir.exists():
|
||||
install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
|
||||
if not install_dir.is_dir():
|
||||
raise NotADirectoryError(install_dir)
|
||||
|
||||
new_dst = install_dir.joinpath(dst)
|
||||
if new_dst.is_symlink() and os.readlink(new_dst) == src:
|
||||
print('File exists: {!r} -> {!r}'.format(new_dst, src))
|
||||
return
|
||||
print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
|
||||
new_dst.symlink_to(src, target_is_directory=dst_is_dir)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Install a symlink',
|
||||
usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
|
||||
'example:\n'
|
||||
' {0} dash sh /bin'.format(pathlib.Path(__file__).name))
|
||||
parser.add_argument('source', help='target to link')
|
||||
parser.add_argument('dest', help='link name')
|
||||
parser.add_argument('install_dir', help='installation directory')
|
||||
parser.add_argument('-d', '--isdir',
|
||||
action='store_true',
|
||||
help='dest is a directory')
|
||||
parser.add_argument('-m', '--mode',
|
||||
help='directory mode on creating if not exist',
|
||||
default='0o755')
|
||||
args = parser.parse_args()
|
||||
|
||||
dir_mode = int(args.mode, 8)
|
||||
|
||||
meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
|
||||
install_dir = pathlib.Path(meson_destdir, args.install_dir)
|
||||
install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
Meson build system for zstandard
|
||||
================================
|
||||
|
||||
Meson is a build system designed to optimize programmer productivity.
|
||||
It aims to do this by providing simple, out-of-the-box support for
|
||||
modern software development tools and practices, such as unit tests,
|
||||
coverage reports, Valgrind, CCache and the like.
|
||||
|
||||
This Meson build system is provided with no guarantee and maintained
|
||||
by Dima Krasner \<dima@dimakrasner.com\>.
|
||||
|
||||
It outputs one `libzstd`, either shared or static, depending on
|
||||
`default_library` option.
|
||||
|
||||
## How to build
|
||||
|
||||
`cd` to this meson directory (`build/meson`)
|
||||
|
||||
```sh
|
||||
meson setup -Dbin_programs=true -Dbin_contrib=true builddir
|
||||
cd builddir
|
||||
ninja # to build
|
||||
ninja install # to install
|
||||
```
|
||||
|
||||
You might want to install it in staging directory:
|
||||
|
||||
```sh
|
||||
DESTDIR=./staging ninja install
|
||||
```
|
||||
|
||||
To configure build options, use:
|
||||
|
||||
```sh
|
||||
meson configure
|
||||
```
|
||||
|
||||
See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).
|
||||
@@ -1,30 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html'))
|
||||
|
||||
gen_html = executable('gen_html',
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
|
||||
include_directories: gen_html_includes,
|
||||
native: true,
|
||||
install: false)
|
||||
|
||||
# Update zstd manual
|
||||
zstd_manual_html = custom_target('zstd_manual.html',
|
||||
output : 'zstd_manual.html',
|
||||
command : [gen_html,
|
||||
zstd_version,
|
||||
join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
|
||||
'@OUTPUT@'],
|
||||
install : false)
|
||||
@@ -1,12 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
subdir('pzstd')
|
||||
subdir('gen_html')
|
||||
@@ -1,24 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd'))
|
||||
pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
|
||||
pzstd = executable('pzstd',
|
||||
pzstd_sources,
|
||||
cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
|
||||
include_directories: pzstd_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: true)
|
||||
@@ -1,131 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'lib/compress'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated'))]
|
||||
|
||||
libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/hist.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
|
||||
|
||||
# Explicit define legacy support
|
||||
add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
|
||||
language: 'c')
|
||||
|
||||
if legacy_level == 0
|
||||
message('Legacy support: DISABLED')
|
||||
else
|
||||
# See ZSTD_LEGACY_SUPPORT of lib/README.md
|
||||
message('Enable legacy support back to version 0.@0@'.format(legacy_level))
|
||||
|
||||
libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
|
||||
foreach i : [1, 2, 3, 4, 5, 6, 7]
|
||||
if legacy_level <= i
|
||||
libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
|
||||
endif
|
||||
endforeach
|
||||
endif
|
||||
|
||||
libzstd_deps = []
|
||||
if use_multi_thread
|
||||
message('Enable multi-threading support')
|
||||
add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
|
||||
libzstd_deps = [ thread_dep ]
|
||||
endif
|
||||
|
||||
libzstd_c_args = []
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
libzstd_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
|
||||
libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
|
||||
'-DZSTD_HEAPMODE=0',
|
||||
'-D_CONSOLE',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
else
|
||||
libzstd_c_args += ['-DZSTD_HEAPMODE=0',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
endif
|
||||
endif
|
||||
|
||||
mingw_ansi_stdio_flags = []
|
||||
if host_machine_os == os_windows and cc_id == compiler_gcc
|
||||
mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
|
||||
endif
|
||||
libzstd_c_args += mingw_ansi_stdio_flags
|
||||
|
||||
libzstd_debug_cflags = []
|
||||
if use_debug
|
||||
libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
|
||||
if cc_id == compiler_gcc or cc_id == compiler_clang
|
||||
libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
|
||||
'-Wdeclaration-after-statement', '-Wstrict-prototypes',
|
||||
'-Wundef', '-Wpointer-arith', '-Wvla',
|
||||
'-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
|
||||
'-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
|
||||
endif
|
||||
endif
|
||||
libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
|
||||
|
||||
libzstd = library('zstd',
|
||||
libzstd_sources,
|
||||
include_directories: libzstd_includes,
|
||||
c_args: libzstd_c_args,
|
||||
dependencies: libzstd_deps,
|
||||
install: true,
|
||||
version: zstd_libversion)
|
||||
|
||||
libzstd_dep = declare_dependency(link_with: libzstd,
|
||||
include_directories: libzstd_includes)
|
||||
|
||||
pkgconfig.generate(libzstd,
|
||||
name: 'libzstd',
|
||||
filebase: 'libzstd',
|
||||
description: 'fast lossless compression algorithm library',
|
||||
version: zstd_libversion,
|
||||
url: 'http://www.zstd.net/')
|
||||
|
||||
install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))
|
||||
@@ -1,147 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
project('zstd',
|
||||
['c', 'cpp'],
|
||||
license: ['BSD', 'GPLv2'],
|
||||
default_options : [
|
||||
'c_std=gnu99',
|
||||
'cpp_std=c++11',
|
||||
'buildtype=release'
|
||||
],
|
||||
version: 'DUMMY',
|
||||
meson_version: '>=0.47.0')
|
||||
|
||||
cc = meson.get_compiler('c')
|
||||
cxx = meson.get_compiler('cpp')
|
||||
pkgconfig = import('pkgconfig')
|
||||
python3 = import('python').find_installation()
|
||||
windows_mod = import('windows')
|
||||
|
||||
host_machine_os = host_machine.system()
|
||||
os_windows = 'windows'
|
||||
os_linux = 'linux'
|
||||
os_darwin = 'darwin'
|
||||
os_freebsd = 'freebsd'
|
||||
os_sun = 'sunos'
|
||||
|
||||
cc_id = cc.get_id()
|
||||
compiler_gcc = 'gcc'
|
||||
compiler_clang = 'clang'
|
||||
compiler_msvc = 'msvc'
|
||||
|
||||
zstd_version = meson.project_version()
|
||||
|
||||
zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
|
||||
GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py')
|
||||
r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file)
|
||||
if r.returncode() == 0
|
||||
zstd_version = r.stdout().strip()
|
||||
message('Project version is now: @0@'.format(zstd_version))
|
||||
else
|
||||
error('Cannot find project version in @0@'.format(zstd_h_file))
|
||||
endif
|
||||
|
||||
zstd_libversion = zstd_version
|
||||
|
||||
# =============================================================================
|
||||
# Installation directories
|
||||
# =============================================================================
|
||||
|
||||
zstd_prefix = get_option('prefix')
|
||||
zstd_bindir = get_option('bindir')
|
||||
zstd_datadir = get_option('datadir')
|
||||
zstd_mandir = get_option('mandir')
|
||||
zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
|
||||
|
||||
# =============================================================================
|
||||
# Project options
|
||||
# =============================================================================
|
||||
|
||||
# Built-in options
|
||||
use_debug = get_option('debug')
|
||||
buildtype = get_option('buildtype')
|
||||
default_library_type = get_option('default_library')
|
||||
|
||||
# Custom options
|
||||
debug_level = get_option('debug_level')
|
||||
legacy_level = get_option('legacy_level')
|
||||
use_backtrace = get_option('backtrace')
|
||||
use_static_runtime = get_option('static_runtime')
|
||||
|
||||
bin_programs = get_option('bin_programs')
|
||||
bin_contrib = get_option('bin_contrib')
|
||||
bin_tests = get_option('bin_tests')
|
||||
|
||||
feature_multi_thread = get_option('multi_thread')
|
||||
feature_zlib = get_option('zlib')
|
||||
feature_lzma = get_option('lzma')
|
||||
feature_lz4 = get_option('lz4')
|
||||
|
||||
# =============================================================================
|
||||
# Dependencies
|
||||
# =============================================================================
|
||||
|
||||
libm_dep = cc.find_library('m', required: bin_tests)
|
||||
thread_dep = dependency('threads', required: feature_multi_thread)
|
||||
use_multi_thread = thread_dep.found()
|
||||
# Arguments in dependency should be equivalent to those passed to pkg-config
|
||||
zlib_dep = dependency('zlib', required: feature_zlib)
|
||||
use_zlib = zlib_dep.found()
|
||||
lzma_dep = dependency('liblzma', required: feature_lzma)
|
||||
use_lzma = lzma_dep.found()
|
||||
lz4_dep = dependency('liblz4', required: feature_lz4)
|
||||
use_lz4 = lz4_dep.found()
|
||||
|
||||
# =============================================================================
|
||||
# Compiler flags
|
||||
# =============================================================================
|
||||
|
||||
add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
|
||||
|
||||
if [compiler_gcc, compiler_clang].contains(cc_id)
|
||||
common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
|
||||
if cc_id == compiler_clang
|
||||
# Should use Meson's own --werror build option
|
||||
#common_warning_flags += '-Werror'
|
||||
common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
|
||||
endif
|
||||
cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
|
||||
cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
|
||||
add_project_arguments(cc_compile_flags, language : 'c')
|
||||
add_project_arguments(cxx_compile_flags, language : 'cpp')
|
||||
elif cc_id == compiler_msvc
|
||||
msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
|
||||
if use_multi_thread
|
||||
msvc_compile_flags += '/MP'
|
||||
endif
|
||||
if use_static_runtime
|
||||
msvc_compile_flags += '/MT'
|
||||
endif
|
||||
add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
|
||||
endif
|
||||
|
||||
# =============================================================================
|
||||
# Subdirs
|
||||
# =============================================================================
|
||||
|
||||
subdir('lib')
|
||||
|
||||
if bin_programs
|
||||
subdir('programs')
|
||||
endif
|
||||
|
||||
if bin_tests
|
||||
subdir('tests')
|
||||
endif
|
||||
|
||||
if bin_contrib
|
||||
subdir('contrib')
|
||||
endif
|
||||
@@ -1,36 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
|
||||
|
||||
option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
|
||||
description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
|
||||
option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
|
||||
description: 'Enable run-time debug. See lib/common/debug.h')
|
||||
option('backtrace', type: 'boolean', value: false,
|
||||
description: 'Display a stack backtrace when execution generates a runtime exception')
|
||||
option('static_runtime', type: 'boolean', value: false,
|
||||
description: 'Link to static run-time libraries on MSVC')
|
||||
|
||||
option('bin_programs', type: 'boolean', value: true,
|
||||
description: 'Enable programs build')
|
||||
option('bin_tests', type: 'boolean', value: false,
|
||||
description: 'Enable tests build')
|
||||
option('bin_contrib', type: 'boolean', value: false,
|
||||
description: 'Enable contrib build')
|
||||
|
||||
option('multi_thread', type: 'feature', value: 'enabled',
|
||||
description: 'Enable multi-threading when pthread is detected')
|
||||
option('zlib', type: 'feature', value: 'auto',
|
||||
description: 'Enable zlib support')
|
||||
option('lzma', type: 'feature', value: 'auto',
|
||||
description: 'Enable lzma support')
|
||||
option('lz4', type: 'feature', value: 'auto',
|
||||
description: 'Enable lz4 support')
|
||||
@@ -1,104 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/dibio.c')]
|
||||
|
||||
zstd_c_args = libzstd_debug_cflags
|
||||
if use_multi_thread
|
||||
zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
|
||||
endif
|
||||
|
||||
zstd_deps = [ libzstd_dep ]
|
||||
if use_zlib
|
||||
zstd_deps += [ zlib_dep ]
|
||||
zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lzma
|
||||
zstd_deps += [ lzma_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lz4
|
||||
zstd_deps += [ lz4_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
|
||||
endif
|
||||
|
||||
export_dynamic_on_windows = false
|
||||
# explicit backtrace enable/disable for Linux & Darwin
|
||||
if not use_backtrace
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=0'
|
||||
elif use_debug and host_machine_os == os_windows # MinGW target
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=1'
|
||||
export_dynamic_on_windows = true
|
||||
endif
|
||||
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
zstd_programs_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
|
||||
endif
|
||||
endif
|
||||
|
||||
zstd = executable('zstd',
|
||||
zstd_programs_sources,
|
||||
c_args: zstd_c_args,
|
||||
dependencies: zstd_deps,
|
||||
export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
|
||||
install: true)
|
||||
|
||||
zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c')]
|
||||
|
||||
# Minimal target, with only zstd compression and decompression.
|
||||
# No bench. No legacy.
|
||||
executable('zstd-frugal',
|
||||
zstd_frugal_sources,
|
||||
dependencies: libzstd_dep,
|
||||
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
|
||||
install: true)
|
||||
|
||||
install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless'),
|
||||
install_dir: zstd_bindir)
|
||||
|
||||
# =============================================================================
|
||||
# Programs and manpages installing
|
||||
# =============================================================================
|
||||
|
||||
install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless.1'))
|
||||
|
||||
InstallSymlink_py = '../InstallSymlink.py'
|
||||
zstd_man1_dir = join_paths(zstd_mandir, 'man1')
|
||||
bin_EXT = host_machine_os == os_windows ? '.exe' : ''
|
||||
man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
|
||||
|
||||
foreach f : ['zstdcat', 'unzstd']
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, f + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
|
||||
endforeach
|
||||
|
||||
if use_multi_thread
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, 'zstdmt' + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
|
||||
endif
|
||||
@@ -1,206 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
|
||||
os_freebsd, 'netbsd', 'dragonfly', os_sun]
|
||||
|
||||
# =============================================================================
|
||||
# Test flags
|
||||
# =============================================================================
|
||||
|
||||
FUZZER_FLAGS = ['--no-big-tests']
|
||||
FUZZERTEST = '-T200s'
|
||||
ZSTREAM_TESTTIME = '-T90s'
|
||||
DECODECORPUS_TESTTIME = '-T30'
|
||||
ZSTDRTTEST = ['--test-large-data']
|
||||
|
||||
# =============================================================================
|
||||
# Executables
|
||||
# =============================================================================
|
||||
|
||||
test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
|
||||
|
||||
datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/datagencli.c')]
|
||||
datagen = executable('datagen',
|
||||
datagen_sources,
|
||||
c_args: [ '-DNDEBUG' ],
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fullbench.c')]
|
||||
fullbench = executable('fullbench',
|
||||
fullbench_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fuzzer.c')]
|
||||
fuzzer = executable('fuzzer',
|
||||
fuzzer_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/seqgen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
|
||||
zstreamtest = executable('zstreamtest',
|
||||
zstreamtest_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'tests/paramgrill.c')]
|
||||
paramgrill = executable('paramgrill',
|
||||
paramgrill_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
|
||||
roundTripCrash = executable('roundTripCrash',
|
||||
roundTripCrash_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
|
||||
longmatch = executable('longmatch',
|
||||
longmatch_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
|
||||
invalidDictionaries = executable('invalidDictionaries',
|
||||
invalidDictionaries_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
|
||||
legacy = executable('legacy',
|
||||
legacy_sources,
|
||||
# Use -Dlegacy_level build option to control it
|
||||
#c_args: '-DZSTD_LEGACY_SUPPORT=4',
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
endif
|
||||
|
||||
decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
|
||||
decodecorpus = executable('decodecorpus',
|
||||
decodecorpus_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/poolTests.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c')]
|
||||
poolTests = executable('poolTests',
|
||||
poolTests_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: false)
|
||||
|
||||
checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
|
||||
checkTag = executable('checkTag',
|
||||
checkTag_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
# =============================================================================
|
||||
# Tests (Use "meson test --list" to list all tests)
|
||||
# =============================================================================
|
||||
|
||||
if tests_supported_oses.contains(host_machine_os)
|
||||
valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
|
||||
valgrindTest_py = files('valgrindTest.py')
|
||||
test('valgrindTest',
|
||||
valgrindTest_py,
|
||||
args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
|
||||
depends: [zstd, datagen, fuzzer, fullbench],
|
||||
timeout: 600) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
if host_machine_os != os_windows
|
||||
playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
|
||||
test('test-zstd',
|
||||
playTests_sh,
|
||||
args: ZSTDRTTEST,
|
||||
env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
|
||||
depends: [datagen],
|
||||
workdir: meson.current_build_dir(),
|
||||
timeout: 2800) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
test('test-fullbench-1',
|
||||
fullbench,
|
||||
args: ['-i1'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
test('test-fullbench-2',
|
||||
fullbench,
|
||||
args: ['-i1', '-P0'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
|
||||
if use_zlib
|
||||
test('test-fuzzer',
|
||||
fuzzer,
|
||||
args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
|
||||
timeout: 480)
|
||||
endif
|
||||
|
||||
test('test-zstream-1',
|
||||
zstreamtest,
|
||||
args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 240)
|
||||
test('test-zstream-2',
|
||||
zstreamtest,
|
||||
args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-zstream-3',
|
||||
zstreamtest,
|
||||
args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-longmatch', longmatch, timeout: 36)
|
||||
test('test-invalidDictionaries', invalidDictionaries) # should be fast
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
test('test-legacy', legacy) # should be fast
|
||||
endif
|
||||
test('test-decodecorpus',
|
||||
decodecorpus,
|
||||
args: ['-t', DECODECORPUS_TESTTIME],
|
||||
timeout: 60)
|
||||
test('test-poolTests', poolTests) # should be fast
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
|
||||
VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
|
||||
|
||||
print('\n ---- valgrind tests : memory analyzer ----')
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
|
||||
|
||||
if subprocess.call([*VALGRIND_ARGS, zstd],
|
||||
stdout=subprocess.DEVNULL) == 0:
|
||||
raise subprocess.CalledProcessError('zstd without argument should have failed')
|
||||
|
||||
with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmp_fd:
|
||||
with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
|
||||
stdin=p1.stdout) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
|
||||
with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
|
||||
subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
|
||||
parser.add_argument('valgrind', help='valgrind path')
|
||||
parser.add_argument('zstd', help='zstd path')
|
||||
parser.add_argument('datagen', help='datagen path')
|
||||
parser.add_argument('fuzzer', help='fuzzer path')
|
||||
parser.add_argument('fullbench', help='fullbench path')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
valgrind = args.valgrind
|
||||
zstd = args.zstd
|
||||
datagen = args.datagen
|
||||
fuzzer = args.fuzzer
|
||||
fullbench = args.fullbench
|
||||
|
||||
valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
dependencies:
|
||||
override:
|
||||
- sudo dpkg --add-architecture i386
|
||||
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
|
||||
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||
- sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev
|
||||
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
|
||||
|
||||
test:
|
||||
override:
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean && make -C lib libzstd-nomt && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gcc6build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make m32build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here
|
||||
:
|
||||
parallel: true
|
||||
|
||||
post:
|
||||
- echo Circle CI tests finished
|
||||
|
||||
# Longer tests
|
||||
#- make -C tests test-zstd-nolegacy && make clean
|
||||
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
||||
#- make zlibwrapper && make clean
|
||||
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
||||
#- make uasan && make clean
|
||||
#- make asan32 && make clean
|
||||
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
||||
# Valgrind tests
|
||||
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
||||
#- make -C tests valgrindTest && make clean
|
||||
# ARM, AArch64, PowerPC, PowerPC64 tests
|
||||
#- make ppctest && make clean
|
||||
#- make ppc64test && make clean
|
||||
#- make armtest && make clean
|
||||
#- make aarch64test && make clean
|
||||
@@ -0,0 +1,76 @@
|
||||
|
||||
ZSTDDIR = ../../lib
|
||||
PRGDIR = ../../programs
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
|
||||
MULTITHREAD_LDFLAGS = -pthread
|
||||
DEBUGFLAGS= -g -DZSTD_DEBUG=1
|
||||
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
|
||||
|
||||
all: adapt datagen
|
||||
|
||||
adapt: $(ZSTD_FILES) adapt.c
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
adapt-debug: $(ZSTD_FILES) adapt.c
|
||||
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
test-adapt-correctness: datagen adapt
|
||||
@./test-correctness.sh
|
||||
@echo "test correctness complete"
|
||||
|
||||
test-adapt-performance: datagen adapt
|
||||
@./test-performance.sh
|
||||
@echo "test performance complete"
|
||||
|
||||
clean:
|
||||
@$(RM) -f adapt datagen
|
||||
@$(RM) -rf *.dSYM
|
||||
@$(RM) -f tmp*
|
||||
@$(RM) -f tests/*.zst
|
||||
@$(RM) -f tests/tmp*
|
||||
@echo "finished cleaning"
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
PREFIX ?= /usr/local
|
||||
DESTDIR ?=
|
||||
BINDIR ?= $(PREFIX)/bin
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL) -m 755
|
||||
|
||||
install: adapt
|
||||
@echo Installing binaries
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
|
||||
@$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive installation completed
|
||||
|
||||
uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive programs successfully uninstalled
|
||||
endif
|
||||
@@ -0,0 +1,91 @@
|
||||
### Summary
|
||||
|
||||
`adapt` is a new compression tool targeted at optimizing performance across network connections and pipelines. The tool is aimed at sensing network speeds and adapting compression level based on network or pipe speeds.
|
||||
In situations where the compression level does not appropriately match the network/pipe speed, compression may be bottlenecking the entire pipeline or the files may not be compressed as much as they potentially could be, therefore losing efficiency. It also becomes quite impractical to manually measure and set an optimalcompression level (which could potentially change over time).
|
||||
|
||||
### Using `adapt`
|
||||
|
||||
In order to build and use the tool, you can simply run `make adapt` in the `adaptive-compression` directory under `contrib`. This will generate an executable available for use. Another possible method of installation is running `make install`, which will create and install the binary as the command `zstd-adaptive`.
|
||||
|
||||
Similar to many other compression utilities, `zstd-adaptive` can be invoked by using the following format:
|
||||
|
||||
`zstd-adaptive [options] [file(s)]`
|
||||
|
||||
Supported options for the above format are described below.
|
||||
|
||||
`zstd-adaptive` also supports reading from `stdin` and writing to `stdout`, which is potentially more useful. By default, if no files are given, `zstd-adaptive` reads from and writes to standard I/O. Therefore, you can simply insert it within a pipeline like so:
|
||||
|
||||
`cat FILE | zstd-adaptive | ssh "cat - > tmp.zst"`
|
||||
|
||||
If a file is provided, it is also possible to force writing to stdout using the `-c` flag like so:
|
||||
|
||||
`zstd-adaptive -c FILE | ssh "cat - > tmp.zst"`
|
||||
|
||||
Several options described below can be used to control the behavior of `zstd-adaptive`. More specifically, using the `-l#` and `-u#` flags will will set upper and lower bounds so that the compression level will always be within that range. The `-i#` flag can also be used to change the initial compression level. If an initial compression level is not provided, the initial compression level will be chosen such that it is within the appropriate range (it becomes equal to the lower bound).
|
||||
|
||||
### Options
|
||||
`-oFILE` : write output to `FILE`
|
||||
|
||||
`-i#` : provide initial compression level (must within the appropriate bounds)
|
||||
|
||||
`-h` : display help/information
|
||||
|
||||
`-f` : force the compression level to stay constant
|
||||
|
||||
`-c` : force write to `stdout`
|
||||
|
||||
`-p` : hide progress bar
|
||||
|
||||
`-q` : quiet mode -- do not show progress bar or other information
|
||||
|
||||
`-l#` : set a lower bound on the compression level (default is 1)
|
||||
|
||||
`-u#` : set an upper bound on the compression level (default is 22)
|
||||
### Benchmarking / Test results
|
||||
#### Artificial Tests
|
||||
These artificial tests were run by using the `pv` command line utility in order to limit pipe speeds (25 MB/s read and 5 MB/s write limits were chosen to mimic severe throughput constraints). A 40 GB backup file was sent through a pipeline, compressed, and written out to a file. Compression time, size, and ratio were computed. Data for `zstd -15` was excluded from these tests because the test runs quite long.
|
||||
|
||||
<table>
|
||||
<tr><th> 25 MB/s read limit </th></tr>
|
||||
<tr><td>
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.108 | 20.718 GB | 29m 48.530s |
|
||||
| zstd-adaptive | 2.230 | 19.581 GB | 29m 48.798s |
|
||||
|
||||
</td><tr>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<tr><th> 5 MB/s write limit </th></tr>
|
||||
<tr><td>
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.108 | 20.718 GB | 1h 10m 43.076s |
|
||||
| zstd-adaptive | 2.249 | 19.412 GB | 1h 06m 15.577s |
|
||||
|
||||
</td></tr>
|
||||
</table>
|
||||
|
||||
The commands used for this test generally followed the form:
|
||||
|
||||
`cat FILE | pv -L 25m -q | COMPRESSION | pv -q > tmp.zst # impose 25 MB/s read limit`
|
||||
|
||||
`cat FILE | pv -q | COMPRESSION | pv -L 5m -q > tmp.zst # impose 5 MB/s write limit`
|
||||
|
||||
#### SSH Tests
|
||||
|
||||
The following tests were performed by piping a relatively large backup file (approximately 80 GB) through compression and over SSH to be stored on a server. The test data includes statistics for time and compressed size on `zstd` at several compression levels, as well as `zstd-adaptive`. The data highlights the potential advantages that `zstd-adaptive` has over using a low static compression level and the negative imapcts that using an excessively high static compression level can have on
|
||||
pipe throughput.
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.212 | 32.426 GB | 1h 17m 59.756s |
|
||||
| zstd -15 | 2.374 | 30.213 GB | 2h 56m 59.441s |
|
||||
| zstd-adaptive | 2.315 | 30.993 GB | 1h 18m 52.860s |
|
||||
|
||||
The commands used for this test generally followed the form:
|
||||
|
||||
`cat FILE | COMPRESSION | ssh dev "cat - > tmp.zst"`
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*/
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Dependencies
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options */
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define SIZE_DEFAULT ((64 KB) + 1)
|
||||
#define SEED_DEFAULT 0
|
||||
#define COMPRESSIBILITY_DEFAULT 50
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n",
|
||||
COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
|
||||
double litProba = 0.0;
|
||||
U64 size = SIZE_DEFAULT;
|
||||
U32 seed = SEED_DEFAULT;
|
||||
const char* const programName = argv[0];
|
||||
|
||||
int argNb;
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (*argument=='-') {
|
||||
argument++;
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
size *= 10, size += *argument++ - '0';
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
seed *= 10, seed += *argument++ - '0';
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
probaU32 = 0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
probaU32 *= 10, probaU32 += *argument++ - '0';
|
||||
if (probaU32>100) probaU32 = 100;
|
||||
break;
|
||||
case 'L': /* hidden argument : Literal distribution probability */
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
litProba *= 10, litProba += *argument++ - '0';
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default:
|
||||
return usage(programName);
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
DISPLAYLEVEL(4, "Compressible data Generator \n");
|
||||
if (probaU32!=COMPRESSIBILITY_DEFAULT)
|
||||
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
|
||||
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
echo "correctness tests -- general"
|
||||
./datagen -s1 -g1GB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s2 -g500MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s3 -g250MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s4 -g125MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s5 -g50MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s6 -g25MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s7 -g10MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s8 -g5MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s9 -g500KB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- streaming"
|
||||
./datagen -s10 -g1GB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s11 -g100MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s12 -g10MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s13 -g1MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s14 -g100KB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s15 -g10KB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- read limit"
|
||||
./datagen -s16 -g1GB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s17 -g100MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s18 -g10MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s19 -g1MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s20 -g100KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s21 -g10KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- write limit"
|
||||
./datagen -s22 -g1GB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s23 -g100MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s24 -g10MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s25 -g1MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s26 -g100KB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s27 -g10KB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- read and write limits"
|
||||
./datagen -s28 -g1GB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s29 -g100MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s30 -g10MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s31 -g1MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s32 -g100KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s33 -g10KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- forced compression level"
|
||||
./datagen -s34 -g1GB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s35 -g100MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s36 -g10MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s37 -g1MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s38 -g100KB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s39 -g10KB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- window size test"
|
||||
./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
|
||||
zstd -d tmp.zst
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- testing bounds"
|
||||
./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
|
||||
rm tmp*
|
||||
make clean
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
echo "testing time -- no limits set"
|
||||
./datagen -s1 -g1GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
./datagen -s2 -g2GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
./datagen -s3 -g4GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ntesting compression ratio -- no limits set"
|
||||
./datagen -s4 -g1GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s5 -g2GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s6 -g4GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
echo e "\ntesting performance at various compression levels -- no limits set"
|
||||
./datagen -s7 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i5 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -5 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s8 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i10 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -10 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s9 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i15 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -15 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
@@ -1 +0,0 @@
|
||||
check_flipped_bits
|
||||
@@ -1,35 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2019-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
.PHONY: all
|
||||
all: check_flipped_bits
|
||||
|
||||
ZSTDLIBDIR ?= ../../lib
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \
|
||||
-I$(ZSTDLIBDIR)/decompress
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls -Wmissing-prototypes
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
.PHONY: $(ZSTDLIBDIR)/libzstd.a
|
||||
$(ZSTDLIBDIR)/libzstd.a:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
|
||||
$(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f check_flipped_bits
|
||||
@@ -1,400 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2019-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef struct {
|
||||
char *input;
|
||||
size_t input_size;
|
||||
|
||||
char *perturbed; /* same size as input */
|
||||
|
||||
char *output;
|
||||
size_t output_size;
|
||||
|
||||
const char *dict_file_name;
|
||||
const char *dict_file_dir_name;
|
||||
int32_t dict_id;
|
||||
char *dict;
|
||||
size_t dict_size;
|
||||
ZSTD_DDict* ddict;
|
||||
|
||||
ZSTD_DCtx* dctx;
|
||||
|
||||
int success_count;
|
||||
int error_counts[ZSTD_error_maxCode];
|
||||
} stuff_t;
|
||||
|
||||
static void free_stuff(stuff_t* stuff) {
|
||||
free(stuff->input);
|
||||
free(stuff->output);
|
||||
ZSTD_freeDDict(stuff->ddict);
|
||||
free(stuff->dict);
|
||||
ZSTD_freeDCtx(stuff->dctx);
|
||||
}
|
||||
|
||||
static void usage(void) {
|
||||
fprintf(stderr, "check_flipped_bits input_filename [-d dict] [-D dict_dir]\n");
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "Arguments:\n");
|
||||
fprintf(stderr, " -d file: path to a dictionary file to use.\n");
|
||||
fprintf(stderr, " -D dir : path to a directory, with files containing dictionaries, of the\n"
|
||||
" form DICTID.zstd-dict, e.g., 12345.zstd-dict.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void print_summary(stuff_t* stuff) {
|
||||
int error_code;
|
||||
fprintf(stderr, "%9d successful decompressions\n", stuff->success_count);
|
||||
for (error_code = 0; error_code < ZSTD_error_maxCode; error_code++) {
|
||||
int count = stuff->error_counts[error_code];
|
||||
if (count) {
|
||||
fprintf(
|
||||
stderr, "%9d failed decompressions with message: %s\n",
|
||||
count, ZSTD_getErrorString(error_code));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static char* readFile(const char* filename, size_t* size) {
|
||||
struct stat statbuf;
|
||||
int ret;
|
||||
FILE* f;
|
||||
char *buf;
|
||||
size_t bytes_read;
|
||||
|
||||
ret = stat(filename, &statbuf);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "stat failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
if ((statbuf.st_mode & S_IFREG) != S_IFREG) {
|
||||
fprintf(stderr, "Input must be regular file\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*size = statbuf.st_size;
|
||||
|
||||
f = fopen(filename, "r");
|
||||
if (f == NULL) {
|
||||
fprintf(stderr, "fopen failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
buf = malloc(*size);
|
||||
if (buf == NULL) {
|
||||
fprintf(stderr, "malloc failed\n");
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes_read = fread(buf, 1, *size, f);
|
||||
if (bytes_read != *size) {
|
||||
fprintf(stderr, "failed to read whole file\n");
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = fclose(f);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "fclose failed: %m\n");
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDict(const char* filename, char **buf, size_t* size, int32_t* dict_id) {
|
||||
ZSTD_DDict* ddict;
|
||||
*buf = readFile(filename, size);
|
||||
if (*buf == NULL) {
|
||||
fprintf(stderr, "Opening dictionary file '%s' failed\n", filename);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ddict = ZSTD_createDDict_advanced(*buf, *size, ZSTD_dlm_byRef, ZSTD_dct_auto, ZSTD_defaultCMem);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict.\n");
|
||||
return NULL;
|
||||
}
|
||||
if (dict_id != NULL) {
|
||||
*dict_id = ZSTD_getDictID_fromDDict(ddict);
|
||||
}
|
||||
return ddict;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDictByID(stuff_t *stuff, int32_t dict_id, char **buf, size_t* size) {
|
||||
if (stuff->dict_file_dir_name == NULL) {
|
||||
return NULL;
|
||||
} else {
|
||||
size_t dir_name_len = strlen(stuff->dict_file_dir_name);
|
||||
int dir_needs_separator = 0;
|
||||
size_t dict_file_name_alloc_size = dir_name_len + 1 /* '/' */ + 10 /* max int32_t len */ + strlen(".zstd-dict") + 1 /* '\0' */;
|
||||
char *dict_file_name = malloc(dict_file_name_alloc_size);
|
||||
ZSTD_DDict* ddict;
|
||||
int32_t read_dict_id;
|
||||
if (dict_file_name == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (dir_name_len > 0 && stuff->dict_file_dir_name[dir_name_len - 1] != '/') {
|
||||
dir_needs_separator = 1;
|
||||
}
|
||||
|
||||
snprintf(
|
||||
dict_file_name,
|
||||
dict_file_name_alloc_size,
|
||||
"%s%s%u.zstd-dict",
|
||||
stuff->dict_file_dir_name,
|
||||
dir_needs_separator ? "/" : "",
|
||||
dict_id);
|
||||
|
||||
/* fprintf(stderr, "Loading dict %u from '%s'.\n", dict_id, dict_file_name); */
|
||||
|
||||
ddict = readDict(dict_file_name, buf, size, &read_dict_id);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", dict_file_name);
|
||||
free(dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
if (read_dict_id != dict_id) {
|
||||
fprintf(stderr, "Read dictID (%u) does not match expected (%u).\n", read_dict_id, dict_id);
|
||||
free(dict_file_name);
|
||||
ZSTD_freeDDict(ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
free(dict_file_name);
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
static int init_stuff(stuff_t* stuff, int argc, char *argv[]) {
|
||||
const char* input_filename;
|
||||
|
||||
if (argc < 2) {
|
||||
usage();
|
||||
}
|
||||
|
||||
input_filename = argv[1];
|
||||
stuff->input_size = 0;
|
||||
stuff->input = readFile(input_filename, &stuff->input_size);
|
||||
if (stuff->input == NULL) {
|
||||
fprintf(stderr, "Failed to read input file.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->perturbed = malloc(stuff->input_size);
|
||||
if (stuff->perturbed == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
memcpy(stuff->perturbed, stuff->input, stuff->input_size);
|
||||
|
||||
stuff->output_size = ZSTD_DStreamOutSize();
|
||||
stuff->output = malloc(stuff->output_size);
|
||||
if (stuff->output == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->dict_file_name = NULL;
|
||||
stuff->dict_file_dir_name = NULL;
|
||||
stuff->dict_id = 0;
|
||||
stuff->dict = NULL;
|
||||
stuff->dict_size = 0;
|
||||
stuff->ddict = NULL;
|
||||
|
||||
if (argc > 2) {
|
||||
if (!strcmp(argv[2], "-d")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else
|
||||
if (!strcmp(argv[2], "-D")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_dir_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
}
|
||||
|
||||
if (stuff->dict_file_dir_name) {
|
||||
int32_t dict_id = ZSTD_getDictID_fromFrame(stuff->input, stuff->input_size);
|
||||
if (dict_id != 0) {
|
||||
stuff->ddict = readDictByID(stuff, dict_id, &stuff->dict, &stuff->dict_size);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create cached ddict.\n");
|
||||
return 0;
|
||||
}
|
||||
stuff->dict_id = dict_id;
|
||||
}
|
||||
} else
|
||||
if (stuff->dict_file_name) {
|
||||
stuff->ddict = readDict(stuff->dict_file_name, &stuff->dict, &stuff->dict_size, &stuff->dict_id);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", stuff->dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->dctx = ZSTD_createDCtx();
|
||||
if (stuff->dctx == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->success_count = 0;
|
||||
memset(stuff->error_counts, 0, sizeof(stuff->error_counts));
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int test_decompress(stuff_t* stuff) {
|
||||
size_t ret;
|
||||
ZSTD_inBuffer in = {stuff->perturbed, stuff->input_size, 0};
|
||||
ZSTD_outBuffer out = {stuff->output, stuff->output_size, 0};
|
||||
ZSTD_DCtx* dctx = stuff->dctx;
|
||||
int32_t custom_dict_id = ZSTD_getDictID_fromFrame(in.src, in.size);
|
||||
char *custom_dict = NULL;
|
||||
size_t custom_dict_size = 0;
|
||||
ZSTD_DDict* custom_ddict = NULL;
|
||||
|
||||
if (custom_dict_id != 0 && custom_dict_id != stuff->dict_id) {
|
||||
/* fprintf(stderr, "Instead of dict %u, this perturbed blob wants dict %u.\n", stuff->dict_id, custom_dict_id); */
|
||||
custom_ddict = readDictByID(stuff, custom_dict_id, &custom_dict, &custom_dict_size);
|
||||
}
|
||||
|
||||
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_DCtx_refDDict(dctx, custom_ddict);
|
||||
} else {
|
||||
ZSTD_DCtx_refDDict(dctx, stuff->ddict);
|
||||
}
|
||||
|
||||
while (in.pos != in.size) {
|
||||
out.pos = 0;
|
||||
ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||
|
||||
if (ZSTD_isError(ret)) {
|
||||
unsigned int code = ZSTD_getErrorCode(ret);
|
||||
if (code >= ZSTD_error_maxCode) {
|
||||
fprintf(stderr, "Received unexpected error code!\n");
|
||||
exit(1);
|
||||
}
|
||||
stuff->error_counts[code]++;
|
||||
/*
|
||||
fprintf(
|
||||
stderr, "Decompression failed: %s\n", ZSTD_getErrorName(ret));
|
||||
*/
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->success_count++;
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bits(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t bit;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (bit = 0; bit < 8; bit++) {
|
||||
unsigned char new_val = old_val ^ (1 << bit);
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Flipping byte %zu bit %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, bit, old_val, new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bytes(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t new_val;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (new_val = 0; new_val < 256; new_val++) {
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Changing byte %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, old_val, (unsigned char)new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
stuff_t stuff;
|
||||
|
||||
if(!init_stuff(&stuff, argc, argv)) {
|
||||
fprintf(stderr, "Failed to init.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (test_decompress(&stuff)) {
|
||||
fprintf(stderr, "Blob already decompresses successfully!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
perturb_bits(&stuff);
|
||||
|
||||
perturb_bytes(&stuff);
|
||||
|
||||
print_summary(&stuff);
|
||||
|
||||
free_stuff(&stuff);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
# Dockerfile
|
||||
# First image to build the binary
|
||||
FROM alpine as builder
|
||||
|
||||
RUN apk --no-cache add make gcc libc-dev
|
||||
COPY . /src
|
||||
RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install
|
||||
|
||||
# Second minimal image to only keep the built binary
|
||||
FROM alpine
|
||||
|
||||
# Copy the built files
|
||||
COPY --from=builder /pkg /
|
||||
|
||||
# Copy the license as well
|
||||
RUN mkdir -p /usr/local/share/licenses/zstd
|
||||
COPY --from=builder /src/LICENSE /usr/local/share/licences/zstd/
|
||||
|
||||
# Just run `zstd` if no other command is given
|
||||
CMD ["/usr/local/bin/zstd"]
|
||||
@@ -1,20 +0,0 @@
|
||||
|
||||
## Requirement
|
||||
|
||||
The `Dockerfile` script requires a version of `docker` >= 17.05
|
||||
|
||||
## Installing docker
|
||||
|
||||
The official docker install docs use a ppa with a modern version available:
|
||||
https://docs.docker.com/install/linux/docker-ce/ubuntu/
|
||||
|
||||
## How to run
|
||||
|
||||
`docker build -t zstd .`
|
||||
|
||||
## test
|
||||
|
||||
```
|
||||
echo foo | docker run -i --rm zstd | docker run -i --rm zstd zstdcat
|
||||
foo
|
||||
```
|
||||
@@ -7,10 +7,10 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
CXXFLAGS ?= -O3
|
||||
CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||
CXXFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS)
|
||||
|
||||
ZSTDAPI = ../../lib/zstd.h
|
||||
ZSTDMANUAL = ../../doc/zstd_manual.html
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# build artifacts
|
||||
largeNbDicts
|
||||
@@ -1,58 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
PROGDIR = ../../programs
|
||||
LIBDIR = ../../lib
|
||||
|
||||
LIBZSTD = $(LIBDIR)/libzstd.a
|
||||
|
||||
CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=gnu99
|
||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
|
||||
|
||||
default: largeNbDicts
|
||||
|
||||
all : largeNbDicts
|
||||
|
||||
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
.PHONY: $(LIBZSTD)
|
||||
$(LIBZSTD):
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
|
||||
|
||||
benchfn.o: $(PROGDIR)/benchfn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
timefn.o: $(PROGDIR)/timefn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
datagen.o: $(PROGDIR)/datagen.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
util.o: $(PROGDIR)/util.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
xxhash.o : $(LIBDIR)/common/xxhash.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
clean:
|
||||
$(RM) *.o
|
||||
$(MAKE) -C $(LIBDIR) clean > /dev/null
|
||||
$(RM) largeNbDicts
|
||||
@@ -1,25 +0,0 @@
|
||||
largeNbDicts
|
||||
=====================
|
||||
|
||||
`largeNbDicts` is a benchmark test tool
|
||||
dedicated to the specific scenario of
|
||||
dictionary decompression using a very large number of dictionaries.
|
||||
When dictionaries are constantly changing, they are always "cold",
|
||||
suffering from increased latency due to cache misses.
|
||||
|
||||
The tool is created in a bid to investigate performance for this scenario,
|
||||
and experiment mitigation techniques.
|
||||
|
||||
Command line :
|
||||
```
|
||||
largeNbDicts [Options] filename(s)
|
||||
|
||||
Options :
|
||||
-r : recursively load all files in subdirectories (default: off)
|
||||
-B# : split input into blocks of size # (default: no split)
|
||||
-# : use compression level # (default: 3)
|
||||
-D # : use # as a dictionary (default: create one)
|
||||
-i# : nb benchmark rounds (default: 6)
|
||||
--nbDicts=# : set nb of dictionaries to # (default: one per block)
|
||||
-h : help (this text)
|
||||
```
|
||||
@@ -1,979 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* largeNbDicts
|
||||
* This is a benchmark test tool
|
||||
* dedicated to the specific case of dictionary decompression
|
||||
* using a very large nb of dictionaries
|
||||
* thus suffering latency from lots of cache misses.
|
||||
* It's created in a bid to investigate performance and find optimizations. */
|
||||
|
||||
|
||||
/*--- Dependencies ---*/
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, abort */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <limits.h> /* UINT_MAX */
|
||||
#include <assert.h> /* assert */
|
||||
|
||||
#include "util.h"
|
||||
#include "benchfn.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*--- Constants --- */
|
||||
|
||||
#define KB *(1<<10)
|
||||
#define MB *(1<<20)
|
||||
|
||||
#define BLOCKSIZE_DEFAULT 0 /* no slicing into blocks */
|
||||
#define DICTSIZE (4 KB)
|
||||
#define CLEVEL_DEFAULT 3
|
||||
|
||||
#define BENCH_TIME_DEFAULT_S 6
|
||||
#define RUN_TIME_DEFAULT_MS 1000
|
||||
#define BENCH_TIME_DEFAULT_MS (BENCH_TIME_DEFAULT_S * RUN_TIME_DEFAULT_MS)
|
||||
|
||||
#define DISPLAY_LEVEL_DEFAULT 3
|
||||
|
||||
#define BENCH_SIZE_MAX (1200 MB)
|
||||
|
||||
|
||||
/*--- Macros ---*/
|
||||
|
||||
#define CONTROL(c) { if (!(c)) abort(); }
|
||||
#undef MIN
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
|
||||
/*--- Display Macros ---*/
|
||||
|
||||
#define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
|
||||
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
|
||||
|
||||
|
||||
/*--- buffer_t ---*/
|
||||
|
||||
typedef struct {
|
||||
void* ptr;
|
||||
size_t size;
|
||||
size_t capacity;
|
||||
} buffer_t;
|
||||
|
||||
static const buffer_t kBuffNull = { NULL, 0, 0 };
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t createBuffer(size_t capacity)
|
||||
{
|
||||
assert(capacity > 0);
|
||||
void* const ptr = malloc(capacity);
|
||||
if (ptr==NULL) return kBuffNull;
|
||||
|
||||
buffer_t buffer;
|
||||
buffer.ptr = ptr;
|
||||
buffer.capacity = capacity;
|
||||
buffer.size = 0;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static void freeBuffer(buffer_t buff)
|
||||
{
|
||||
free(buff.ptr);
|
||||
}
|
||||
|
||||
|
||||
static void fillBuffer_fromHandle(buffer_t* buff, FILE* f)
|
||||
{
|
||||
size_t const readSize = fread(buff->ptr, 1, buff->capacity, f);
|
||||
buff->size = readSize;
|
||||
}
|
||||
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t createBuffer_fromFile(const char* fileName)
|
||||
{
|
||||
U64 const fileSize = UTIL_getFileSize(fileName);
|
||||
size_t const bufferSize = (size_t) fileSize;
|
||||
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull;
|
||||
assert((U64)bufferSize == fileSize); /* check overflow */
|
||||
|
||||
{ FILE* const f = fopen(fileName, "rb");
|
||||
if (f == NULL) return kBuffNull;
|
||||
|
||||
buffer_t buff = createBuffer(bufferSize);
|
||||
CONTROL(buff.ptr != NULL);
|
||||
|
||||
fillBuffer_fromHandle(&buff, f);
|
||||
CONTROL(buff.size == buff.capacity);
|
||||
|
||||
fclose(f); /* do nothing specific if fclose() fails */
|
||||
return buff;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_t
|
||||
createDictionaryBuffer(const char* dictionaryName,
|
||||
const void* srcBuffer,
|
||||
const size_t* srcBlockSizes, size_t nbBlocks,
|
||||
size_t requestedDictSize)
|
||||
{
|
||||
if (dictionaryName) {
|
||||
DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName);
|
||||
return createBuffer_fromFile(dictionaryName); /* note : result might be kBuffNull */
|
||||
|
||||
} else {
|
||||
|
||||
DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n",
|
||||
(unsigned)requestedDictSize);
|
||||
void* const dictBuffer = malloc(requestedDictSize);
|
||||
CONTROL(dictBuffer != NULL);
|
||||
|
||||
assert(nbBlocks <= UINT_MAX);
|
||||
size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
|
||||
srcBuffer,
|
||||
srcBlockSizes, (unsigned)nbBlocks);
|
||||
CONTROL(!ZSTD_isError(dictSize));
|
||||
|
||||
buffer_t result;
|
||||
result.ptr = dictBuffer;
|
||||
result.capacity = requestedDictSize;
|
||||
result.size = dictSize;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
* Loads `buffer`, with content from files listed within `fileNamesTable`.
|
||||
* Fills `buffer` entirely.
|
||||
* @return : 0 on success, !=0 on error */
|
||||
static int loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char* const * fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0, totalSize = 0;
|
||||
|
||||
for (unsigned n=0; n<nbFiles; n++) {
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
assert(f!=NULL);
|
||||
|
||||
assert(pos <= bufferSize);
|
||||
assert(fileSize <= bufferSize - pos);
|
||||
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
assert(readSize == fileSize);
|
||||
pos += readSize;
|
||||
}
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
totalSize += (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
assert(totalSize == bufferSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*--- slice_collection_t ---*/
|
||||
|
||||
typedef struct {
|
||||
void** slicePtrs;
|
||||
size_t* capacities;
|
||||
size_t nbSlices;
|
||||
} slice_collection_t;
|
||||
|
||||
static const slice_collection_t kNullCollection = { NULL, NULL, 0 };
|
||||
|
||||
static void freeSliceCollection(slice_collection_t collection)
|
||||
{
|
||||
free(collection.slicePtrs);
|
||||
free(collection.capacities);
|
||||
}
|
||||
|
||||
/* shrinkSizes() :
|
||||
* downsizes sizes of slices within collection, according to `newSizes`.
|
||||
* every `newSizes` entry must be <= than its corresponding collection size */
|
||||
void shrinkSizes(slice_collection_t collection,
|
||||
const size_t* newSizes) /* presumed same size as collection */
|
||||
{
|
||||
size_t const nbSlices = collection.nbSlices;
|
||||
for (size_t blockNb = 0; blockNb < nbSlices; blockNb++) {
|
||||
assert(newSizes[blockNb] <= collection.capacities[blockNb]);
|
||||
collection.capacities[blockNb] = newSizes[blockNb];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* splitSlices() :
|
||||
* nbSlices : if == 0, nbSlices is automatically determined from srcSlices and blockSize.
|
||||
* otherwise, creates exactly nbSlices slices,
|
||||
* by either truncating input (when smaller)
|
||||
* or repeating input from beginning */
|
||||
static slice_collection_t
|
||||
splitSlices(slice_collection_t srcSlices, size_t blockSize, size_t nbSlices)
|
||||
{
|
||||
if (blockSize==0) blockSize = (size_t)(-1); /* means "do not cut" */
|
||||
size_t nbSrcBlocks = 0;
|
||||
for (size_t ssnb=0; ssnb < srcSlices.nbSlices; ssnb++) {
|
||||
size_t pos = 0;
|
||||
while (pos <= srcSlices.capacities[ssnb]) {
|
||||
nbSrcBlocks++;
|
||||
pos += blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
if (nbSlices == 0) nbSlices = nbSrcBlocks;
|
||||
|
||||
void** const sliceTable = (void**)malloc(nbSlices * sizeof(*sliceTable));
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
if (sliceTable == NULL || capacities == NULL) {
|
||||
free(sliceTable);
|
||||
free(capacities);
|
||||
return kNullCollection;
|
||||
}
|
||||
|
||||
size_t ssnb = 0;
|
||||
for (size_t sliceNb=0; sliceNb < nbSlices; ) {
|
||||
ssnb = (ssnb + 1) % srcSlices.nbSlices;
|
||||
size_t pos = 0;
|
||||
char* const ptr = (char*)srcSlices.slicePtrs[ssnb];
|
||||
while (pos < srcSlices.capacities[ssnb] && sliceNb < nbSlices) {
|
||||
size_t const size = MIN(blockSize, srcSlices.capacities[ssnb] - pos);
|
||||
sliceTable[sliceNb] = ptr + pos;
|
||||
capacities[sliceNb] = size;
|
||||
sliceNb++;
|
||||
pos += blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
slice_collection_t result;
|
||||
result.nbSlices = nbSlices;
|
||||
result.slicePtrs = sliceTable;
|
||||
result.capacities = capacities;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
static size_t sliceCollection_totalCapacity(slice_collection_t sc)
|
||||
{
|
||||
size_t totalSize = 0;
|
||||
for (size_t n=0; n<sc.nbSlices; n++)
|
||||
totalSize += sc.capacities[n];
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
|
||||
/* --- buffer collection --- */
|
||||
|
||||
typedef struct {
|
||||
buffer_t buffer;
|
||||
slice_collection_t slices;
|
||||
} buffer_collection_t;
|
||||
|
||||
|
||||
static void freeBufferCollection(buffer_collection_t bc)
|
||||
{
|
||||
freeBuffer(bc.buffer);
|
||||
freeSliceCollection(bc.slices);
|
||||
}
|
||||
|
||||
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc)
|
||||
{
|
||||
size_t const bufferSize = sliceCollection_totalCapacity(sc);
|
||||
|
||||
buffer_t buffer = createBuffer(bufferSize);
|
||||
CONTROL(buffer.ptr != NULL);
|
||||
|
||||
size_t const nbSlices = sc.nbSlices;
|
||||
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
|
||||
CONTROL(slices != NULL);
|
||||
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
CONTROL(capacities != NULL);
|
||||
|
||||
char* const ptr = (char*)buffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t n=0; n < nbSlices; n++) {
|
||||
capacities[n] = sc.capacities[n];
|
||||
slices[n] = ptr + pos;
|
||||
pos += capacities[n];
|
||||
}
|
||||
|
||||
buffer_collection_t result;
|
||||
result.buffer = buffer;
|
||||
result.slices.nbSlices = nbSlices;
|
||||
result.slices.capacities = capacities;
|
||||
result.slices.slicePtrs = slices;
|
||||
return result;
|
||||
}
|
||||
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromSliceCollection(slice_collection_t sc)
|
||||
{
|
||||
size_t const bufferSize = sliceCollection_totalCapacity(sc);
|
||||
|
||||
buffer_t buffer = createBuffer(bufferSize);
|
||||
CONTROL(buffer.ptr != NULL);
|
||||
|
||||
size_t const nbSlices = sc.nbSlices;
|
||||
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
|
||||
CONTROL(slices != NULL);
|
||||
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
CONTROL(capacities != NULL);
|
||||
|
||||
char* const ptr = (char*)buffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t n=0; n < nbSlices; n++) {
|
||||
capacities[n] = sc.capacities[n];
|
||||
slices[n] = ptr + pos;
|
||||
pos += capacities[n];
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < nbSlices; i++) {
|
||||
memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]);
|
||||
capacities[i] = sc.capacities[i];
|
||||
}
|
||||
|
||||
buffer_collection_t result;
|
||||
result.buffer = buffer;
|
||||
result.slices.nbSlices = nbSlices;
|
||||
result.slices.capacities = capacities;
|
||||
result.slices.slicePtrs = slices;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromFiles(const char* const * fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
assert(totalSizeToLoad != UTIL_FILESIZE_UNKNOWN);
|
||||
assert(totalSizeToLoad <= BENCH_SIZE_MAX);
|
||||
size_t const loadedSize = (size_t)totalSizeToLoad;
|
||||
assert(loadedSize > 0);
|
||||
void* const srcBuffer = malloc(loadedSize);
|
||||
assert(srcBuffer != NULL);
|
||||
|
||||
assert(nbFiles > 0);
|
||||
size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(*fileSizes));
|
||||
assert(fileSizes != NULL);
|
||||
|
||||
/* Load input buffer */
|
||||
int const errorCode = loadFiles(srcBuffer, loadedSize,
|
||||
fileSizes,
|
||||
fileNamesTable, nbFiles);
|
||||
assert(errorCode == 0);
|
||||
|
||||
void** sliceTable = (void**)malloc(nbFiles * sizeof(*sliceTable));
|
||||
assert(sliceTable != NULL);
|
||||
|
||||
char* const ptr = (char*)srcBuffer;
|
||||
size_t pos = 0;
|
||||
unsigned fileNb = 0;
|
||||
for ( ; (pos < loadedSize) && (fileNb < nbFiles); fileNb++) {
|
||||
sliceTable[fileNb] = ptr + pos;
|
||||
pos += fileSizes[fileNb];
|
||||
}
|
||||
assert(pos == loadedSize);
|
||||
assert(fileNb == nbFiles);
|
||||
|
||||
|
||||
buffer_t buffer;
|
||||
buffer.ptr = srcBuffer;
|
||||
buffer.capacity = loadedSize;
|
||||
buffer.size = loadedSize;
|
||||
|
||||
slice_collection_t slices;
|
||||
slices.slicePtrs = sliceTable;
|
||||
slices.capacities = fileSizes;
|
||||
slices.nbSlices = nbFiles;
|
||||
|
||||
buffer_collection_t bc;
|
||||
bc.buffer = buffer;
|
||||
bc.slices = slices;
|
||||
return bc;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*--- ddict_collection_t ---*/
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DDict** ddicts;
|
||||
size_t nbDDict;
|
||||
} ddict_collection_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CDict** cdicts;
|
||||
size_t nbCDict;
|
||||
} cdict_collection_t;
|
||||
|
||||
static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeCDictCollection(cdict_collection_t cdictc)
|
||||
{
|
||||
for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
|
||||
ZSTD_freeCDict(cdictc.cdicts[dictNb]);
|
||||
}
|
||||
free(cdictc.cdicts);
|
||||
}
|
||||
|
||||
/* returns .buffers=NULL if operation fails */
|
||||
static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel)
|
||||
{
|
||||
ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
|
||||
if (cdicts==NULL) return kNullCDictCollection;
|
||||
for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
|
||||
cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
|
||||
CONTROL(cdicts[dictNb] != NULL);
|
||||
}
|
||||
cdict_collection_t cdictc;
|
||||
cdictc.cdicts = cdicts;
|
||||
cdictc.nbCDict = nbCDict;
|
||||
return cdictc;
|
||||
}
|
||||
|
||||
static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeDDictCollection(ddict_collection_t ddictc)
|
||||
{
|
||||
for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) {
|
||||
ZSTD_freeDDict(ddictc.ddicts[dictNb]);
|
||||
}
|
||||
free(ddictc.ddicts);
|
||||
}
|
||||
|
||||
/* returns .buffers=NULL if operation fails */
|
||||
static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict)
|
||||
{
|
||||
ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*));
|
||||
assert(ddicts != NULL);
|
||||
if (ddicts==NULL) return kNullDDictCollection;
|
||||
for (size_t dictNb=0; dictNb < nbDDict; dictNb++) {
|
||||
ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize);
|
||||
assert(ddicts[dictNb] != NULL);
|
||||
}
|
||||
ddict_collection_t ddictc;
|
||||
ddictc.ddicts = ddicts;
|
||||
ddictc.nbDDict = nbDDict;
|
||||
return ddictc;
|
||||
}
|
||||
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleCDictionaries(cdict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbCDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d];
|
||||
dicts.cdicts[d] = dicts.cdicts[r];
|
||||
dicts.cdicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d1];
|
||||
dicts.cdicts[d1] = dicts.cdicts[d2];
|
||||
dicts.cdicts[d2] = tmpd;
|
||||
}
|
||||
}
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleDDictionaries(ddict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbDDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d];
|
||||
dicts.ddicts[d] = dicts.ddicts[r];
|
||||
dicts.ddicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d1];
|
||||
dicts.ddicts[d1] = dicts.ddicts[d2];
|
||||
dicts.ddicts[d2] = tmpd;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* --- Compression --- */
|
||||
|
||||
/* compressBlocks() :
|
||||
* @return : total compressed size of all blocks,
|
||||
* or 0 if error.
|
||||
*/
|
||||
static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If present, must contain at least nbBlocks fields */
|
||||
slice_collection_t dstBlockBuffers,
|
||||
slice_collection_t srcBlockBuffers,
|
||||
ZSTD_CDict* cdict, int cLevel)
|
||||
{
|
||||
size_t const nbBlocks = srcBlockBuffers.nbSlices;
|
||||
assert(dstBlockBuffers.nbSlices == srcBlockBuffers.nbSlices);
|
||||
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
assert(cctx != NULL);
|
||||
|
||||
size_t totalCSize = 0;
|
||||
for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) {
|
||||
size_t cBlockSize;
|
||||
if (cdict == NULL) {
|
||||
cBlockSize = ZSTD_compressCCtx(cctx,
|
||||
dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb],
|
||||
srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb],
|
||||
cLevel);
|
||||
} else {
|
||||
cBlockSize = ZSTD_compress_usingCDict(cctx,
|
||||
dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb],
|
||||
srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb],
|
||||
cdict);
|
||||
}
|
||||
CONTROL(!ZSTD_isError(cBlockSize));
|
||||
if (cSizes) cSizes[blockNb] = cBlockSize;
|
||||
totalCSize += cBlockSize;
|
||||
}
|
||||
return totalCSize;
|
||||
}
|
||||
|
||||
|
||||
/* --- Benchmark --- */
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CCtx* cctx;
|
||||
size_t nbDicts;
|
||||
size_t dictNb;
|
||||
cdict_collection_t dictionaries;
|
||||
} compressInstructions;
|
||||
|
||||
compressInstructions createCompressInstructions(cdict_collection_t dictionaries)
|
||||
{
|
||||
compressInstructions ci;
|
||||
ci.cctx = ZSTD_createCCtx();
|
||||
CONTROL(ci.cctx != NULL);
|
||||
ci.nbDicts = dictionaries.nbCDict;
|
||||
ci.dictNb = 0;
|
||||
ci.dictionaries = dictionaries;
|
||||
return ci;
|
||||
}
|
||||
|
||||
void freeCompressInstructions(compressInstructions ci)
|
||||
{
|
||||
ZSTD_freeCCtx(ci.cctx);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DCtx* dctx;
|
||||
size_t nbDicts;
|
||||
size_t dictNb;
|
||||
ddict_collection_t dictionaries;
|
||||
} decompressInstructions;
|
||||
|
||||
decompressInstructions createDecompressInstructions(ddict_collection_t dictionaries)
|
||||
{
|
||||
decompressInstructions di;
|
||||
di.dctx = ZSTD_createDCtx();
|
||||
assert(di.dctx != NULL);
|
||||
di.nbDicts = dictionaries.nbDDict;
|
||||
di.dictNb = 0;
|
||||
di.dictionaries = dictionaries;
|
||||
return di;
|
||||
}
|
||||
|
||||
void freeDecompressInstructions(decompressInstructions di)
|
||||
{
|
||||
ZSTD_freeDCtx(di.dctx);
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
compressInstructions* const ci = (compressInstructions*) payload;
|
||||
(void)dstCapacity;
|
||||
|
||||
ZSTD_compress_usingCDict(ci->cctx,
|
||||
dst, srcSize,
|
||||
src, srcSize,
|
||||
ci->dictionaries.cdicts[ci->dictNb]);
|
||||
|
||||
ci->dictNb = ci->dictNb + 1;
|
||||
if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
decompressInstructions* const di = (decompressInstructions*) payload;
|
||||
|
||||
size_t const result = ZSTD_decompress_usingDDict(di->dctx,
|
||||
dst, dstCapacity,
|
||||
src, srcSize,
|
||||
di->dictionaries.ddicts[di->dictNb]);
|
||||
|
||||
di->dictNb = di->dictNb + 1;
|
||||
if (di->dictNb >= di->nbDicts) di->dictNb = 0;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
static int benchMem(slice_collection_t dstBlocks,
|
||||
slice_collection_t srcBlocks,
|
||||
ddict_collection_t ddictionaries,
|
||||
cdict_collection_t cdictionaries,
|
||||
unsigned nbRounds, int benchCompression)
|
||||
{
|
||||
assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
|
||||
|
||||
unsigned const ms_per_round = RUN_TIME_DEFAULT_MS;
|
||||
unsigned const total_time_ms = nbRounds * ms_per_round;
|
||||
|
||||
double bestSpeed = 0.;
|
||||
|
||||
BMK_timedFnState_t* const benchState =
|
||||
BMK_createTimedFnState(total_time_ms, ms_per_round);
|
||||
|
||||
decompressInstructions di = createDecompressInstructions(ddictionaries);
|
||||
compressInstructions ci = createCompressInstructions(cdictionaries);
|
||||
void* payload = benchCompression ? (void*)&ci : (void*)&di;
|
||||
BMK_benchParams_t const bp = {
|
||||
.benchFn = benchCompression ? compress : decompress,
|
||||
.benchPayload = payload,
|
||||
.initFn = NULL,
|
||||
.initPayload = NULL,
|
||||
.errorFn = ZSTD_isError,
|
||||
.blockCount = dstBlocks.nbSlices,
|
||||
.srcBuffers = (const void* const*) srcBlocks.slicePtrs,
|
||||
.srcSizes = srcBlocks.capacities,
|
||||
.dstBuffers = dstBlocks.slicePtrs,
|
||||
.dstCapacities = dstBlocks.capacities,
|
||||
.blockResults = NULL
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, bp);
|
||||
CONTROL(BMK_isSuccessful_runOutcome(outcome));
|
||||
|
||||
BMK_runTime_t const result = BMK_extract_runTime(outcome);
|
||||
double const dTime_ns = result.nanoSecPerRun;
|
||||
double const dTime_sec = (double)dTime_ns / 1000000000;
|
||||
size_t const srcSize = result.sumOfReturn;
|
||||
double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
|
||||
if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
|
||||
if (benchCompression)
|
||||
DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
|
||||
else
|
||||
DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
|
||||
|
||||
fflush(stdout);
|
||||
if (BMK_isCompleted_TimedFn(benchState)) break;
|
||||
}
|
||||
DISPLAY("\n");
|
||||
|
||||
freeDecompressInstructions(di);
|
||||
freeCompressInstructions(ci);
|
||||
BMK_freeTimedFnState(benchState);
|
||||
|
||||
return 0; /* success */
|
||||
}
|
||||
|
||||
|
||||
/*! bench() :
|
||||
* fileName : file to load for benchmarking purpose
|
||||
* dictionary : optional (can be NULL), file to load as dictionary,
|
||||
* if none provided : will be calculated on the fly by the program.
|
||||
* @return : 0 is success, 1+ otherwise */
|
||||
int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
const char* dictionary,
|
||||
size_t blockSize, int clevel,
|
||||
unsigned nbDictMax, unsigned nbBlocks,
|
||||
unsigned nbRounds, int benchCompression)
|
||||
{
|
||||
int result = 0;
|
||||
|
||||
DISPLAYLEVEL(3, "loading %u files... \n", nbFiles);
|
||||
buffer_collection_t const srcs = createBufferCollection_fromFiles(fileNameTable, nbFiles);
|
||||
CONTROL(srcs.buffer.ptr != NULL);
|
||||
buffer_t srcBuffer = srcs.buffer;
|
||||
size_t const srcSize = srcBuffer.size;
|
||||
DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n",
|
||||
(double)srcSize / (1 MB));
|
||||
|
||||
slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize, nbBlocks);
|
||||
nbBlocks = (unsigned)(srcSlices.nbSlices);
|
||||
DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks);
|
||||
if (blockSize)
|
||||
DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
|
||||
DISPLAYLEVEL(3, "\n");
|
||||
size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
|
||||
|
||||
|
||||
size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
|
||||
CONTROL(dstCapacities != NULL);
|
||||
size_t dstBufferCapacity = 0;
|
||||
for (size_t bnb=0; bnb<nbBlocks; bnb++) {
|
||||
dstCapacities[bnb] = ZSTD_compressBound(srcSlices.capacities[bnb]);
|
||||
dstBufferCapacity += dstCapacities[bnb];
|
||||
}
|
||||
|
||||
buffer_t dstBuffer = createBuffer(dstBufferCapacity);
|
||||
CONTROL(dstBuffer.ptr != NULL);
|
||||
|
||||
void** const sliceTable = malloc(nbBlocks * sizeof(*sliceTable));
|
||||
CONTROL(sliceTable != NULL);
|
||||
|
||||
{ char* const ptr = dstBuffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t snb=0; snb < nbBlocks; snb++) {
|
||||
sliceTable[snb] = ptr + pos;
|
||||
pos += dstCapacities[snb];
|
||||
} }
|
||||
|
||||
slice_collection_t dstSlices;
|
||||
dstSlices.capacities = dstCapacities;
|
||||
dstSlices.slicePtrs = sliceTable;
|
||||
dstSlices.nbSlices = nbBlocks;
|
||||
|
||||
|
||||
/* dictionary determination */
|
||||
buffer_t const dictBuffer = createDictionaryBuffer(dictionary,
|
||||
srcs.buffer.ptr,
|
||||
srcs.slices.capacities, srcs.slices.nbSlices,
|
||||
DICTSIZE);
|
||||
CONTROL(dictBuffer.ptr != NULL);
|
||||
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel);
|
||||
CONTROL(cdict != NULL);
|
||||
|
||||
size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel);
|
||||
CONTROL(cTotalSizeNoDict != 0);
|
||||
DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
clevel,
|
||||
(double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
|
||||
|
||||
size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
|
||||
CONTROL(cSizes != NULL);
|
||||
|
||||
size_t const cTotalSize = compressBlocks(cSizes, dstSlices, srcSlices, cdict, clevel);
|
||||
CONTROL(cTotalSize != 0);
|
||||
DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
(unsigned)dictBuffer.size,
|
||||
(double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
|
||||
|
||||
/* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
|
||||
shrinkSizes(dstSlices, cSizes);
|
||||
|
||||
unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
|
||||
|
||||
cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel);
|
||||
CONTROL(cdictionaries.cdicts != NULL);
|
||||
|
||||
ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
|
||||
CONTROL(ddictionaries.ddicts != NULL);
|
||||
|
||||
if (benchCompression) {
|
||||
size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
shuffleCDictionaries(cdictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
} else {
|
||||
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
shuffleDDictionaries(ddictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
}
|
||||
|
||||
/* free all heap objects in reverse order */
|
||||
freeCDictCollection(cdictionaries);
|
||||
freeDDictCollection(ddictionaries);
|
||||
free(cSizes);
|
||||
ZSTD_freeCDict(cdict);
|
||||
freeBuffer(dictBuffer);
|
||||
freeSliceCollection(dstSlices);
|
||||
freeBuffer(dstBuffer);
|
||||
freeSliceCollection(srcSlices);
|
||||
freeBufferCollection(srcs);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* --- Command Line --- */
|
||||
|
||||
/*! readU32FromChar() :
|
||||
* @return : unsigned integer value read from input in `char` format.
|
||||
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
* Note : function will exit() program if digit sequence overflows */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
assert(result <= max); /* check overflow */
|
||||
result *= 10, result += (unsigned)**stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
assert(result <= maxK); /* check overflow */
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') {
|
||||
assert(result <= maxK); /* check overflow */
|
||||
result <<= 10;
|
||||
}
|
||||
(*stringPtr)++; /* skip `K` or `M` */
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check if *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static int longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int usage(const char* exeName)
|
||||
{
|
||||
DISPLAY (" \n");
|
||||
DISPLAY (" %s [Options] filename(s) \n", exeName);
|
||||
DISPLAY (" \n");
|
||||
DISPLAY ("Options : \n");
|
||||
DISPLAY ("-z : benchmark compression (default) \n");
|
||||
DISPLAY ("-d : benchmark decompression \n");
|
||||
DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n");
|
||||
DISPLAY ("-B# : split input into blocks of size # (default: no split) \n");
|
||||
DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
|
||||
DISPLAY ("-D # : use # as a dictionary (default: create one) \n");
|
||||
DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
|
||||
DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
|
||||
DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
|
||||
DISPLAY ("-h : help (this text) \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int bad_usage(const char* exeName)
|
||||
{
|
||||
DISPLAY (" bad usage : \n");
|
||||
usage(exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main (int argc, const char** argv)
|
||||
{
|
||||
int recursiveMode = 0;
|
||||
int benchCompression = 1;
|
||||
unsigned nbRounds = BENCH_TIME_DEFAULT_S;
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc < 2) return bad_usage(exeName);
|
||||
|
||||
const char** nameTable = (const char**)malloc((size_t)argc * sizeof(const char*));
|
||||
assert(nameTable != NULL);
|
||||
unsigned nameIdx = 0;
|
||||
|
||||
const char* dictionary = NULL;
|
||||
int cLevel = CLEVEL_DEFAULT;
|
||||
size_t blockSize = BLOCKSIZE_DEFAULT;
|
||||
unsigned nbDicts = 0; /* determine nbDicts automatically: 1 dictionary per block */
|
||||
unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
|
||||
|
||||
for (int argNb = 1; argNb < argc ; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
|
||||
if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
|
||||
if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
|
||||
if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
|
||||
if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
|
||||
if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--dictionary=")) { dictionary = argument; continue; }
|
||||
if (longCommandWArg(&argument, "-B")) { blockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
/* anything that's not a command is a filename */
|
||||
nameTable[nameIdx++] = argument;
|
||||
}
|
||||
|
||||
FileNamesTable* filenameTable;
|
||||
|
||||
if (recursiveMode) {
|
||||
#ifndef UTIL_HAS_CREATEFILELIST
|
||||
assert(0); /* missing capability, do not run */
|
||||
#endif
|
||||
filenameTable = UTIL_createExpandedFNT(nameTable, nameIdx, 1 /* follow_links */);
|
||||
} else {
|
||||
filenameTable = UTIL_assembleFileNamesTable(nameTable, nameIdx, NULL);
|
||||
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
|
||||
}
|
||||
|
||||
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression);
|
||||
|
||||
UTIL_freeFileNamesTable(filenameTable);
|
||||
free(nameTable);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -4,7 +4,7 @@ Date: Mon, 17 Jul 2017 17:08:19 -0700
|
||||
Subject: [PATCH v5 2/5] lib: Add zstd modules
|
||||
|
||||
Add zstd compression and decompression kernel modules.
|
||||
zstd offers a wide variety of compression speed and quality trade-offs.
|
||||
zstd offers a wide varity of compression speed and quality trade-offs.
|
||||
It can compress at speeds approaching lz4, and quality approaching lzma.
|
||||
zstd decompressions at speeds more than twice as fast as zlib, and
|
||||
decompression speed remains roughly the same across all compression levels.
|
||||
@@ -21,7 +21,7 @@ will be easier to keep the kernel zstd up to date.
|
||||
I benchmarked zstd compression as a special character device. I ran zstd
|
||||
and zlib compression at several levels, as well as performing no
|
||||
compression, which measure the time spent copying the data to kernel space.
|
||||
Data is passed to the compressor 4096 B at a time. The benchmark file is
|
||||
Data is passed to the compresser 4096 B at a time. The benchmark file is
|
||||
located in the upstream zstd source repository under
|
||||
`contrib/linux-kernel/zstd_compress_test.c` [2].
|
||||
|
||||
@@ -86,7 +86,7 @@ Tested in userland using the test-suite in the zstd repo under
|
||||
`contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel
|
||||
functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under
|
||||
`contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8]
|
||||
with ASAN, UBSAN, and MSAN. Additionally, it was tested while testing the
|
||||
with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the
|
||||
BtrFS and SquashFS patches coming next.
|
||||
|
||||
[1] https://clang.llvm.org/docs/ClangFormat.html
|
||||
@@ -4200,14 +4200,14 @@ index 0000000..ff18ae6
|
||||
+ BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
+ U32 const fcsCode =
|
||||
+ params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
|
||||
+ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
+ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
+ size_t pos;
|
||||
+
|
||||
+ if (dstCapacity < ZSTD_frameHeaderSize_max)
|
||||
+ return ERROR(dstSize_tooSmall);
|
||||
+
|
||||
+ ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
+ op[4] = frameHeaderDescriptionByte;
|
||||
+ op[4] = frameHeaderDecriptionByte;
|
||||
+ pos = 5;
|
||||
+ if (!singleSegment)
|
||||
+ op[pos++] = windowLogByte;
|
||||
@@ -8812,8 +8812,8 @@ index 0000000..ef3d174
|
||||
+ U32 position = 0;
|
||||
+ U32 symbol;
|
||||
+ for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
|
||||
+ int nbOccurrences;
|
||||
+ for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
|
||||
+ int nbOccurences;
|
||||
+ for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
|
||||
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
+ position = (position + step) & tableMask;
|
||||
+ while (position > highThreshold)
|
||||
@@ -9944,7 +9944,7 @@ index 0000000..2143da2
|
||||
+ HUF_repeat_none, /**< Cannot use the previous table */
|
||||
+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
|
||||
+ 4}X_repeat */
|
||||
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
+ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
+} HUF_repeat;
|
||||
+/** HUF_compress4X_repeat() :
|
||||
+* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
|
||||
@@ -11,7 +11,7 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v4 -> v5:
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
- Don't strip trailing whitespace of unrelated code
|
||||
- Don't strip trailing whitespace of unreleated code
|
||||
- Make zstd_display_options() static
|
||||
|
||||
v5 -> v6:
|
||||
@@ -224,7 +224,7 @@ index 0000000..dcab75a
|
||||
+ * set the default options, this is to ensure any user supplied
|
||||
+ * -X options on the appending mksquashfs command line are over-ridden.
|
||||
+ *
|
||||
+ * This function returns 0 on successful extraction of options, and -1 on error.
|
||||
+ * This function returns 0 on sucessful extraction of options, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_extract_options(int block_size, void *buffer, int size)
|
||||
+{
|
||||
|
||||
@@ -2436,14 +2436,14 @@ static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_paramete
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode =
|
||||
params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
|
||||
BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max)
|
||||
return ERROR(dstSize_tooSmall);
|
||||
|
||||
ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
op[4] = frameHeaderDescriptionByte;
|
||||
op[4] = frameHeaderDecriptionByte;
|
||||
pos = 5;
|
||||
if (!singleSegment)
|
||||
op[pos++] = windowLogByte;
|
||||
|
||||
@@ -232,7 +232,7 @@ If there is an error, the function will return an error code, which can be teste
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) (size + (size >> 7) + 4 /* constant for initial fse states */ )
|
||||
#define FSE_BLOCKBOUND(size) (size + (size >> 7))
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
|
||||
|
||||
@@ -141,8 +141,8 @@ size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsi
|
||||
U32 position = 0;
|
||||
U32 symbol;
|
||||
for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
|
||||
int nbOccurrences;
|
||||
for (nbOccurrences = 0; nbOccurrences < normalizedCounter[symbol]; nbOccurrences++) {
|
||||
int nbOccurences;
|
||||
for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
|
||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold)
|
||||
|
||||
@@ -134,7 +134,7 @@ typedef enum {
|
||||
HUF_repeat_none, /**< Cannot use the previous table */
|
||||
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
|
||||
4}X_repeat */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
||||
HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
|
||||
} HUF_repeat;
|
||||
/** HUF_compress4X_repeat() :
|
||||
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#ifndef LINUX_COMPILER_H_
|
||||
#define LINUX_COMPILER_H_
|
||||
#ifndef LINUX_COMIPLER_H_
|
||||
#define LINUX_COMIPLER_H_
|
||||
|
||||
#ifndef __always_inline
|
||||
# define __always_inline inline
|
||||
@@ -9,4 +9,4 @@
|
||||
# define noinline __attribute__((__noinline__))
|
||||
#endif
|
||||
|
||||
#endif // LINUX_COMPILER_H_
|
||||
#endif // LINUX_COMIPLER_H_
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2017-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
CPPFLAGS+= -I../../lib/common
|
||||
CFLAGS ?= -O3
|
||||
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
||||
|
||||
LDFLAGS += -lzstd
|
||||
|
||||
.PHONY: default all clean
|
||||
|
||||
default: all
|
||||
|
||||
all: ldm
|
||||
|
||||
ldm: ldm_common.c ldm.c main.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
|
||||
ldm
|
||||
@echo Cleaning completed
|
||||
@@ -0,0 +1,102 @@
|
||||
This is a compression algorithm focused on finding long distance matches.
|
||||
|
||||
It is based upon lz4 and uses nearly the same block format (github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md). The number of bytes to encode the offset is four instead of two in lz4 to reflect the longer distance matching. The block format is described in `ldm.h`.
|
||||
|
||||
### Build
|
||||
|
||||
Run `make`.
|
||||
|
||||
### Compressing a file
|
||||
|
||||
`ldm <filename>`
|
||||
|
||||
Decompression and verification can be enabled by defining `DECOMPRESS_AND_VERIFY` in `main.c`.
|
||||
The output file names are as follows:
|
||||
- `<filename>.ldm` : compressed file
|
||||
- `<filename>.ldm.dec` : decompressed file
|
||||
|
||||
### Parameters
|
||||
|
||||
There are various parameters that can be tuned. These parameters can be tuned in `ldm.h` or, alternatively if `ldm_params.h` is included, in `ldm_params.h` (for easier configuration).
|
||||
|
||||
The parameters are as follows and must all be defined:
|
||||
- `LDM_MEMORY_USAGE` : the memory usage of the underlying hash table in bytes.
|
||||
- `HASH_BUCKET_SIZE_LOG` : the log size of each bucket in the hash table (used in collision resolution).
|
||||
- `LDM_LAG` : the lag (in bytes) in inserting entries into the hash table.
|
||||
- `LDM_WINDOW_SIZE_LOG` : the log maximum window size when searching for matches.
|
||||
- `LDM_MIN_MATCH_LENGTH` : the minimum match length.
|
||||
- `INSERT_BY_TAG` : insert entries into the hash table as a function of the hash. This increases speed by reducing the number of hash table lookups and match comparisons. Certain hashes will never be inserted.
|
||||
- `USE_CHECKSUM` : store a checksum with the hash table entries for faster comparison. This halves the number of entries the hash table can contain.
|
||||
|
||||
The optional parameter `HASH_ONLY_EVERY_LOG` is the log inverse frequency of insertion into the hash table. That is, an entry is inserted approximately every `1 << HASH_ONLY_EVERY_LOG` times. If this parameter is not defined, the value is computed as a function of the window size and memory usage to approximate an even coverage of the window.
|
||||
|
||||
|
||||
### Benchmark
|
||||
|
||||
Below is a comparison of various compression methods on a tar of four versions of llvm (versions `3.9.0`, `3.9.1`, `4.0.0`, `4.0.1`) with a total size of `727900160` B.
|
||||
|
||||
| Method | Size | Ratio |
|
||||
|:---|---:|---:|
|
||||
|lrzip -p 32 -n -w 1 | `369968714` | `1.97`|
|
||||
|ldm | `209391361` | `3.48`|
|
||||
|lz4 | `189954338` | `3.83`|
|
||||
|lrzip -p 32 -l -w 1 | `163940343` | `4.44`|
|
||||
|zstd -1 | `126080293` | `5.77`|
|
||||
|lrzip -p 32 -n | `124821009` | `5.83`|
|
||||
|lrzip -p 32 -n -w 1 & zstd -1 | `120317909` | `6.05`|
|
||||
|zstd -3 -o | `115290952` | `6.31`|
|
||||
|lrzip -p 32 -g -L 9 -w 1 | `107168979` | `6.79`|
|
||||
|zstd -6 -o | `102772098` | `7.08`|
|
||||
|zstd -T16 -9 | `98040470` | `7.42`|
|
||||
|lrzip -p 32 -n -w 1 & zstd -T32 -19 | `88050289` | `8.27`|
|
||||
|zstd -T32 -19 | `83626098` | `8.70`|
|
||||
|lrzip -p 32 -n & zstd -1 | `36335117` | `20.03`|
|
||||
|ldm & zstd -6 | `32856232` | `22.15`|
|
||||
|lrzip -p 32 -g -L 9 | `32243594` | `22.58`|
|
||||
|lrzip -p 32 -n & zstd -6 | `30954572` | `23.52`|
|
||||
|lrzip -p 32 -n & zstd -T32 -19 | `26472064` | `27.50`|
|
||||
|
||||
The method marked `ldm` was run with the following parameters:
|
||||
|
||||
| Parameter | Value |
|
||||
|:---|---:|
|
||||
| `LDM_MEMORY_USAGE` | `23`|
|
||||
|`HASH_BUCKET_SIZE_LOG` | `3`|
|
||||
|`LDM_LAG` | `0`|
|
||||
|`LDM_WINDOW_SIZE_LOG` | `28`|
|
||||
|`LDM_MIN_MATCH_LENGTH`| `64`|
|
||||
|`INSERT_BY_TAG` | `1`|
|
||||
|`USE_CHECKSUM` | `1`|
|
||||
|
||||
The compression speed was `220.5 MB/s`.
|
||||
|
||||
### Parameter selection
|
||||
|
||||
Below is a brief discussion of the effects of the parameters on the speed and compression ratio.
|
||||
|
||||
#### Speed
|
||||
|
||||
A large bottleneck in terms of speed is finding the matches and comparing to see if they are greater than the minimum match length. Generally:
|
||||
- The fewer matches found (or the lower the percentage of the literals matched), the slower the algorithm will behave.
|
||||
- Increasing `HASH_ONLY_EVERY_LOG` results in fewer inserts and, if `INSERT_BY_TAG` is set, fewer lookups in the table. This has a large effect on speed, as well as compression ratio.
|
||||
- If `HASH_ONLY_EVERY_LOG` is not set, its value is calculated based on `LDM_WINDOW_SIZE_LOG` and `LDM_MEMORY_USAGE`. Increasing `LDM_WINDOW_SIZE_LOG` has the effect of increasing `HASH_ONLY_EVERY_LOG` and increasing `LDM_MEMORY_USAGE` decreases `HASH_ONLY_EVERY_LOG`.
|
||||
- `USE_CHECKSUM` generally improves speed with hash table lookups.
|
||||
|
||||
#### Compression ratio
|
||||
|
||||
The compression ratio is highly correlated with the coverage of matches. As a long distance matcher, the algorithm was designed to "optimize" for long distance matches outside the zstd compression window. The compression ratio after recompressing the output of the long-distance matcher with zstd was a more important signal in development than the raw compression ratio itself.
|
||||
|
||||
Generally, increasing `LDM_MEMORY_USAGE` will improve the compression ratio. However when using the default computed value of `HASH_ONLY_EVERY_LOG`, this increases the frequency of insertion and lookup in the table and thus may result in a decrease in speed.
|
||||
|
||||
Below is a table showing the speed and compression ratio when compressing the llvm tar (as described above) using different settings for `LDM_MEMORY_USAGE`. The other parameters were the same as used in the benchmark above.
|
||||
|
||||
| `LDM_MEMORY_USAGE` | Ratio | Speed (MB/s) | Ratio after zstd -6 |
|
||||
|---:| ---: | ---: | ---: |
|
||||
| `18` | `1.85` | `232.4` | `10.92` |
|
||||
| `21` | `2.79` | `233.9` | `15.92` |
|
||||
| `23` | `3.48` | `220.5` | `18.29` |
|
||||
| `25` | `4.56` | `140.8` | `19.21` |
|
||||
|
||||
### Compression statistics
|
||||
|
||||
Compression statistics (and the configuration) can be enabled/disabled via `COMPUTE_STATS` and `OUTPUT_CONFIGURATION` in `ldm.h`.
|
||||
@@ -0,0 +1,857 @@
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "ldm.h"
|
||||
|
||||
#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
|
||||
#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
|
||||
#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
|
||||
|
||||
#if USE_CHECKSUM
|
||||
#define LDM_HASH_ENTRY_SIZE_LOG 3
|
||||
#else
|
||||
#define LDM_HASH_ENTRY_SIZE_LOG 2
|
||||
#endif
|
||||
|
||||
// Entries are inserted into the table HASH_ONLY_EVERY + 1 times "on average".
|
||||
#ifndef HASH_ONLY_EVERY_LOG
|
||||
#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
|
||||
#endif
|
||||
|
||||
#define HASH_ONLY_EVERY ((1 << (HASH_ONLY_EVERY_LOG)) - 1)
|
||||
|
||||
#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
|
||||
#define NUM_HASH_BUCKETS_LOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
|
||||
|
||||
#define HASH_CHAR_OFFSET 10
|
||||
|
||||
// Take the first match in the hash bucket only.
|
||||
//#define ZSTD_SKIP
|
||||
|
||||
static const U64 prime8bytes = 11400714785074694791ULL;
|
||||
|
||||
// Type of the small hash used to index into the hash table.
|
||||
typedef U32 hash_t;
|
||||
|
||||
#if USE_CHECKSUM
|
||||
typedef struct LDM_hashEntry {
|
||||
U32 offset;
|
||||
U32 checksum;
|
||||
} LDM_hashEntry;
|
||||
#else
|
||||
typedef struct LDM_hashEntry {
|
||||
U32 offset;
|
||||
} LDM_hashEntry;
|
||||
#endif
|
||||
|
||||
struct LDM_compressStats {
|
||||
U32 windowSizeLog, hashTableSizeLog;
|
||||
U32 numMatches;
|
||||
U64 totalMatchLength;
|
||||
U64 totalLiteralLength;
|
||||
U64 totalOffset;
|
||||
|
||||
U32 matchLengthHistogram[32];
|
||||
|
||||
U32 minOffset, maxOffset;
|
||||
U32 offsetHistogram[32];
|
||||
};
|
||||
|
||||
typedef struct LDM_hashTable LDM_hashTable;
|
||||
|
||||
struct LDM_CCtx {
|
||||
size_t isize; /* Input size */
|
||||
size_t maxOSize; /* Maximum output size */
|
||||
|
||||
const BYTE *ibase; /* Base of input */
|
||||
const BYTE *ip; /* Current input position */
|
||||
const BYTE *iend; /* End of input */
|
||||
|
||||
// Maximum input position such that hashing at the position does not exceed
|
||||
// end of input.
|
||||
const BYTE *ihashLimit;
|
||||
|
||||
// Maximum input position such that finding a match of at least the minimum
|
||||
// match length does not exceed end of input.
|
||||
const BYTE *imatchLimit;
|
||||
|
||||
const BYTE *obase; /* Base of output */
|
||||
BYTE *op; /* Output */
|
||||
|
||||
const BYTE *anchor; /* Anchor to start of current (match) block */
|
||||
|
||||
LDM_compressStats stats; /* Compression statistics */
|
||||
|
||||
LDM_hashTable *hashTable;
|
||||
|
||||
const BYTE *lastPosHashed; /* Last position hashed */
|
||||
U64 lastHash;
|
||||
|
||||
const BYTE *nextIp; // TODO: this is redundant (ip + step)
|
||||
const BYTE *nextPosHashed;
|
||||
U64 nextHash;
|
||||
|
||||
unsigned step; // ip step, should be 1.
|
||||
|
||||
const BYTE *lagIp;
|
||||
U64 lagHash;
|
||||
};
|
||||
|
||||
struct LDM_hashTable {
|
||||
U32 numBuckets; // The number of buckets.
|
||||
U32 numEntries; // numBuckets * HASH_BUCKET_SIZE.
|
||||
|
||||
LDM_hashEntry *entries;
|
||||
BYTE *bucketOffsets; // A pointer (per bucket) to the next insert position.
|
||||
};
|
||||
|
||||
static void HASH_destroyTable(LDM_hashTable *table) {
|
||||
free(table->entries);
|
||||
free(table->bucketOffsets);
|
||||
free(table);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a hash table that can contain size elements.
|
||||
* The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
|
||||
*
|
||||
* Returns NULL if table creation failed.
|
||||
*/
|
||||
static LDM_hashTable *HASH_createTable(U32 size) {
|
||||
LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
|
||||
if (!table) return NULL;
|
||||
|
||||
table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
|
||||
table->numEntries = size;
|
||||
table->entries = calloc(size, sizeof(LDM_hashEntry));
|
||||
table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
|
||||
|
||||
if (!table->entries || !table->bucketOffsets) {
|
||||
HASH_destroyTable(table);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
|
||||
static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
|
||||
return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
|
||||
}
|
||||
|
||||
static unsigned ZSTD_NbCommonBytes (register size_t val) {
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanForward64( &r, (U64)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
|
||||
0, 3, 1, 3, 1, 4, 2, 7,
|
||||
0, 2, 3, 6, 1, 5, 3, 5,
|
||||
1, 3, 4, 4, 2, 5, 6, 7,
|
||||
7, 0, 1, 2, 3, 3, 4, 6,
|
||||
2, 6, 5, 5, 3, 4, 5, 6,
|
||||
7, 1, 2, 4, 6, 4, 4, 5,
|
||||
7, 2, 6, 5, 7, 6, 7, 7 };
|
||||
return DeBruijnBytePos[
|
||||
((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r=0;
|
||||
_BitScanForward( &r, (U32)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
|
||||
3, 2, 2, 1, 3, 2, 0, 1,
|
||||
3, 3, 1, 2, 2, 2, 2, 0,
|
||||
3, 1, 2, 0, 1, 0, 1, 1 };
|
||||
return DeBruijnBytePos[
|
||||
((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
} else { /* Big Endian CPU */
|
||||
if (MEM_64bits()) {
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse64( &r, val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
/* calculate this way due to compiler complaining in 32-bits mode */
|
||||
const unsigned n32 = sizeof(size_t)*4;
|
||||
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
|
||||
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
} else { /* 32 bits */
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse( &r, (unsigned long)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// From lib/compress/zstd_compress.c
|
||||
static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
|
||||
const BYTE *const pInLimit) {
|
||||
const BYTE * const pStart = pIn;
|
||||
const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
|
||||
|
||||
while (pIn < pInLoopLimit) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) {
|
||||
pIn += sizeof(size_t);
|
||||
pMatch += sizeof(size_t);
|
||||
continue;
|
||||
}
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
|
||||
if (MEM_64bits()) {
|
||||
if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
|
||||
pIn += 4;
|
||||
pMatch += 4;
|
||||
}
|
||||
}
|
||||
if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
|
||||
pIn += 2;
|
||||
pMatch += 2;
|
||||
}
|
||||
if ((pIn < pInLimit) && (*pMatch == *pIn)) {
|
||||
pIn++;
|
||||
}
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
|
||||
/**
|
||||
* Count number of bytes that match backwards before pIn and pMatch.
|
||||
*
|
||||
* We count only bytes where pMatch > pBase and pIn > pAnchor.
|
||||
*/
|
||||
static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
|
||||
const BYTE *pMatch, const BYTE *pBase) {
|
||||
size_t matchLength = 0;
|
||||
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
|
||||
pIn--;
|
||||
pMatch--;
|
||||
matchLength++;
|
||||
}
|
||||
return matchLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a pointer to the entry in the hash table matching the hash and
|
||||
* checksum with the "longest match length" as defined below. The forward and
|
||||
* backward match lengths are written to *pForwardMatchLength and
|
||||
* *pBackwardMatchLength.
|
||||
*
|
||||
* The match length is defined based on cctx->ip and the entry's offset.
|
||||
* The forward match is computed from cctx->ip and entry->offset + cctx->ibase.
|
||||
* The backward match is computed backwards from cctx->ip and
|
||||
* cctx->ibase only if the forward match is longer than LDM_MIN_MATCH_LENGTH.
|
||||
*/
|
||||
static LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
|
||||
const hash_t hash,
|
||||
const U32 checksum,
|
||||
U64 *pForwardMatchLength,
|
||||
U64 *pBackwardMatchLength) {
|
||||
LDM_hashTable *table = cctx->hashTable;
|
||||
LDM_hashEntry *bucket = getBucket(table, hash);
|
||||
LDM_hashEntry *cur;
|
||||
LDM_hashEntry *bestEntry = NULL;
|
||||
U64 bestMatchLength = 0;
|
||||
#if !(USE_CHECKSUM)
|
||||
(void)checksum;
|
||||
#endif
|
||||
for (cur = bucket; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
|
||||
const BYTE *pMatch = cur->offset + cctx->ibase;
|
||||
|
||||
// Check checksum for faster check.
|
||||
#if USE_CHECKSUM
|
||||
if (cur->checksum == checksum &&
|
||||
cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
|
||||
#else
|
||||
if (cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
|
||||
#endif
|
||||
U64 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
|
||||
U64 backwardMatchLength, totalMatchLength;
|
||||
|
||||
// Only take matches where the forward match length is large enough
|
||||
// for speed.
|
||||
if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
|
||||
continue;
|
||||
}
|
||||
|
||||
backwardMatchLength =
|
||||
countBackwardsMatch(cctx->ip, cctx->anchor,
|
||||
cur->offset + cctx->ibase,
|
||||
cctx->ibase);
|
||||
|
||||
totalMatchLength = forwardMatchLength + backwardMatchLength;
|
||||
|
||||
if (totalMatchLength >= bestMatchLength) {
|
||||
bestMatchLength = totalMatchLength;
|
||||
*pForwardMatchLength = forwardMatchLength;
|
||||
*pBackwardMatchLength = backwardMatchLength;
|
||||
|
||||
bestEntry = cur;
|
||||
#ifdef ZSTD_SKIP
|
||||
return cur;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bestEntry != NULL) {
|
||||
return bestEntry;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert an entry into the hash table. The table uses a "circular buffer",
|
||||
* with the oldest entry overwritten.
|
||||
*/
|
||||
static void HASH_insert(LDM_hashTable *table,
|
||||
const hash_t hash, const LDM_hashEntry entry) {
|
||||
*(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
|
||||
table->bucketOffsets[hash]++;
|
||||
table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
|
||||
}
|
||||
|
||||
static void HASH_outputTableOccupancy(const LDM_hashTable *table) {
|
||||
U32 ctr = 0;
|
||||
LDM_hashEntry *cur = table->entries;
|
||||
LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
|
||||
for (; cur < end; ++cur) {
|
||||
if (cur->offset == 0) {
|
||||
ctr++;
|
||||
}
|
||||
}
|
||||
|
||||
// The number of buckets is repeated as a check for now.
|
||||
printf("Num buckets, bucket size: %d (2^%d), %d\n",
|
||||
table->numBuckets, NUM_HASH_BUCKETS_LOG, HASH_BUCKET_SIZE);
|
||||
printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
|
||||
table->numEntries, ctr,
|
||||
100.0 * (double)(ctr) / table->numEntries);
|
||||
}
|
||||
|
||||
// TODO: This can be done more efficiently, for example by using builtin
|
||||
// functions (but it is not that important as it is only used for computing
|
||||
// stats).
|
||||
static int intLog2(U64 x) {
|
||||
int ret = 0;
|
||||
while (x >>= 1) {
|
||||
ret++;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void LDM_printCompressStats(const LDM_compressStats *stats) {
|
||||
printf("=====================\n");
|
||||
printf("Compression statistics\n");
|
||||
printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
|
||||
stats->windowSizeLog, stats->hashTableSizeLog);
|
||||
printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
|
||||
stats->numMatches,
|
||||
stats->totalMatchLength,
|
||||
100.0 * (double)stats->totalMatchLength /
|
||||
(double)(stats->totalMatchLength + stats->totalLiteralLength));
|
||||
printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
|
||||
(double)stats->numMatches);
|
||||
printf("avg literal length, total literalLength: %.1f, %llu\n",
|
||||
((double)stats->totalLiteralLength) / (double)stats->numMatches,
|
||||
stats->totalLiteralLength);
|
||||
printf("avg offset length: %.1f\n",
|
||||
((double)stats->totalOffset) / (double)stats->numMatches);
|
||||
printf("min offset, max offset: %u, %u\n",
|
||||
stats->minOffset, stats->maxOffset);
|
||||
|
||||
printf("\n");
|
||||
printf("offset histogram | match length histogram\n");
|
||||
printf("offset/ML, num matches, %% of matches | num matches, %% of matches\n");
|
||||
|
||||
{
|
||||
int i;
|
||||
int logMaxOffset = intLog2(stats->maxOffset);
|
||||
for (i = 0; i <= logMaxOffset; i++) {
|
||||
printf("2^%*d: %10u %6.3f%% |2^%*d: %10u %6.3f \n",
|
||||
2, i,
|
||||
stats->offsetHistogram[i],
|
||||
100.0 * (double) stats->offsetHistogram[i] /
|
||||
(double) stats->numMatches,
|
||||
2, i,
|
||||
stats->matchLengthHistogram[i],
|
||||
100.0 * (double) stats->matchLengthHistogram[i] /
|
||||
(double) stats->numMatches);
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
printf("=====================\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the upper (most significant) NUM_HASH_BUCKETS_LOG bits.
|
||||
*/
|
||||
static hash_t getSmallHash(U64 hash) {
|
||||
return hash >> (64 - NUM_HASH_BUCKETS_LOG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the 32 bits after the upper NUM_HASH_BUCKETS_LOG bits.
|
||||
*/
|
||||
static U32 getChecksum(U64 hash) {
|
||||
return (hash >> (64 - 32 - NUM_HASH_BUCKETS_LOG)) & 0xFFFFFFFF;
|
||||
}
|
||||
|
||||
#if INSERT_BY_TAG
|
||||
static U32 lowerBitsFromHfHash(U64 hash) {
|
||||
// The number of bits used so far is NUM_HASH_BUCKETS_LOG + 32.
|
||||
// So there are 32 - NUM_HASH_BUCKETS_LOG bits left.
|
||||
// Occasional hashing requires HASH_ONLY_EVERY_LOG bits.
|
||||
// So if 32 - LDMHASHLOG < HASH_ONLY_EVERY_LOG, just return lower bits
|
||||
// allowing for reuse of bits.
|
||||
if (32 - NUM_HASH_BUCKETS_LOG < HASH_ONLY_EVERY_LOG) {
|
||||
return hash & HASH_ONLY_EVERY;
|
||||
} else {
|
||||
// Otherwise shift by
|
||||
// (32 - NUM_HASH_BUCKETS_LOG - HASH_ONLY_EVERY_LOG) bits first.
|
||||
return (hash >> (32 - NUM_HASH_BUCKETS_LOG - HASH_ONLY_EVERY_LOG)) &
|
||||
HASH_ONLY_EVERY;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Get a 64-bit hash using the first len bytes from buf.
|
||||
*
|
||||
* Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
|
||||
* H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
|
||||
*
|
||||
* where the constant a is defined to be prime8bytes.
|
||||
*
|
||||
* The implementation adds an offset to each byte, so
|
||||
* H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ...
|
||||
*/
|
||||
static U64 getHash(const BYTE *buf, U32 len) {
|
||||
U64 ret = 0;
|
||||
U32 i;
|
||||
for (i = 0; i < len; i++) {
|
||||
ret *= prime8bytes;
|
||||
ret += buf[i] + HASH_CHAR_OFFSET;
|
||||
}
|
||||
return ret;
|
||||
|
||||
}
|
||||
|
||||
static U64 ipow(U64 base, U64 exp) {
|
||||
U64 ret = 1;
|
||||
while (exp) {
|
||||
if (exp & 1) {
|
||||
ret *= base;
|
||||
}
|
||||
exp >>= 1;
|
||||
base *= base;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static U64 updateHash(U64 hash, U32 len,
|
||||
BYTE toRemove, BYTE toAdd) {
|
||||
// TODO: this relies on compiler optimization.
|
||||
// The exponential can be calculated explicitly as len is constant.
|
||||
hash -= ((toRemove + HASH_CHAR_OFFSET) *
|
||||
ipow(prime8bytes, len - 1));
|
||||
hash *= prime8bytes;
|
||||
hash += toAdd + HASH_CHAR_OFFSET;
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cctx->nextHash and cctx->nextPosHashed
|
||||
* based on cctx->lastHash and cctx->lastPosHashed.
|
||||
*
|
||||
* This uses a rolling hash and requires that the last position hashed
|
||||
* corresponds to cctx->nextIp - step.
|
||||
*/
|
||||
static void setNextHash(LDM_CCtx *cctx) {
|
||||
cctx->nextHash = updateHash(
|
||||
cctx->lastHash, LDM_HASH_LENGTH,
|
||||
cctx->lastPosHashed[0],
|
||||
cctx->lastPosHashed[LDM_HASH_LENGTH]);
|
||||
cctx->nextPosHashed = cctx->nextIp;
|
||||
|
||||
#if LDM_LAG
|
||||
if (cctx->ip - cctx->ibase > LDM_LAG) {
|
||||
cctx->lagHash = updateHash(
|
||||
cctx->lagHash, LDM_HASH_LENGTH,
|
||||
cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
|
||||
cctx->lagIp++;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
|
||||
// Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
|
||||
// Note: this works only when cctx->step is 1.
|
||||
#if LDM_LAG
|
||||
if (cctx -> lagIp - cctx->ibase > 0) {
|
||||
#if INSERT_BY_TAG
|
||||
U32 hashEveryMask = lowerBitsFromHfHash(cctx->lagHash);
|
||||
if (hashEveryMask == HASH_ONLY_EVERY) {
|
||||
#else
|
||||
if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
|
||||
#endif
|
||||
U32 smallHash = getSmallHash(cctx->lagHash);
|
||||
|
||||
# if USE_CHECKSUM
|
||||
U32 checksum = getChecksum(cctx->lagHash);
|
||||
const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
|
||||
# else
|
||||
const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
|
||||
# endif
|
||||
|
||||
HASH_insert(cctx->hashTable, smallHash, entry);
|
||||
}
|
||||
} else {
|
||||
#endif // LDM_LAG
|
||||
#if INSERT_BY_TAG
|
||||
U32 hashEveryMask = lowerBitsFromHfHash(hash);
|
||||
if (hashEveryMask == HASH_ONLY_EVERY) {
|
||||
#else
|
||||
if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
|
||||
#endif
|
||||
U32 smallHash = getSmallHash(hash);
|
||||
|
||||
#if USE_CHECKSUM
|
||||
U32 checksum = getChecksum(hash);
|
||||
const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
|
||||
#else
|
||||
const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
|
||||
#endif
|
||||
HASH_insert(cctx->hashTable, smallHash, entry);
|
||||
}
|
||||
#if LDM_LAG
|
||||
}
|
||||
#endif
|
||||
|
||||
cctx->lastPosHashed = cctx->ip;
|
||||
cctx->lastHash = hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy over the cctx->lastHash, and cctx->lastPosHashed
|
||||
* fields from the "next" fields.
|
||||
*
|
||||
* This requires that cctx->ip == cctx->nextPosHashed.
|
||||
*/
|
||||
static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
|
||||
putHashOfCurrentPositionFromHash(cctx, cctx->nextHash);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert hash of the current position into the hash table.
|
||||
*/
|
||||
static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
|
||||
U64 hash = getHash(cctx->ip, LDM_HASH_LENGTH);
|
||||
|
||||
putHashOfCurrentPositionFromHash(cctx, hash);
|
||||
}
|
||||
|
||||
size_t LDM_initializeCCtx(LDM_CCtx *cctx,
|
||||
const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
cctx->isize = srcSize;
|
||||
cctx->maxOSize = maxDstSize;
|
||||
|
||||
cctx->ibase = (const BYTE *)src;
|
||||
cctx->ip = cctx->ibase;
|
||||
cctx->iend = cctx->ibase + srcSize;
|
||||
|
||||
cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
|
||||
cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
|
||||
|
||||
cctx->obase = (BYTE *)dst;
|
||||
cctx->op = (BYTE *)dst;
|
||||
|
||||
cctx->anchor = cctx->ibase;
|
||||
|
||||
memset(&(cctx->stats), 0, sizeof(cctx->stats));
|
||||
#if USE_CHECKSUM
|
||||
cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64);
|
||||
#else
|
||||
cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32);
|
||||
#endif
|
||||
|
||||
if (!cctx->hashTable) return 1;
|
||||
|
||||
cctx->stats.minOffset = UINT_MAX;
|
||||
cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
|
||||
cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
|
||||
|
||||
cctx->lastPosHashed = NULL;
|
||||
|
||||
cctx->step = 1; // Fixed to be 1 for now. Changing may break things.
|
||||
cctx->nextIp = cctx->ip + cctx->step;
|
||||
cctx->nextPosHashed = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void LDM_destroyCCtx(LDM_CCtx *cctx) {
|
||||
HASH_destroyTable(cctx->hashTable);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the "best" match.
|
||||
*
|
||||
* Returns 0 if successful and 1 otherwise (i.e. no match can be found
|
||||
* in the remaining input that is long enough).
|
||||
*
|
||||
* forwardMatchLength contains the forward length of the match.
|
||||
*/
|
||||
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
|
||||
U64 *forwardMatchLength, U64 *backwardMatchLength) {
|
||||
|
||||
LDM_hashEntry *entry = NULL;
|
||||
cctx->nextIp = cctx->ip + cctx->step;
|
||||
|
||||
while (entry == NULL) {
|
||||
U64 hash;
|
||||
hash_t smallHash;
|
||||
U32 checksum;
|
||||
#if INSERT_BY_TAG
|
||||
U32 hashEveryMask;
|
||||
#endif
|
||||
setNextHash(cctx);
|
||||
|
||||
hash = cctx->nextHash;
|
||||
smallHash = getSmallHash(hash);
|
||||
checksum = getChecksum(hash);
|
||||
#if INSERT_BY_TAG
|
||||
hashEveryMask = lowerBitsFromHfHash(hash);
|
||||
#endif
|
||||
|
||||
cctx->ip = cctx->nextIp;
|
||||
cctx->nextIp += cctx->step;
|
||||
|
||||
if (cctx->ip > cctx->imatchLimit) {
|
||||
return 1;
|
||||
}
|
||||
#if INSERT_BY_TAG
|
||||
if (hashEveryMask == HASH_ONLY_EVERY) {
|
||||
|
||||
entry = HASH_getBestEntry(cctx, smallHash, checksum,
|
||||
forwardMatchLength, backwardMatchLength);
|
||||
}
|
||||
#else
|
||||
entry = HASH_getBestEntry(cctx, smallHash, checksum,
|
||||
forwardMatchLength, backwardMatchLength);
|
||||
#endif
|
||||
|
||||
if (entry != NULL) {
|
||||
*match = entry->offset + cctx->ibase;
|
||||
}
|
||||
|
||||
putHashOfCurrentPositionFromHash(cctx, hash);
|
||||
|
||||
}
|
||||
setNextHash(cctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void LDM_encodeLiteralLengthAndLiterals(
|
||||
LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
|
||||
/* Encode the literal length. */
|
||||
if (literalLength >= RUN_MASK) {
|
||||
U64 len = (U64)literalLength - RUN_MASK;
|
||||
*pToken = (RUN_MASK << ML_BITS);
|
||||
for (; len >= 255; len -= 255) {
|
||||
*(cctx->op)++ = 255;
|
||||
}
|
||||
*(cctx->op)++ = (BYTE)len;
|
||||
} else {
|
||||
*pToken = (BYTE)(literalLength << ML_BITS);
|
||||
}
|
||||
|
||||
/* Encode the literals. */
|
||||
memcpy(cctx->op, cctx->anchor, literalLength);
|
||||
cctx->op += literalLength;
|
||||
}
|
||||
|
||||
void LDM_outputBlock(LDM_CCtx *cctx,
|
||||
const U64 literalLength,
|
||||
const U32 offset,
|
||||
const U64 matchLength) {
|
||||
BYTE *pToken = cctx->op++;
|
||||
|
||||
/* Encode the literal length and literals. */
|
||||
LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
|
||||
|
||||
/* Encode the offset. */
|
||||
MEM_write32(cctx->op, offset);
|
||||
cctx->op += LDM_OFFSET_SIZE;
|
||||
|
||||
/* Encode the match length. */
|
||||
if (matchLength >= ML_MASK) {
|
||||
U64 matchLengthRemaining = matchLength;
|
||||
*pToken += ML_MASK;
|
||||
matchLengthRemaining -= ML_MASK;
|
||||
MEM_write32(cctx->op, 0xFFFFFFFF);
|
||||
while (matchLengthRemaining >= 4*0xFF) {
|
||||
cctx->op += 4;
|
||||
MEM_write32(cctx->op, 0xffffffff);
|
||||
matchLengthRemaining -= 4*0xFF;
|
||||
}
|
||||
cctx->op += matchLengthRemaining / 255;
|
||||
*(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
|
||||
} else {
|
||||
*pToken += (BYTE)(matchLength);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: maxDstSize is unused. This function may seg fault when writing
|
||||
// beyond the size of dst, as it does not check maxDstSize. Writing to
|
||||
// a buffer and performing checks is a possible solution.
|
||||
//
|
||||
// This is based upon lz4.
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
LDM_CCtx cctx;
|
||||
const BYTE *match = NULL;
|
||||
U64 forwardMatchLength = 0;
|
||||
U64 backwardsMatchLength = 0;
|
||||
|
||||
if (LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize)) {
|
||||
// Initialization failed.
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef OUTPUT_CONFIGURATION
|
||||
LDM_outputConfiguration();
|
||||
#endif
|
||||
|
||||
/* Hash the first position and put it into the hash table. */
|
||||
LDM_putHashOfCurrentPosition(&cctx);
|
||||
|
||||
cctx.lagIp = cctx.ip;
|
||||
cctx.lagHash = cctx.lastHash;
|
||||
|
||||
/**
|
||||
* Find a match.
|
||||
* If no more matches can be found (i.e. the length of the remaining input
|
||||
* is less than the minimum match length), then stop searching for matches
|
||||
* and encode the final literals.
|
||||
*/
|
||||
while (!LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
|
||||
&backwardsMatchLength)) {
|
||||
|
||||
#ifdef COMPUTE_STATS
|
||||
cctx.stats.numMatches++;
|
||||
#endif
|
||||
|
||||
cctx.ip -= backwardsMatchLength;
|
||||
match -= backwardsMatchLength;
|
||||
|
||||
/**
|
||||
* Write current block (literals, literal length, match offset, match
|
||||
* length) and update pointers and hashes.
|
||||
*/
|
||||
{
|
||||
const U64 literalLength = cctx.ip - cctx.anchor;
|
||||
const U32 offset = cctx.ip - match;
|
||||
const U64 matchLength = forwardMatchLength +
|
||||
backwardsMatchLength -
|
||||
LDM_MIN_MATCH_LENGTH;
|
||||
|
||||
LDM_outputBlock(&cctx, literalLength, offset, matchLength);
|
||||
|
||||
#ifdef COMPUTE_STATS
|
||||
cctx.stats.totalLiteralLength += literalLength;
|
||||
cctx.stats.totalOffset += offset;
|
||||
cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
|
||||
cctx.stats.minOffset =
|
||||
offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
|
||||
cctx.stats.maxOffset =
|
||||
offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
|
||||
cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
|
||||
cctx.stats.matchLengthHistogram[
|
||||
(U32)intLog2(matchLength + LDM_MIN_MATCH_LENGTH)]++;
|
||||
#endif
|
||||
|
||||
// Move ip to end of block, inserting hashes at each position.
|
||||
cctx.nextIp = cctx.ip + cctx.step;
|
||||
while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
|
||||
matchLength + literalLength) {
|
||||
if (cctx.ip > cctx.lastPosHashed) {
|
||||
// TODO: Simplify.
|
||||
LDM_updateLastHashFromNextHash(&cctx);
|
||||
setNextHash(&cctx);
|
||||
}
|
||||
cctx.ip++;
|
||||
cctx.nextIp++;
|
||||
}
|
||||
}
|
||||
|
||||
// Set start of next block to current input pointer.
|
||||
cctx.anchor = cctx.ip;
|
||||
LDM_updateLastHashFromNextHash(&cctx);
|
||||
}
|
||||
|
||||
/* Encode the last literals (no more matches). */
|
||||
{
|
||||
const U64 lastRun = cctx.iend - cctx.anchor;
|
||||
BYTE *pToken = cctx.op++;
|
||||
LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
|
||||
}
|
||||
|
||||
#ifdef COMPUTE_STATS
|
||||
LDM_printCompressStats(&cctx.stats);
|
||||
HASH_outputTableOccupancy(cctx.hashTable);
|
||||
#endif
|
||||
|
||||
{
|
||||
const size_t ret = cctx.op - cctx.obase;
|
||||
LDM_destroyCCtx(&cctx);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
void LDM_outputConfiguration(void) {
|
||||
printf("=====================\n");
|
||||
printf("Configuration\n");
|
||||
printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
|
||||
printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
|
||||
LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
|
||||
printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
|
||||
printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
|
||||
printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
|
||||
printf("LDM_LAG: %d\n", LDM_LAG);
|
||||
printf("USE_CHECKSUM: %d\n", USE_CHECKSUM);
|
||||
printf("INSERT_BY_TAG: %d\n", INSERT_BY_TAG);
|
||||
printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
|
||||
printf("=====================\n");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
#ifndef LDM_H
|
||||
#define LDM_H
|
||||
|
||||
#include "mem.h" // from /lib/common/mem.h
|
||||
|
||||
//#include "ldm_params.h"
|
||||
|
||||
// =============================================================================
|
||||
// Modify the parameters in ldm_params.h if "ldm_params.h" is included.
|
||||
// Otherwise, modify the parameters here.
|
||||
// =============================================================================
|
||||
|
||||
#ifndef LDM_PARAMS_H
|
||||
// Defines the size of the hash table.
|
||||
// Note that this is not the number of buckets.
|
||||
// Currently this should be less than WINDOW_SIZE_LOG + 4.
|
||||
#define LDM_MEMORY_USAGE 23
|
||||
|
||||
// The number of entries in a hash bucket.
|
||||
#define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
|
||||
|
||||
// Defines the lag in inserting elements into the hash table.
|
||||
#define LDM_LAG 0
|
||||
|
||||
// The maximum window size when searching for matches.
|
||||
// The maximum value is 30
|
||||
#define LDM_WINDOW_SIZE_LOG 28
|
||||
|
||||
// The minimum match length.
|
||||
// This should be a multiple of four.
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
|
||||
// If INSERT_BY_TAG, insert entries into the hash table as a function of the
|
||||
// hash. Certain hashes will not be inserted.
|
||||
//
|
||||
// Otherwise, insert as a function of the position.
|
||||
#define INSERT_BY_TAG 1
|
||||
|
||||
// Store a checksum with the hash table entries for faster comparison.
|
||||
// This halves the number of entries the hash table can contain.
|
||||
#define USE_CHECKSUM 1
|
||||
#endif
|
||||
|
||||
// Output compression statistics.
|
||||
#define COMPUTE_STATS
|
||||
|
||||
// Output the configuration.
|
||||
#define OUTPUT_CONFIGURATION
|
||||
|
||||
// If defined, forces the probability of insertion to be approximately
|
||||
// one per (1 << HASH_ONLY_EVERY_LOG). If not defined, the probability will be
|
||||
// calculated based on the memory usage and window size for "even" insertion
|
||||
// throughout the window.
|
||||
|
||||
// #define HASH_ONLY_EVERY_LOG 8
|
||||
|
||||
// =============================================================================
|
||||
|
||||
// The number of bytes storing the compressed and decompressed size
|
||||
// in the header.
|
||||
#define LDM_COMPRESSED_SIZE 8
|
||||
#define LDM_DECOMPRESSED_SIZE 8
|
||||
#define LDM_HEADER_SIZE ((LDM_COMPRESSED_SIZE)+(LDM_DECOMPRESSED_SIZE))
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
#define RUN_BITS (8-ML_BITS)
|
||||
#define RUN_MASK ((1U<<RUN_BITS)-1)
|
||||
|
||||
// The number of bytes storing the offset.
|
||||
#define LDM_OFFSET_SIZE 4
|
||||
|
||||
#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
|
||||
|
||||
// TODO: Match lengths that are too small do not use the hash table efficiently.
|
||||
// There should be a minimum hash length given the hash table size.
|
||||
#define LDM_HASH_LENGTH LDM_MIN_MATCH_LENGTH
|
||||
|
||||
typedef struct LDM_compressStats LDM_compressStats;
|
||||
typedef struct LDM_CCtx LDM_CCtx;
|
||||
typedef struct LDM_DCtx LDM_DCtx;
|
||||
|
||||
/**
|
||||
* Compresses src into dst.
|
||||
* Returns the compressed size if successful, 0 otherwise.
|
||||
*
|
||||
* NB: This currently ignores maxDstSize and assumes enough space is available.
|
||||
*
|
||||
* Block format (see lz4 documentation for more information):
|
||||
* github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
|
||||
*
|
||||
* A block is composed of sequences. Each sequence begins with a token, which
|
||||
* is a one-byte value separated into two 4-bit fields.
|
||||
*
|
||||
* The first field uses the four high bits of the token and encodes the literal
|
||||
* length. If the field value is 0, there is no literal. If it is 15,
|
||||
* additional bytes are added (each ranging from 0 to 255) to the previous
|
||||
* value to produce a total length.
|
||||
*
|
||||
* Following the token and optional length bytes are the literals.
|
||||
*
|
||||
* Next are the 4 bytes representing the offset of the match (2 in lz4),
|
||||
* representing the position to copy the literals.
|
||||
*
|
||||
* The lower four bits of the token encode the match length. With additional
|
||||
* bytes added similarly to the additional literal length bytes after the offset.
|
||||
*
|
||||
* The last sequence is incomplete and stops right after the literals.
|
||||
*/
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
/**
|
||||
* Initialize the compression context.
|
||||
*
|
||||
* Allocates memory for the hash table.
|
||||
*
|
||||
* Returns 0 if successful, 1 otherwise.
|
||||
*/
|
||||
size_t LDM_initializeCCtx(LDM_CCtx *cctx,
|
||||
const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
/**
|
||||
* Frees up memory allocated in LDM_initializeCCtx().
|
||||
*/
|
||||
void LDM_destroyCCtx(LDM_CCtx *cctx);
|
||||
|
||||
/**
|
||||
* Prints the distribution of offsets in the hash table.
|
||||
*
|
||||
* The offsets are defined as the distance of the hash table entry from the
|
||||
* current input position of the cctx.
|
||||
*/
|
||||
void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
|
||||
|
||||
/**
|
||||
* Outputs compression statistics to stdout.
|
||||
*/
|
||||
void LDM_printCompressStats(const LDM_compressStats *stats);
|
||||
|
||||
/**
|
||||
* Encode the literal length followed by the literals.
|
||||
*
|
||||
* The literal length is written to the upper four bits of pToken, with
|
||||
* additional bytes written to the output as needed (see lz4).
|
||||
*
|
||||
* This is followed by literalLength bytes corresponding to the literals.
|
||||
*/
|
||||
void LDM_encodeLiteralLengthAndLiterals(LDM_CCtx *cctx, BYTE *pToken,
|
||||
const U64 literalLength);
|
||||
|
||||
/**
|
||||
* Write current block (literals, literal length, match offset,
|
||||
* match length).
|
||||
*/
|
||||
void LDM_outputBlock(LDM_CCtx *cctx,
|
||||
const U64 literalLength,
|
||||
const U32 offset,
|
||||
const U64 matchLength);
|
||||
|
||||
/**
|
||||
* Decompresses src into dst.
|
||||
*
|
||||
* Note: assumes src does not have a header.
|
||||
*/
|
||||
size_t LDM_decompress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
/**
|
||||
* Initialize the decompression context.
|
||||
*/
|
||||
void LDM_initializeDCtx(LDM_DCtx *dctx,
|
||||
const void *src, size_t compressedSize,
|
||||
void *dst, size_t maxDecompressedSize);
|
||||
|
||||
/**
|
||||
* Reads the header from src and writes the compressed size and
|
||||
* decompressed size into compressedSize and decompressedSize respectively.
|
||||
*
|
||||
* NB: LDM_compress and LDM_decompress currently do not add/read headers.
|
||||
*/
|
||||
void LDM_readHeader(const void *src, U64 *compressedSize,
|
||||
U64 *decompressedSize);
|
||||
|
||||
/**
|
||||
* Write the compressed and decompressed size.
|
||||
*/
|
||||
void LDM_writeHeader(void *memPtr, U64 compressedSize,
|
||||
U64 decompressedSize);
|
||||
|
||||
/**
|
||||
* Output the configuration used.
|
||||
*/
|
||||
void LDM_outputConfiguration(void);
|
||||
|
||||
#endif /* LDM_H */
|
||||
@@ -0,0 +1,109 @@
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ldm.h"
|
||||
|
||||
/**
|
||||
* This function reads the header at the beginning of src and writes
|
||||
* the compressed and decompressed size to compressedSize and
|
||||
* decompressedSize.
|
||||
*
|
||||
* The header consists of 16 bytes: 8 bytes each in little-endian format
|
||||
* of the compressed size and the decompressed size.
|
||||
*/
|
||||
void LDM_readHeader(const void *src, U64 *compressedSize,
|
||||
U64 *decompressedSize) {
|
||||
const BYTE *ip = (const BYTE *)src;
|
||||
*compressedSize = MEM_readLE64(ip);
|
||||
*decompressedSize = MEM_readLE64(ip + 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the 16-byte header (8-bytes each of the compressedSize and
|
||||
* decompressedSize in little-endian format) to memPtr.
|
||||
*/
|
||||
void LDM_writeHeader(void *memPtr, U64 compressedSize,
|
||||
U64 decompressedSize) {
|
||||
MEM_writeLE64(memPtr, compressedSize);
|
||||
MEM_writeLE64((BYTE *)memPtr + 8, decompressedSize);
|
||||
}
|
||||
|
||||
struct LDM_DCtx {
|
||||
size_t compressedSize;
|
||||
size_t maxDecompressedSize;
|
||||
|
||||
const BYTE *ibase; /* Base of input */
|
||||
const BYTE *ip; /* Current input position */
|
||||
const BYTE *iend; /* End of source */
|
||||
|
||||
const BYTE *obase; /* Base of output */
|
||||
BYTE *op; /* Current output position */
|
||||
const BYTE *oend; /* End of output */
|
||||
};
|
||||
|
||||
void LDM_initializeDCtx(LDM_DCtx *dctx,
|
||||
const void *src, size_t compressedSize,
|
||||
void *dst, size_t maxDecompressedSize) {
|
||||
dctx->compressedSize = compressedSize;
|
||||
dctx->maxDecompressedSize = maxDecompressedSize;
|
||||
|
||||
dctx->ibase = src;
|
||||
dctx->ip = (const BYTE *)src;
|
||||
dctx->iend = dctx->ip + dctx->compressedSize;
|
||||
dctx->op = dst;
|
||||
dctx->oend = dctx->op + dctx->maxDecompressedSize;
|
||||
}
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t compressedSize,
|
||||
void *dst, size_t maxDecompressedSize) {
|
||||
|
||||
LDM_DCtx dctx;
|
||||
LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
|
||||
|
||||
while (dctx.ip < dctx.iend) {
|
||||
BYTE *cpy;
|
||||
const BYTE *match;
|
||||
size_t length, offset;
|
||||
|
||||
/* Get the literal length. */
|
||||
const unsigned token = *(dctx.ip)++;
|
||||
if ((length = (token >> ML_BITS)) == RUN_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *(dctx.ip)++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
|
||||
/* Copy the literals. */
|
||||
cpy = dctx.op + length;
|
||||
memcpy(dctx.op, dctx.ip, length);
|
||||
dctx.ip += length;
|
||||
dctx.op = cpy;
|
||||
|
||||
//TODO: dynamic offset size?
|
||||
/* Encode the offset. */
|
||||
offset = MEM_read32(dctx.ip);
|
||||
dctx.ip += LDM_OFFSET_SIZE;
|
||||
match = dctx.op - offset;
|
||||
|
||||
/* Get the match length. */
|
||||
length = token & ML_MASK;
|
||||
if (length == ML_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *(dctx.ip)++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
length += LDM_MIN_MATCH_LENGTH;
|
||||
|
||||
/* Copy match. */
|
||||
cpy = dctx.op + length;
|
||||
|
||||
// TODO: this can be made more efficient.
|
||||
while (match < cpy - offset && dctx.op < dctx.oend) {
|
||||
*(dctx.op)++ = *match++;
|
||||
}
|
||||
}
|
||||
return dctx.op - (BYTE *)dst;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef LDM_PARAMS_H
|
||||
#define LDM_PARAMS_H
|
||||
|
||||
#define LDM_MEMORY_USAGE 23
|
||||
#define HASH_BUCKET_SIZE_LOG 3
|
||||
#define LDM_LAG 0
|
||||
#define LDM_WINDOW_SIZE_LOG 28
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define INSERT_BY_TAG 1
|
||||
#define USE_CHECKSUM 1
|
||||
|
||||
#endif // LDM_PARAMS_H
|
||||
@@ -0,0 +1,269 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zstd.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include "ldm.h"
|
||||
#include "zstd.h"
|
||||
|
||||
// #define DECOMPRESS_AND_VERIFY
|
||||
|
||||
/* Compress file given by fname and output to oname.
|
||||
* Returns 0 if successful, error code otherwise.
|
||||
*
|
||||
* This adds a header from LDM_writeHeader to the beginning of the output.
|
||||
*
|
||||
* This might seg fault if the compressed size is > the decompress
|
||||
* size due to the mmapping and output file size allocated to be the input size
|
||||
* The compress function should check before writing or buffer writes.
|
||||
*/
|
||||
static int compress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
size_t maxCompressedSize, compressedSize;
|
||||
|
||||
struct timeval tv1, tv2;
|
||||
double timeTaken;
|
||||
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
maxCompressedSize = (statbuf.st_size + LDM_HEADER_SIZE);
|
||||
|
||||
// Handle case where compressed size is > decompressed size.
|
||||
// TODO: The compress function should check before writing or buffer writes.
|
||||
maxCompressedSize += statbuf.st_size / 255;
|
||||
|
||||
ftruncate(fdout, maxCompressedSize);
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file. */
|
||||
if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
gettimeofday(&tv1, NULL);
|
||||
|
||||
compressedSize = LDM_HEADER_SIZE +
|
||||
LDM_compress(src, statbuf.st_size,
|
||||
dst + LDM_HEADER_SIZE, maxCompressedSize);
|
||||
|
||||
gettimeofday(&tv2, NULL);
|
||||
|
||||
// Write the header.
|
||||
LDM_writeHeader(dst, compressedSize, statbuf.st_size);
|
||||
|
||||
// Truncate file to compressedSize.
|
||||
ftruncate(fdout, compressedSize);
|
||||
|
||||
printf("%25s : %10lu -> %10lu - %s \n", fname,
|
||||
(size_t)statbuf.st_size, (size_t)compressedSize, oname);
|
||||
printf("Compression ratio: %.2fx --- %.1f%%\n",
|
||||
(double)statbuf.st_size / (double)compressedSize,
|
||||
(double)compressedSize / (double)(statbuf.st_size) * 100.0);
|
||||
|
||||
timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec),
|
||||
|
||||
printf("Total compress time = %.3f seconds, Average scanning speed: %.3f MB/s\n",
|
||||
timeTaken,
|
||||
((double)statbuf.st_size / (double) (1 << 20)) / timeTaken);
|
||||
|
||||
// Close files.
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef DECOMPRESS_AND_VERIFY
|
||||
/* Decompress file compressed using LDM_compress.
|
||||
* The input file should have the LDM_HEADER followed by payload.
|
||||
* Returns 0 if succesful, and an error code otherwise.
|
||||
*/
|
||||
static int decompress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
U64 compressedSize, decompressedSize;
|
||||
size_t outSize;
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Read the header. */
|
||||
LDM_readHeader(src, &compressedSize, &decompressedSize);
|
||||
|
||||
ftruncate(fdout, decompressedSize);
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
outSize = LDM_decompress(
|
||||
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
|
||||
dst, decompressedSize);
|
||||
printf("Ret size out: %zu\n", outSize);
|
||||
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Compare two files.
|
||||
* Returns 0 iff they are the same.
|
||||
*/
|
||||
static int compare(FILE *fp0, FILE *fp1) {
|
||||
int result = 0;
|
||||
while (result == 0) {
|
||||
char b0[1024];
|
||||
char b1[1024];
|
||||
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
|
||||
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
|
||||
|
||||
result = (int)r0 - (int)r1;
|
||||
|
||||
if (0 == r0 || 0 == r1) break;
|
||||
|
||||
if (0 == result) result = memcmp(b0, b1, r0);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Verify the input file is the same as the decompressed file. */
|
||||
static int verify(const char *inpFilename, const char *decFilename) {
|
||||
FILE *inpFp, *decFp;
|
||||
|
||||
if ((inpFp = fopen(inpFilename, "rb")) == NULL) {
|
||||
perror("Could not open input file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if ((decFp = fopen(decFilename, "rb")) == NULL) {
|
||||
perror("Could not open decompressed file\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
{
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
const char * const exeName = argv[0];
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Wrong arguments\n");
|
||||
printf("Usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
/* Compress */
|
||||
{
|
||||
if (compress(inpFilename, ldmFilename)) {
|
||||
printf("Compress error\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef DECOMPRESS_AND_VERIFY
|
||||
/* Decompress */
|
||||
{
|
||||
struct timeval tv1, tv2;
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (decompress(ldmFilename, decFilename)) {
|
||||
printf("Decompress error\n");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total decompress time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
}
|
||||
/* verify */
|
||||
if (verify(inpFilename, decFilename)) {
|
||||
printf("Verification error\n");
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
## Edit Distance Match Finder
|
||||
|
||||
```
|
||||
/* This match finder leverages techniques used in file comparison algorithms
|
||||
* to find matches between a dictionary and a source file.
|
||||
*
|
||||
* The original motivation for studying this approach was to try and optimize
|
||||
* Zstandard for the use case of patching: the most common scenario being
|
||||
* updating an existing software package with the next version. When patching,
|
||||
* the difference between the old version of the package and the new version
|
||||
* is generally tiny (most of the new file will be identical to
|
||||
* the old one). In more technical terms, the edit distance (the minimal number
|
||||
* of changes required to take one sequence of bytes to another) between the
|
||||
* files would be small relative to the size of the file.
|
||||
*
|
||||
* Various 'diffing' algorithms utilize this notion of edit distance and
|
||||
* the corrensponding concept of a minimal edit script between two
|
||||
* sequences to identify the regions within two files where they differ.
|
||||
* The core algorithm used in this match finder is described in:
|
||||
*
|
||||
* "An O(ND) Difference Algorithm and its Variations", Eugene W. Myers,
|
||||
* Algorithmica Vol. 1, 1986, pp. 251-266,
|
||||
* <https://doi.org/10.1007/BF01840446>.
|
||||
*
|
||||
* Additional algorithmic heuristics for speed improvement have also been included.
|
||||
* These we inspired from implementations of various regular and binary diffing
|
||||
* algorithms such as GNU diff, bsdiff, and Xdelta.
|
||||
*
|
||||
* Note: after some experimentation, this approach proved to not provide enough
|
||||
* utility to justify the additional CPU used in finding matches. The one area
|
||||
* where this approach consistenly outperforms Zstandard even on level 19 is
|
||||
* when compressing small files (<10 KB) using a equally small dictionary that
|
||||
* is very similar to the source file. For the use case that this was intended,
|
||||
* (large similar files) this approach by itself took 5-10X longer than zstd-19 and
|
||||
* generally resulted in 2-3X larger files. The core advantage that zstd-19 has
|
||||
* over this appraoch for match finding is the overlapping matches. This approach
|
||||
* cannot find any.
|
||||
*
|
||||
* I'm leaving this in the contrib section in case this ever becomes interesting
|
||||
* to explore again.
|
||||
* */
|
||||
```
|
||||
@@ -1,558 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
|
||||
/* Currently relies on qsort when combining contiguous matches. This can probabily
|
||||
* be avoided but would require changes to the algorithm. The qsort is far from
|
||||
* the bottleneck in this algorithm even for medium sized files so it's probably
|
||||
* not worth trying to address */
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "zstd_edist.h"
|
||||
#include "mem.h"
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
|
||||
/* Just a sential for the entires of the diagnomal matrix */
|
||||
#define ZSTD_EDIST_DIAG_MAX (S32)(1 << 30)
|
||||
|
||||
/* How large should a snake be to be considered a 'big' snake.
|
||||
* For an explanation of what a 'snake' is with respect to the
|
||||
* edit distance matrix, see the linked paper in zstd_edist.h */
|
||||
#define ZSTD_EDIST_SNAKE_THRESH 20
|
||||
|
||||
/* After how many iterations should we start to use the heuristic
|
||||
* based on 'big' snakes */
|
||||
#define ZSTD_EDIST_SNAKE_ITER_THRESH 200
|
||||
|
||||
/* After how many iterations should be just give up and take
|
||||
* the best availabe edit script for this round */
|
||||
#define ZSTD_EDIST_EXPENSIVE_THRESH 1024
|
||||
|
||||
/*-*************************************
|
||||
* Structures
|
||||
***************************************/
|
||||
|
||||
typedef struct {
|
||||
U32 dictIdx;
|
||||
U32 srcIdx;
|
||||
U32 matchLength;
|
||||
} ZSTD_eDist_match;
|
||||
|
||||
typedef struct {
|
||||
const BYTE* dict;
|
||||
const BYTE* src;
|
||||
size_t dictSize;
|
||||
size_t srcSize;
|
||||
S32* forwardDiag; /* Entires of the forward diagonal stored here */
|
||||
S32* backwardDiag; /* Entires of the backward diagonal stored here.
|
||||
* Note: this buffer and the 'forwardDiag' buffer
|
||||
* are contiguous. See the ZSTD_eDist_genSequences */
|
||||
ZSTD_eDist_match* matches; /* Accumulate matches of length 1 in this buffer.
|
||||
* In a subsequence post-processing step, we combine
|
||||
* contiguous matches. */
|
||||
U32 nbMatches;
|
||||
} ZSTD_eDist_state;
|
||||
|
||||
typedef struct {
|
||||
S32 dictMid; /* The mid diagonal for the dictionary */
|
||||
S32 srcMid; /* The mid diagonal for the source */
|
||||
int lowUseHeuristics; /* Should we use heuristics for the low part */
|
||||
int highUseHeuristics; /* Should we use heuristics for the high part */
|
||||
} ZSTD_eDist_partition;
|
||||
|
||||
/*-*************************************
|
||||
* Internal
|
||||
***************************************/
|
||||
|
||||
static void ZSTD_eDist_diag(ZSTD_eDist_state* state,
|
||||
ZSTD_eDist_partition* partition,
|
||||
S32 dictLow, S32 dictHigh, S32 srcLow,
|
||||
S32 srcHigh, int useHeuristics)
|
||||
{
|
||||
S32* const forwardDiag = state->forwardDiag;
|
||||
S32* const backwardDiag = state->backwardDiag;
|
||||
const BYTE* const dict = state->dict;
|
||||
const BYTE* const src = state->src;
|
||||
|
||||
S32 const diagMin = dictLow - srcHigh;
|
||||
S32 const diagMax = dictHigh - srcLow;
|
||||
S32 const forwardMid = dictLow - srcLow;
|
||||
S32 const backwardMid = dictHigh - srcHigh;
|
||||
|
||||
S32 forwardMin = forwardMid;
|
||||
S32 forwardMax = forwardMid;
|
||||
S32 backwardMin = backwardMid;
|
||||
S32 backwardMax = backwardMid;
|
||||
int odd = (forwardMid - backwardMid) & 1;
|
||||
U32 iterations;
|
||||
|
||||
forwardDiag[forwardMid] = dictLow;
|
||||
backwardDiag[backwardMid] = dictHigh;
|
||||
|
||||
/* Main loop for updating diag entries. Unless useHeuristics is
|
||||
* set to false, this loop will run until it finds the minimal
|
||||
* edit script */
|
||||
for (iterations = 1;;iterations++) {
|
||||
S32 diag;
|
||||
int bigSnake = 0;
|
||||
|
||||
if (forwardMin > diagMin) {
|
||||
forwardMin--;
|
||||
forwardDiag[forwardMin - 1] = -1;
|
||||
} else {
|
||||
forwardMin++;
|
||||
}
|
||||
|
||||
if (forwardMax < diagMax) {
|
||||
forwardMax++;
|
||||
forwardDiag[forwardMax + 1] = -1;
|
||||
} else {
|
||||
forwardMax--;
|
||||
}
|
||||
|
||||
for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
|
||||
S32 dictIdx;
|
||||
S32 srcIdx;
|
||||
S32 low = forwardDiag[diag - 1];
|
||||
S32 high = forwardDiag[diag + 1];
|
||||
S32 dictIdx0 = low < high ? high : low + 1;
|
||||
|
||||
for (dictIdx = dictIdx0, srcIdx = dictIdx0 - diag;
|
||||
dictIdx < dictHigh && srcIdx < srcHigh && dict[dictIdx] == src[srcIdx];
|
||||
dictIdx++, srcIdx++) continue;
|
||||
|
||||
if (dictIdx - dictIdx0 > ZSTD_EDIST_SNAKE_THRESH)
|
||||
bigSnake = 1;
|
||||
|
||||
forwardDiag[diag] = dictIdx;
|
||||
|
||||
if (odd && backwardMin <= diag && diag <= backwardMax && backwardDiag[diag] <= dictIdx) {
|
||||
partition->dictMid = dictIdx;
|
||||
partition->srcMid = srcIdx;
|
||||
partition->lowUseHeuristics = 0;
|
||||
partition->highUseHeuristics = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (backwardMin > diagMin) {
|
||||
backwardMin--;
|
||||
backwardDiag[backwardMin - 1] = ZSTD_EDIST_DIAG_MAX;
|
||||
} else {
|
||||
backwardMin++;
|
||||
}
|
||||
|
||||
if (backwardMax < diagMax) {
|
||||
backwardMax++;
|
||||
backwardDiag[backwardMax + 1] = ZSTD_EDIST_DIAG_MAX;
|
||||
} else {
|
||||
backwardMax--;
|
||||
}
|
||||
|
||||
|
||||
for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
|
||||
S32 dictIdx;
|
||||
S32 srcIdx;
|
||||
S32 low = backwardDiag[diag - 1];
|
||||
S32 high = backwardDiag[diag + 1];
|
||||
S32 dictIdx0 = low < high ? low : high - 1;
|
||||
|
||||
for (dictIdx = dictIdx0, srcIdx = dictIdx0 - diag;
|
||||
dictLow < dictIdx && srcLow < srcIdx && dict[dictIdx - 1] == src[srcIdx - 1];
|
||||
dictIdx--, srcIdx--) continue;
|
||||
|
||||
if (dictIdx0 - dictIdx > ZSTD_EDIST_SNAKE_THRESH)
|
||||
bigSnake = 1;
|
||||
|
||||
backwardDiag[diag] = dictIdx;
|
||||
|
||||
if (!odd && forwardMin <= diag && diag <= forwardMax && dictIdx <= forwardDiag[diag]) {
|
||||
partition->dictMid = dictIdx;
|
||||
partition->srcMid = srcIdx;
|
||||
partition->lowUseHeuristics = 0;
|
||||
partition->highUseHeuristics = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!useHeuristics)
|
||||
continue;
|
||||
|
||||
/* Everything under this point is a heuritic. Using these will
|
||||
* substantially speed up the match finding. In some cases, taking
|
||||
* the total match finding time from several minutes to seconds.
|
||||
* Of course, the caveat is that the edit script found may no longer
|
||||
* be optimal */
|
||||
|
||||
/* Big snake heuristic */
|
||||
if (iterations > ZSTD_EDIST_SNAKE_ITER_THRESH && bigSnake) {
|
||||
{
|
||||
S32 best = 0;
|
||||
|
||||
for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
|
||||
S32 diagDiag = diag - forwardMid;
|
||||
S32 dictIdx = forwardDiag[diag];
|
||||
S32 srcIdx = dictIdx - diag;
|
||||
S32 v = (dictIdx - dictLow) * 2 - diagDiag;
|
||||
|
||||
if (v > 12 * (iterations + (diagDiag < 0 ? -diagDiag : diagDiag))) {
|
||||
if (v > best
|
||||
&& dictLow + ZSTD_EDIST_SNAKE_THRESH <= dictIdx && dictIdx <= dictHigh
|
||||
&& srcLow + ZSTD_EDIST_SNAKE_THRESH <= srcIdx && srcIdx <= srcHigh) {
|
||||
S32 k;
|
||||
for (k = 1; dict[dictIdx - k] == src[srcIdx - k]; k++) {
|
||||
if (k == ZSTD_EDIST_SNAKE_THRESH) {
|
||||
best = v;
|
||||
partition->dictMid = dictIdx;
|
||||
partition->srcMid = srcIdx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (best > 0) {
|
||||
partition->lowUseHeuristics = 0;
|
||||
partition->highUseHeuristics = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
S32 best = 0;
|
||||
|
||||
for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
|
||||
S32 diagDiag = diag - backwardMid;
|
||||
S32 dictIdx = backwardDiag[diag];
|
||||
S32 srcIdx = dictIdx - diag;
|
||||
S32 v = (dictHigh - dictIdx) * 2 + diagDiag;
|
||||
|
||||
if (v > 12 * (iterations + (diagDiag < 0 ? -diagDiag : diagDiag))) {
|
||||
if (v > best
|
||||
&& dictLow < dictIdx && dictIdx <= dictHigh - ZSTD_EDIST_SNAKE_THRESH
|
||||
&& srcLow < srcIdx && srcIdx <= srcHigh - ZSTD_EDIST_SNAKE_THRESH) {
|
||||
int k;
|
||||
for (k = 0; dict[dictIdx + k] == src[srcIdx + k]; k++) {
|
||||
if (k == ZSTD_EDIST_SNAKE_THRESH - 1) {
|
||||
best = v;
|
||||
partition->dictMid = dictIdx;
|
||||
partition->srcMid = srcIdx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (best > 0) {
|
||||
partition->lowUseHeuristics = 1;
|
||||
partition->highUseHeuristics = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* More general 'too expensive' heuristic */
|
||||
if (iterations >= ZSTD_EDIST_EXPENSIVE_THRESH) {
|
||||
S32 forwardDictSrcBest;
|
||||
S32 forwardDictBest = 0;
|
||||
S32 backwardDictSrcBest;
|
||||
S32 backwardDictBest = 0;
|
||||
|
||||
forwardDictSrcBest = -1;
|
||||
for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
|
||||
S32 dictIdx = MIN(forwardDiag[diag], dictHigh);
|
||||
S32 srcIdx = dictIdx - diag;
|
||||
|
||||
if (srcHigh < srcIdx) {
|
||||
dictIdx = srcHigh + diag;
|
||||
srcIdx = srcHigh;
|
||||
}
|
||||
|
||||
if (forwardDictSrcBest < dictIdx + srcIdx) {
|
||||
forwardDictSrcBest = dictIdx + srcIdx;
|
||||
forwardDictBest = dictIdx;
|
||||
}
|
||||
}
|
||||
|
||||
backwardDictSrcBest = ZSTD_EDIST_DIAG_MAX;
|
||||
for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
|
||||
S32 dictIdx = MAX(dictLow, backwardDiag[diag]);
|
||||
S32 srcIdx = dictIdx - diag;
|
||||
|
||||
if (srcIdx < srcLow) {
|
||||
dictIdx = srcLow + diag;
|
||||
srcIdx = srcLow;
|
||||
}
|
||||
|
||||
if (dictIdx + srcIdx < backwardDictSrcBest) {
|
||||
backwardDictSrcBest = dictIdx + srcIdx;
|
||||
backwardDictBest = dictIdx;
|
||||
}
|
||||
}
|
||||
|
||||
if ((dictHigh + srcHigh) - backwardDictSrcBest < forwardDictSrcBest - (dictLow + srcLow)) {
|
||||
partition->dictMid = forwardDictBest;
|
||||
partition->srcMid = forwardDictSrcBest - forwardDictBest;
|
||||
partition->lowUseHeuristics = 0;
|
||||
partition->highUseHeuristics = 1;
|
||||
} else {
|
||||
partition->dictMid = backwardDictBest;
|
||||
partition->srcMid = backwardDictSrcBest - backwardDictBest;
|
||||
partition->lowUseHeuristics = 1;
|
||||
partition->highUseHeuristics = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void ZSTD_eDist_insertMatch(ZSTD_eDist_state* state,
|
||||
S32 const dictIdx, S32 const srcIdx)
|
||||
{
|
||||
state->matches[state->nbMatches].dictIdx = dictIdx;
|
||||
state->matches[state->nbMatches].srcIdx = srcIdx;
|
||||
state->matches[state->nbMatches].matchLength = 1;
|
||||
state->nbMatches++;
|
||||
}
|
||||
|
||||
static int ZSTD_eDist_compare(ZSTD_eDist_state* state,
|
||||
S32 dictLow, S32 dictHigh, S32 srcLow,
|
||||
S32 srcHigh, int useHeuristics)
|
||||
{
|
||||
const BYTE* const dict = state->dict;
|
||||
const BYTE* const src = state->src;
|
||||
|
||||
/* Found matches while traversing from the low end */
|
||||
while (dictLow < dictHigh && srcLow < srcHigh && dict[dictLow] == src[srcLow]) {
|
||||
ZSTD_eDist_insertMatch(state, dictLow, srcLow);
|
||||
dictLow++;
|
||||
srcLow++;
|
||||
}
|
||||
|
||||
/* Found matches while traversing from the high end */
|
||||
while (dictLow < dictHigh && srcLow < srcHigh && dict[dictHigh - 1] == src[srcHigh - 1]) {
|
||||
ZSTD_eDist_insertMatch(state, dictHigh - 1, srcHigh - 1);
|
||||
dictHigh--;
|
||||
srcHigh--;
|
||||
}
|
||||
|
||||
/* If the low and high end end up touching. If we wanted to make
|
||||
* note of the differences like most diffing algorithms do, we would
|
||||
* do so here. In our case, we're only concerned with matches
|
||||
* Note: if you wanted to find the edit distance of the algorithm,
|
||||
* you could just accumulate the cost for an insertion/deletion
|
||||
* below. */
|
||||
if (dictLow == dictHigh) {
|
||||
while (srcLow < srcHigh) {
|
||||
/* Reaching this point means inserting src[srcLow] into
|
||||
* the current position of dict */
|
||||
srcLow++;
|
||||
}
|
||||
} else if (srcLow == srcHigh) {
|
||||
while (dictLow < dictHigh) {
|
||||
/* Reaching this point means deleteing dict[dictLow] from
|
||||
* the current positino of dict */
|
||||
dictLow++;
|
||||
}
|
||||
} else {
|
||||
ZSTD_eDist_partition partition;
|
||||
partition.dictMid = 0;
|
||||
partition.srcMid = 0;
|
||||
ZSTD_eDist_diag(state, &partition, dictLow, dictHigh,
|
||||
srcLow, srcHigh, useHeuristics);
|
||||
if (ZSTD_eDist_compare(state, dictLow, partition.dictMid,
|
||||
srcLow, partition.srcMid, partition.lowUseHeuristics))
|
||||
return 1;
|
||||
if (ZSTD_eDist_compare(state, partition.dictMid, dictHigh,
|
||||
partition.srcMid, srcHigh, partition.highUseHeuristics))
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ZSTD_eDist_matchComp(const void* p, const void* q)
|
||||
{
|
||||
S32 const l = ((ZSTD_eDist_match*)p)->srcIdx;
|
||||
S32 const r = ((ZSTD_eDist_match*)q)->srcIdx;
|
||||
return (l - r);
|
||||
}
|
||||
|
||||
/* The matches from the approach above will all be of the form
|
||||
* (dictIdx, srcIdx, 1). this method combines contiguous matches
|
||||
* of length MINMATCH or greater. Matches less than MINMATCH
|
||||
* are discarded */
|
||||
static void ZSTD_eDist_combineMatches(ZSTD_eDist_state* state)
|
||||
{
|
||||
/* Create a new buffer to put the combined matches into
|
||||
* and memcpy to state->matches after */
|
||||
ZSTD_eDist_match* combinedMatches =
|
||||
ZSTD_malloc(state->nbMatches * sizeof(ZSTD_eDist_match),
|
||||
ZSTD_defaultCMem);
|
||||
|
||||
U32 nbCombinedMatches = 1;
|
||||
size_t i;
|
||||
|
||||
/* Make sure that the srcIdx and dictIdx are in sorted order.
|
||||
* The combination step won't work otherwise */
|
||||
qsort(state->matches, state->nbMatches, sizeof(ZSTD_eDist_match), ZSTD_eDist_matchComp);
|
||||
|
||||
memcpy(combinedMatches, state->matches, sizeof(ZSTD_eDist_match));
|
||||
for (i = 1; i < state->nbMatches; i++) {
|
||||
ZSTD_eDist_match const match = state->matches[i];
|
||||
ZSTD_eDist_match const combinedMatch =
|
||||
combinedMatches[nbCombinedMatches - 1];
|
||||
if (combinedMatch.srcIdx + combinedMatch.matchLength == match.srcIdx &&
|
||||
combinedMatch.dictIdx + combinedMatch.matchLength == match.dictIdx) {
|
||||
combinedMatches[nbCombinedMatches - 1].matchLength++;
|
||||
} else {
|
||||
/* Discard matches that are less than MINMATCH */
|
||||
if (combinedMatches[nbCombinedMatches - 1].matchLength < MINMATCH) {
|
||||
nbCombinedMatches--;
|
||||
}
|
||||
|
||||
memcpy(combinedMatches + nbCombinedMatches,
|
||||
state->matches + i, sizeof(ZSTD_eDist_match));
|
||||
nbCombinedMatches++;
|
||||
}
|
||||
}
|
||||
memcpy(state->matches, combinedMatches, nbCombinedMatches * sizeof(ZSTD_eDist_match));
|
||||
state->nbMatches = nbCombinedMatches;
|
||||
ZSTD_free(combinedMatches, ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
static size_t ZSTD_eDist_convertMatchesToSequences(ZSTD_Sequence* sequences,
|
||||
ZSTD_eDist_state* state)
|
||||
{
|
||||
const ZSTD_eDist_match* matches = state->matches;
|
||||
size_t const nbMatches = state->nbMatches;
|
||||
size_t const dictSize = state->dictSize;
|
||||
size_t nbSequences = 0;
|
||||
size_t i;
|
||||
for (i = 0; i < nbMatches; i++) {
|
||||
ZSTD_eDist_match const match = matches[i];
|
||||
U32 const litLength = !i ? match.srcIdx :
|
||||
match.srcIdx - (matches[i - 1].srcIdx + matches[i - 1].matchLength);
|
||||
U32 const offset = (match.srcIdx + dictSize) - match.dictIdx;
|
||||
U32 const matchLength = match.matchLength;
|
||||
sequences[nbSequences].offset = offset;
|
||||
sequences[nbSequences].litLength = litLength;
|
||||
sequences[nbSequences].matchLength = matchLength;
|
||||
nbSequences++;
|
||||
}
|
||||
return nbSequences;
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Interal utils
|
||||
***************************************/
|
||||
|
||||
static size_t ZSTD_eDist_hamingDist(const BYTE* const a,
|
||||
const BYTE* const b, size_t n)
|
||||
{
|
||||
size_t i;
|
||||
size_t dist = 0;
|
||||
for (i = 0; i < n; i++)
|
||||
dist += a[i] != b[i];
|
||||
return dist;
|
||||
}
|
||||
|
||||
/* This is a pretty naive recursive implementation that should only
|
||||
* be used for quick tests obviously. Don't try and run this on a
|
||||
* GB file or something. There are faster implementations. Use those
|
||||
* if you need to run it for large files. */
|
||||
static size_t ZSTD_eDist_levenshteinDist(const BYTE* const s,
|
||||
size_t const sn, const BYTE* const t,
|
||||
size_t const tn)
|
||||
{
|
||||
size_t a, b, c;
|
||||
|
||||
if (!sn)
|
||||
return tn;
|
||||
if (!tn)
|
||||
return sn;
|
||||
|
||||
if (s[sn - 1] == t[tn - 1])
|
||||
return ZSTD_eDist_levenshteinDist(
|
||||
s, sn - 1, t, tn - 1);
|
||||
|
||||
a = ZSTD_eDist_levenshteinDist(s, sn - 1, t, tn - 1);
|
||||
b = ZSTD_eDist_levenshteinDist(s, sn, t, tn - 1);
|
||||
c = ZSTD_eDist_levenshteinDist(s, sn - 1, t, tn);
|
||||
|
||||
if (a > b)
|
||||
a = b;
|
||||
if (a > c)
|
||||
a = c;
|
||||
|
||||
return a + 1;
|
||||
}
|
||||
|
||||
static void ZSTD_eDist_validateMatches(ZSTD_eDist_match* matches,
|
||||
size_t const nbMatches, const BYTE* const dict,
|
||||
size_t const dictSize, const BYTE* const src,
|
||||
size_t const srcSize)
|
||||
{
|
||||
size_t i;
|
||||
for (i = 0; i < nbMatches; i++) {
|
||||
ZSTD_eDist_match match = matches[i];
|
||||
U32 const dictIdx = match.dictIdx;
|
||||
U32 const srcIdx = match.srcIdx;
|
||||
U32 const matchLength = match.matchLength;
|
||||
|
||||
assert(dictIdx + matchLength < dictSize);
|
||||
assert(srcIdx + matchLength < srcSize);
|
||||
assert(!memcmp(dict + dictIdx, src + srcIdx, matchLength));
|
||||
}
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* API
|
||||
***************************************/
|
||||
|
||||
size_t ZSTD_eDist_genSequences(ZSTD_Sequence* sequences,
|
||||
const void* dict, size_t dictSize,
|
||||
const void* src, size_t srcSize,
|
||||
int useHeuristics)
|
||||
{
|
||||
size_t const nbDiags = dictSize + srcSize + 3;
|
||||
S32* buffer = ZSTD_malloc(nbDiags * 2 * sizeof(S32), ZSTD_defaultCMem);
|
||||
ZSTD_eDist_state state;
|
||||
size_t nbSequences = 0;
|
||||
|
||||
state.dict = (const BYTE*)dict;
|
||||
state.src = (const BYTE*)src;
|
||||
state.dictSize = dictSize;
|
||||
state.srcSize = srcSize;
|
||||
state.forwardDiag = buffer;
|
||||
state.backwardDiag = buffer + nbDiags;
|
||||
state.forwardDiag += srcSize + 1;
|
||||
state.backwardDiag += srcSize + 1;
|
||||
state.matches = ZSTD_malloc(srcSize * sizeof(ZSTD_eDist_match), ZSTD_defaultCMem);
|
||||
state.nbMatches = 0;
|
||||
|
||||
ZSTD_eDist_compare(&state, 0, dictSize, 0, srcSize, 1);
|
||||
ZSTD_eDist_combineMatches(&state);
|
||||
nbSequences = ZSTD_eDist_convertMatchesToSequences(sequences, &state);
|
||||
|
||||
ZSTD_free(buffer, ZSTD_defaultCMem);
|
||||
ZSTD_free(state.matches, ZSTD_defaultCMem);
|
||||
|
||||
return nbSequences;
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* This match finder leverages techniques used in file comparison algorithms
|
||||
* to find matches between a dictionary and a source file.
|
||||
*
|
||||
* The original motivation for studying this approach was to try and optimize
|
||||
* Zstandard for the use case of patching: the most common scenario being
|
||||
* updating an existing software package with the next version. When patching,
|
||||
* the difference between the old version of the package and the new version
|
||||
* is generally tiny (most of the new file will be identical to
|
||||
* the old one). In more technical terms, the edit distance (the minimal number
|
||||
* of changes required to take one sequence of bytes to another) between the
|
||||
* files would be small relative to the size of the file.
|
||||
*
|
||||
* Various 'diffing' algorithms utilize this notion of edit distance and
|
||||
* the corrensponding concept of a minimal edit script between two
|
||||
* sequences to identify the regions within two files where they differ.
|
||||
* The core algorithm used in this match finder is described in:
|
||||
*
|
||||
* "An O(ND) Difference Algorithm and its Variations", Eugene W. Myers,
|
||||
* Algorithmica Vol. 1, 1986, pp. 251-266,
|
||||
* <https://doi.org/10.1007/BF01840446>.
|
||||
*
|
||||
* Additional algorithmic heuristics for speed improvement have also been included.
|
||||
* These we inspired from implementations of various regular and binary diffing
|
||||
* algorithms such as GNU diff, bsdiff, and Xdelta.
|
||||
*
|
||||
* Note: after some experimentation, this approach proved to not provide enough
|
||||
* utility to justify the additional CPU used in finding matches. The one area
|
||||
* where this approach consistenly outperforms Zstandard even on level 19 is
|
||||
* when compressing small files (<10 KB) using a equally small dictionary that
|
||||
* is very similar to the source file. For the use case that this was intended,
|
||||
* (large similar files) this approach by itself took 5-10X longer than zstd-19 and
|
||||
* generally resulted in 2-3X larger files. The core advantage that zstd-19 has
|
||||
* over this appraoch for match finding is the overlapping matches. This approach
|
||||
* cannot find any.
|
||||
*
|
||||
* I'm leaving this in the contrib section in case this ever becomes interesting
|
||||
* to explore again.
|
||||
* */
|
||||
|
||||
#ifndef ZSTD_EDIST_H
|
||||
#define ZSTD_EDIST_H
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
|
||||
#include <stddef.h>
|
||||
#include "zstd_internal.h" /* ZSTD_Sequence */
|
||||
|
||||
/*! ZSTD_eDist_genSequences() :
|
||||
* Will populate the provided ZSTD_Sequence buffer with sequences
|
||||
* based on the optimal or near-optimal (depending on 'useHeuristics')
|
||||
* edit script between 'dict' and 'src.'
|
||||
* @return : the number of sequences found */
|
||||
size_t ZSTD_eDist_genSequences(ZSTD_Sequence* sequences,
|
||||
const void* dict, size_t dictSize,
|
||||
const void* src, size_t srcSize,
|
||||
int useHeuristics);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,3 @@
|
||||
This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
|
||||
|
||||
It outputs one libzstd, either shared or static, depending on default_library.
|
||||
@@ -0,0 +1,79 @@
|
||||
project('zstd', 'c', license: 'BSD')
|
||||
|
||||
libm = meson.get_compiler('c').find_library('m', required: true)
|
||||
|
||||
lib_dir = join_paths(meson.source_root(), '..', '..', 'lib')
|
||||
common_dir = join_paths(lib_dir, 'common')
|
||||
compress_dir = join_paths(lib_dir, 'compress')
|
||||
decompress_dir = join_paths(lib_dir, 'decompress')
|
||||
dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
|
||||
deprecated_dir = join_paths(lib_dir, 'deprecated')
|
||||
|
||||
libzstd_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')]
|
||||
|
||||
libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
|
||||
|
||||
if get_option('legacy_support')
|
||||
message('Enabling legacy support')
|
||||
libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4']
|
||||
|
||||
legacy_dir = join_paths(lib_dir, 'legacy')
|
||||
libzstd_includes += [include_directories(legacy_dir)]
|
||||
libzstd_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')]
|
||||
else
|
||||
libzstd_cflags = []
|
||||
endif
|
||||
|
||||
if get_option('multithread')
|
||||
message('Enabling multi-threading support')
|
||||
add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
|
||||
libzstd_deps = [dependency('threads')]
|
||||
else
|
||||
libzstd_deps = []
|
||||
endif
|
||||
|
||||
libzstd = library('zstd',
|
||||
libzstd_srcs,
|
||||
include_directories: libzstd_includes,
|
||||
c_args: libzstd_cflags,
|
||||
dependencies: libzstd_deps,
|
||||
install: true)
|
||||
|
||||
programs_dir = join_paths(meson.source_root(), '..', '..', 'programs')
|
||||
|
||||
zstd = executable('zstd',
|
||||
join_paths(programs_dir, 'bench.c'), join_paths(programs_dir, 'datagen.c'), join_paths(programs_dir, 'dibio.c'), join_paths(programs_dir, 'fileio.c'), join_paths(programs_dir, 'zstdcli.c'),
|
||||
include_directories: libzstd_includes,
|
||||
c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
|
||||
link_with: libzstd,
|
||||
install: true)
|
||||
|
||||
tests_dir = join_paths(meson.source_root(), '..', '..', 'tests')
|
||||
datagen_c = join_paths(programs_dir, 'datagen.c')
|
||||
test_includes = libzstd_includes + [include_directories(programs_dir)]
|
||||
|
||||
fullbench = executable('fullbench',
|
||||
datagen_c, join_paths(tests_dir, 'fullbench.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
test('fullbench', fullbench)
|
||||
|
||||
fuzzer = executable('fuzzer',
|
||||
datagen_c, join_paths(tests_dir, 'fuzzer.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
test('fuzzer', fuzzer)
|
||||
|
||||
if target_machine.system() != 'windows'
|
||||
paramgrill = executable('paramgrill',
|
||||
datagen_c, join_paths(tests_dir, 'paramgrill.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd,
|
||||
dependencies: libm)
|
||||
test('paramgrill', paramgrill)
|
||||
|
||||
datagen = executable('datagen',
|
||||
datagen_c, join_paths(tests_dir, 'datagencli.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user