Compare commits
35
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yann Collet
|
09a2dbf1e8 | ||
|
Yann Collet
|
bc04d3e9fc | ||
|
Yann Collet
|
4fa456d7f1 | ||
|
Yann Collet
|
90ae50224d | ||
|
Yann Collet
|
d1745245e0 | ||
|
Yann Collet
|
d947801569 | ||
|
Yann Collet
|
a2263a93ed | ||
|
Yann Collet
|
255597502c | ||
|
Yann Collet
|
11a3e92883 | ||
|
Yann Collet
|
f3a8bd553a | ||
|
Yann Collet
|
ed6e36e809 | ||
|
Yann Collet
|
3423a69f7c | ||
|
Yann Collet
|
1070bbc5f9 | ||
|
Yann Collet
|
f4340f46b2 | ||
|
Yann Collet
|
d29bf941b9 | ||
|
Yann Collet
|
969e6ef44f | ||
|
Yann Collet
|
2b2f3c6f64 | ||
|
Yann Collet
|
b0a3ad96d3 | ||
|
Yann Collet
|
aecf3b479c | ||
|
Yann Collet
|
b72808a49b | ||
|
Yann Collet
|
fe007ba8bd | ||
|
Yann Collet
|
7d30795931 | ||
|
Yann Collet
|
da0c157fc4 | ||
|
Yann Collet
|
39d0c6cde7 | ||
|
Sean Purcell
|
02c44a7257 | ||
|
Yann Collet
|
585bb51413 | ||
|
Sean Purcell
|
bb8a7601bc | ||
|
Yann Collet
|
ebf275912a | ||
|
Yann Collet
|
f2ea8932fc | ||
|
Yann Collet
|
2c2ee2b4b0 | ||
|
Sean Purcell
|
c4f2490050 | ||
|
Yann Collet
|
cbc5225d38 | ||
|
Yann Collet
|
817995bb7a | ||
|
Yann Collet
|
99b6d9f1a3 | ||
|
Yann Collet
|
b9edd2f9fb |
+44
-62
@@ -1,19 +1,36 @@
|
||||
version: 2
|
||||
|
||||
references:
|
||||
# Install the dependencies required for tests.
|
||||
# Add the step "- *install-dependencies" to the beginning of your job to run
|
||||
# this command.
|
||||
install-dependencies: &install-dependencies
|
||||
run:
|
||||
name: Install dependencies
|
||||
# TODO: We can split these dependencies up by job to reduce installation
|
||||
# time.
|
||||
command: |
|
||||
sudo dpkg --add-architecture i386
|
||||
sudo apt-get -y -qq update
|
||||
sudo apt-get -y install \
|
||||
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
|
||||
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
|
||||
libc6-dev-ppc64-powerpc-cross
|
||||
|
||||
jobs:
|
||||
# the first half of the jobs are in this test
|
||||
short-tests-0:
|
||||
# TODO: Create a small custom docker image with all the dependencies we need
|
||||
# preinstalled to reduce installation time.
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
- image: circleci/buildpack-deps:bionic
|
||||
steps:
|
||||
- checkout
|
||||
- *install-dependencies
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
./tests/test-license.py
|
||||
cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
|
||||
cc -v; CFLAGS="-O0 -Werror" make all && make clean
|
||||
make c99build ; make clean
|
||||
make c11build ; make clean
|
||||
make aarch64build ; make clean
|
||||
@@ -23,62 +40,49 @@ jobs:
|
||||
# the second half of the jobs are in this test
|
||||
short-tests-1:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
- image: circleci/buildpack-deps:bionic
|
||||
steps:
|
||||
- checkout
|
||||
- *install-dependencies
|
||||
- run:
|
||||
name: Test
|
||||
command: |
|
||||
make gnu90build; make clean
|
||||
make gnu99build; make clean
|
||||
make ppc64build V=1; make clean
|
||||
make ppcbuild V=1; make clean
|
||||
make armbuild V=1; make clean
|
||||
make -C tests test-legacy test-longmatch; make clean
|
||||
make ppc64build; make clean
|
||||
make ppcbuild ; make clean
|
||||
make armbuild ; make clean
|
||||
make -C tests test-legacy test-longmatch test-symbols; make clean
|
||||
make -C lib libzstd-nomt; make clean
|
||||
# This step should only be run in a cron job
|
||||
regression-test:
|
||||
# This step is only run on release tags.
|
||||
# It publishes the source tarball as artifacts and if the GITHUB_TOKEN
|
||||
# environment variable is set it will publish the source tarball to the
|
||||
# tagged release.
|
||||
publish-github-release:
|
||||
docker:
|
||||
- image: fbopensource/zstd-circleci-primary:0.0.1
|
||||
- image: cibuilds/github:0.12.0
|
||||
environment:
|
||||
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||
steps:
|
||||
- checkout
|
||||
# Restore the cached resources.
|
||||
- restore_cache:
|
||||
# We try our best to bust the cache when the data changes by hashing
|
||||
# data.c. If that doesn't work, simply update the version number here
|
||||
# and below. If we fail to bust the cache, the regression testing will
|
||||
# still work, since it has its own stamp, but will need to redownload
|
||||
# everything.
|
||||
keys:
|
||||
- regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
- run:
|
||||
name: Regression Test
|
||||
name: Install dependencies
|
||||
command: |
|
||||
make -C programs zstd
|
||||
make -C tests/regression test
|
||||
apk add -q gzip coreutils
|
||||
- run:
|
||||
name: Publish
|
||||
command: |
|
||||
export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
|
||||
export ZSTD_VERSION=zstd-$VERSION
|
||||
git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
|
||||
-o $ZSTD_VERSION.tar
|
||||
gzip -9 $ZSTD_VERSION.tar
|
||||
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256sum
|
||||
mkdir -p $CIRCLE_ARTIFACTS
|
||||
./tests/regression/test \
|
||||
--cache tests/regression/cache \
|
||||
--output $CIRCLE_ARTIFACTS/results.csv \
|
||||
--zstd programs/zstd
|
||||
echo "NOTE: The new results.csv is uploaded as an artifact to this job"
|
||||
echo " If this fails, go to the Artifacts pane in CircleCI, "
|
||||
echo " download /tmp/circleci-artifacts/results.csv, and if they "
|
||||
echo " are still good, copy it into the repo and commit it."
|
||||
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
|
||||
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
|
||||
# Only save the cache on success (default), since if the failure happened
|
||||
# before we stamp the data cache, we will have a bad cache for this key.
|
||||
- save_cache:
|
||||
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
|
||||
paths:
|
||||
- tests/regression/cache
|
||||
cp $ZSTD_VERSION.tar.gz{,.sha256sum} $CIRCLE_ARTIFACTS
|
||||
- store_artifacts:
|
||||
path: /tmp/circleci-artifacts
|
||||
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
commit:
|
||||
@@ -92,13 +96,6 @@ workflows:
|
||||
filters:
|
||||
tags:
|
||||
only: /.*/
|
||||
# Create a branch called regression and set it to dev to force a
|
||||
# regression test run
|
||||
- regression-test:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- regression
|
||||
# Only run on release tags.
|
||||
- publish-github-release:
|
||||
requires:
|
||||
@@ -109,21 +106,6 @@ workflows:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
nightly:
|
||||
triggers:
|
||||
- schedule:
|
||||
cron: "0 0 * * *"
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- release
|
||||
- dev
|
||||
- master
|
||||
jobs:
|
||||
# Run daily long regression tests
|
||||
- regression-test
|
||||
|
||||
|
||||
|
||||
# Longer tests
|
||||
#- make -C tests test-zstd-nolegacy && make clean
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
FROM circleci/buildpack-deps:bionic
|
||||
|
||||
RUN sudo dpkg --add-architecture i386
|
||||
RUN sudo apt-get -y -qq update
|
||||
RUN sudo apt-get -y install \
|
||||
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
|
||||
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
|
||||
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
|
||||
libcurl4-openssl-dev
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
task:
|
||||
name: FreeBSD (shortest)
|
||||
freebsd_instance:
|
||||
matrix:
|
||||
image_family: freebsd-12-2
|
||||
# The stable 11.3 image causes "Agent is not responding" so use a snapshot
|
||||
image_family: freebsd-11-3-snap
|
||||
install_script: pkg install -y gmake coreutils
|
||||
script: |
|
||||
MOREFLAGS="-Werror" gmake -j all
|
||||
gmake shortest
|
||||
@@ -19,3 +19,6 @@
|
||||
# Windows
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
# .travis.yml merging
|
||||
.travis.yml merge=ours
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Downloads data '...'
|
||||
2. Run '...' with flags '...'
|
||||
3. Scroll up on the log to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots and charts**
|
||||
If applicable, add screenshots and charts to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. Mac]
|
||||
- Version [e.g. 22]
|
||||
- Compiler [e.g gcc]
|
||||
- Flags [e.g O2]
|
||||
- Other relevant hardware specs [e.g. Dual-core]
|
||||
- Build system [e.g. Makefile]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -1,214 +0,0 @@
|
||||
name: generic-dev
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ dev, release, actionsTest ]
|
||||
|
||||
jobs:
|
||||
|
||||
# Dev PR jobs that still have to be migrated from travis
|
||||
#
|
||||
# versionTag (only on release tags)
|
||||
# valgrindTest (keeps failing for some reason. need investigation)
|
||||
# staticAnalyze (need trusty so need self-hosted)
|
||||
# pcc-fuzz: (need trusty so need self-hosted)
|
||||
# min-decomp-macros (flakey)
|
||||
#
|
||||
# setting up self-hosted is pretty straightforward, but
|
||||
# I need admins permissions to the repo for that it looks like
|
||||
# So I'm tabling that for now
|
||||
#
|
||||
# The release branch exclusive jobs will be in a separate
|
||||
# workflow file (the osx tests and meson build that is)
|
||||
|
||||
benchmarking:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make benchmarking
|
||||
run: make benchmarking
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make test
|
||||
run: make test
|
||||
|
||||
check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: make check on 32-bit
|
||||
run: |
|
||||
sudo apt update
|
||||
APT_PACKAGES="gcc-multilib" make apt-install
|
||||
CFLAGS="-m32 -O1 -fstack-protector" make check V=1
|
||||
|
||||
gcc-7-libzstd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-7 + libzstdmt compilation
|
||||
run: |
|
||||
make gcc7install
|
||||
CC=gcc-7 CFLAGS=-Werror make -j all
|
||||
make clean
|
||||
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||
make -C tests zbufftest-dll
|
||||
|
||||
# candidate test (to check) : underlink test
|
||||
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
|
||||
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
|
||||
|
||||
gcc-8-asan-ubsan-testzstd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Test Zstd
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 CFLAGS="-Werror" make -j all
|
||||
make clean
|
||||
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
|
||||
|
||||
gcc-asan-ubsan-testzstd-32bit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ASan + UBSan + Test Zstd, 32bit mode
|
||||
run: |
|
||||
make libc6install
|
||||
CFLAGS="-Werror -m32" make -j all32
|
||||
make clean
|
||||
make -j uasan-test-zstd32 V=1
|
||||
|
||||
clang-msan-testzstd:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: clang + MSan + Test Zstd
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install clang
|
||||
CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
|
||||
|
||||
# Note : external libraries must be turned off when using MSAN tests,
|
||||
# because they are not msan-instrumented,
|
||||
# so any data coming from these libraries is always considered "uninitialized"
|
||||
|
||||
cmake-build-and-test-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: cmake build and test check
|
||||
run: |
|
||||
FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
|
||||
cp -r ./ "../zstd source"
|
||||
cd "../zstd source"
|
||||
FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
|
||||
|
||||
gcc-8-asan-ubsan-fuzz:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: gcc-8 + ASan + UBSan + Fuzz Test
|
||||
run: |
|
||||
make gcc8install
|
||||
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
|
||||
|
||||
gcc-asan-ubsan-fuzz32:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ASan + UBSan + Fuzz Test 32bit
|
||||
run: |
|
||||
make libc6install
|
||||
CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
|
||||
|
||||
asan-ubsan-msan-regression:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: ASan + UBSan + MSan + Regression Test
|
||||
run: |
|
||||
make -j uasanregressiontest
|
||||
make clean
|
||||
make -j msanregressiontest
|
||||
|
||||
cpp-gnu90-c99-compatibility:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: C++, gnu90 and c99 compatibility
|
||||
run: |
|
||||
make cxxtest
|
||||
make clean
|
||||
make gnu90build
|
||||
make clean
|
||||
make c99build
|
||||
make clean
|
||||
make travis-install # just ensures `make install` works
|
||||
|
||||
mingw-cross-compilation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: mingw cross-compilation
|
||||
run: |
|
||||
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
|
||||
sudo apt-get install gcc-mingw-w64
|
||||
CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
||||
|
||||
# TODO: Broken test - fix and uncomment
|
||||
# armbuild:
|
||||
# runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v2
|
||||
# - name: ARM Build Test
|
||||
# run: |
|
||||
# make arminstall
|
||||
# make armbuild
|
||||
|
||||
# TODO: Broken test - fix and uncomment
|
||||
# armfuzz:
|
||||
# runs-on: ubuntu-16.04 # doesn't work on latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v2
|
||||
# - name: Qemu ARM emulation + Fuzz Test
|
||||
# run: |
|
||||
# make arminstall
|
||||
# make armfuzz
|
||||
|
||||
bourne-shell:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Bourne shell compatibility (shellcheck)
|
||||
run: |
|
||||
wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
|
||||
shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
|
||||
|
||||
# For reference : icc tests
|
||||
# icc tests are currently failing on Github Actions, likely to issues during installation stage
|
||||
# To be fixed later
|
||||
#
|
||||
# icc:
|
||||
# name: icc-check
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - name: install icc
|
||||
# run: |
|
||||
# export DEBIAN_FRONTEND=noninteractive
|
||||
# sudo apt-get -qqq update
|
||||
# sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
|
||||
# sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
|
||||
# sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
|
||||
# sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
|
||||
# sudo apt-get update
|
||||
# sudo apt-get install -y intel-basekit intel-hpckit
|
||||
# - uses: actions/checkout@v2
|
||||
# - name: make check
|
||||
# run: |
|
||||
# make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
|
||||
@@ -1,67 +0,0 @@
|
||||
name: generic-release
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# This will eventually only be for pushes to release
|
||||
# but for dogfooding purposes, I'm running it even
|
||||
# on dev pushes
|
||||
branches: [ dev, release, actionsTest ]
|
||||
|
||||
jobs:
|
||||
# missing jobs
|
||||
#
|
||||
# ppc64le + fuzz test
|
||||
# Qemu PPC64 + Fuzz test
|
||||
# Qemu aarch64 + Fuzz Test (on Xenial)
|
||||
# versions comp
|
||||
# meson test
|
||||
|
||||
osx:
|
||||
runs-on: macos-10.15
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: OS-X
|
||||
run: |
|
||||
make test
|
||||
# make -c lib all (need to fix. not working right now)
|
||||
|
||||
zbuff:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zbuff test
|
||||
run: |
|
||||
make -C tests test-zbuff
|
||||
|
||||
tsan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: thread sanitizer
|
||||
run: |
|
||||
CC=clang make tsan-test-zstream
|
||||
CC=clang make tsan-fuzztest
|
||||
|
||||
zlib-wrapper:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: zlib wrapper test
|
||||
run: |
|
||||
make valgrindinstall
|
||||
make -C zlibWrapper test
|
||||
make -C zlibWrapper valgrindTest
|
||||
|
||||
lz4-threadpool-partial-libs:
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: LZ4, thread pool, and partial libs testslib wrapper test
|
||||
run: |
|
||||
make lz4install
|
||||
make -C tests test-lz4
|
||||
make check < /dev/null | tee # mess with lz4 console detection
|
||||
make clean
|
||||
make -C tests test-pool
|
||||
make clean
|
||||
bash tests/libzstd_partial_builds.sh
|
||||
@@ -1,13 +0,0 @@
|
||||
name: linux-kernel
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ dev, release, actionsTest ]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Create linux kernel library + build + test
|
||||
run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
|
||||
@@ -1,30 +0,0 @@
|
||||
name: CIFuzz
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
Fuzzing:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
sanitizer: [address, undefined]
|
||||
steps:
|
||||
- name: Build Fuzzers (${{ matrix.sanitizer }})
|
||||
id: build
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Run Fuzzers (${{ matrix.sanitizer }})
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'zstd'
|
||||
fuzz-seconds: 600
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Upload Crash
|
||||
uses: actions/upload-artifact@v1
|
||||
if: failure() && steps.build.outcome == 'success'
|
||||
with:
|
||||
name: ${{ matrix.sanitizer }}-artifacts
|
||||
path: ./out/artifacts
|
||||
@@ -1,68 +0,0 @@
|
||||
name: publish-release-artifacts
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
|
||||
jobs:
|
||||
publish-release-artifacts:
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Archive
|
||||
env:
|
||||
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
|
||||
RELEASE_SIGNING_KEY_PASSPHRASE: ${{ secrets.RELEASE_SIGNING_KEY_PASSPHRASE }}
|
||||
run: |
|
||||
# compute file name
|
||||
export TAG="$(echo "$GITHUB_REF" | sed -n 's_^refs/tags/__p')"
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "action must be run on a tag. GITHUB_REF is not a tag: $GITHUB_REF"
|
||||
exit 1
|
||||
fi
|
||||
# Attempt to extract "1.2.3" from "v1.2.3" to maintain artifact name backwards compat.
|
||||
# Otherwise, degrade to using full tag.
|
||||
export VERSION="$(echo "$TAG" | sed 's_^v\([0-9]\+\.[0-9]\+\.[0-9]\+\)$_\1_')"
|
||||
export ZSTD_VERSION="zstd-$VERSION"
|
||||
|
||||
# archive
|
||||
git archive $TAG \
|
||||
--prefix $ZSTD_VERSION/ \
|
||||
--format tar \
|
||||
-o $ZSTD_VERSION.tar
|
||||
|
||||
# Do the rest of the work in a sub-dir so we can glob everything we want to publish.
|
||||
mkdir artifacts/
|
||||
mv $ZSTD_VERSION.tar artifacts/
|
||||
cd artifacts/
|
||||
|
||||
# compress
|
||||
zstd -k -19 $ZSTD_VERSION.tar
|
||||
gzip -k -9 $ZSTD_VERSION.tar
|
||||
|
||||
# we only publish the compressed tarballs
|
||||
rm $ZSTD_VERSION.tar
|
||||
|
||||
# hash
|
||||
sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
|
||||
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
|
||||
|
||||
# sign
|
||||
if [ -n "$RELEASE_SIGNING_KEY" ]; then
|
||||
export GPG_BATCH_OPTS="--batch --no-use-agent --pinentry-mode loopback --no-tty --yes"
|
||||
echo "$RELEASE_SIGNING_KEY" | gpg $GPG_BATCH_OPTS --import
|
||||
gpg $GPG_BATCH_OPTS --armor --sign --sign-with signing@zstd.net --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.zst.sig $ZSTD_VERSION.tar.zst
|
||||
gpg $GPG_BATCH_OPTS --armor --sign --sign-with signing@zstd.net --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.gz.sig $ZSTD_VERSION.tar.gz
|
||||
fi
|
||||
|
||||
- name: Publish
|
||||
uses: skx/github-action-publish-binaries@release-1.3
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
args: artifacts/*
|
||||
+2
-11
@@ -14,7 +14,7 @@
|
||||
*.dylib
|
||||
|
||||
# Executables
|
||||
/zstd
|
||||
zstd
|
||||
zstdmt
|
||||
*.exe
|
||||
*.out
|
||||
@@ -22,10 +22,7 @@ zstdmt
|
||||
|
||||
# Test artefacts
|
||||
tmp*
|
||||
*.zst
|
||||
*.zstd
|
||||
dictionary.
|
||||
dictionary
|
||||
dictionary*
|
||||
NUL
|
||||
|
||||
# Build artefacts
|
||||
@@ -33,8 +30,6 @@ projects/
|
||||
bin/
|
||||
.buckd/
|
||||
buck-out/
|
||||
build-*
|
||||
*.gcda
|
||||
|
||||
# Other files
|
||||
.directory
|
||||
@@ -46,7 +41,3 @@ _zstdbench/
|
||||
.DS_Store
|
||||
googletest/
|
||||
*.d
|
||||
*.vscode
|
||||
*.code-workspace
|
||||
compile_commands.json
|
||||
.clangd
|
||||
|
||||
+60
-147
@@ -1,6 +1,59 @@
|
||||
# Medium Tests: Run on all commits/PRs to dev branch
|
||||
|
||||
language: c
|
||||
dist: trusty
|
||||
sudo: required
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
|
||||
matrix:
|
||||
include:
|
||||
# Ubuntu 14.04
|
||||
- env: Cmd='make test'
|
||||
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 CFLAGS=-Werror make -j all
|
||||
&& make clean && CC=gcc-6 make -j uasan-test-zstd </dev/null' # test when stdin is not a tty
|
||||
- env: Cmd='make gcc6install libc6install
|
||||
&& make clean && CC=gcc-6 make -j uasan-test-zstd32'
|
||||
- env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd'
|
||||
- env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
||||
|
||||
- env: Cmd='make staticAnalyze'
|
||||
|
||||
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
|
||||
- env: Cmd='make gcc6install libc6install
|
||||
&& make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
||||
|
||||
- env: Cmd='make -j uasanregressiontest
|
||||
&& make clean && make -j msanregressiontest'
|
||||
|
||||
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
|
||||
&& make clean && make -C tests test-fuzzer-stackmode'
|
||||
|
||||
- env: Cmd='make arminstall && make armfuzz'
|
||||
- env: Cmd='make arminstall && make aarch64fuzz'
|
||||
- env: Cmd='make ppcinstall && make ppcfuzz'
|
||||
- env: Cmd='make ppcinstall && make ppc64fuzz'
|
||||
|
||||
- env: Cmd='make lz4install && make -C tests test-lz4
|
||||
&& make clean && make -C tests test-pool
|
||||
&& make clean && bash tests/libzstd_partial_builds.sh'
|
||||
|
||||
# tag-specific test
|
||||
- if: tag =~ ^v[0-9]\.[0-9]
|
||||
env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'
|
||||
|
||||
# additional tests for master branch and cron job
|
||||
- env: Cmd="make test"
|
||||
os: osx
|
||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
||||
- env: Cmd='make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install'
|
||||
- env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
|
||||
|
||||
|
||||
git:
|
||||
depth: 1
|
||||
@@ -8,153 +61,13 @@ git:
|
||||
branches:
|
||||
only:
|
||||
- dev
|
||||
- release
|
||||
- master
|
||||
- travisTest
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
|
||||
env:
|
||||
global:
|
||||
- FUZZERTEST=-T2mn
|
||||
ZSTREAM_TESTTIME=-T2mn
|
||||
DECODECORPUS_TESTTIME=-T1mn
|
||||
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- name: arm64 # ~2.5 mn
|
||||
os: linux
|
||||
arch: arm64
|
||||
script:
|
||||
- make check
|
||||
|
||||
- name: Minimal Decompressor Macros # ~5mn
|
||||
script:
|
||||
- make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
||||
- make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
||||
- make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
|
||||
- make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT"
|
||||
- make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
|
||||
- make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
|
||||
- make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||
- make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||
|
||||
- name: static analyzer scanbuild # ~26mn
|
||||
dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives
|
||||
script:
|
||||
- make staticAnalyze
|
||||
|
||||
- name: Valgrind + Fuzz Test Stack Mode # ~ 7mn
|
||||
script:
|
||||
- make valgrindinstall
|
||||
- make -C tests clean valgrindTest
|
||||
- make clean
|
||||
- make -C tests test-fuzzer-stackmode
|
||||
|
||||
- name: Qemu ARM emulation + Fuzz Test # ~13.5mn
|
||||
script:
|
||||
- make arminstall
|
||||
- make armfuzz
|
||||
|
||||
# Introduced to check compat with old toolchains, to prevent e.g. #1872
|
||||
- name: ARM Build Test (on Trusty)
|
||||
dist: trusty
|
||||
script:
|
||||
- make arminstall
|
||||
- make armbuild
|
||||
|
||||
- name: Qemu PPC + Fuzz Test # ~13mn
|
||||
dist: trusty # it seems ppc cross-compilation fails on "current"
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppcfuzz
|
||||
|
||||
# check release number (release only)
|
||||
- name: Tag-Specific Test
|
||||
if: tag =~ ^v[0-9]\.[0-9]
|
||||
script:
|
||||
- make -C tests checkTag
|
||||
- tests/checkTag "$TRAVIS_BRANCH"
|
||||
|
||||
# tests for release branch and cron job only
|
||||
- name: OS-X # ~13mn
|
||||
if: branch = release
|
||||
os: osx
|
||||
script:
|
||||
- make test
|
||||
- make -C lib all
|
||||
|
||||
- name: Versions Compatibility Test # 11.5mn
|
||||
if: branch = release
|
||||
script:
|
||||
- make -C tests versionsTest
|
||||
|
||||
- name: thread sanitizer # ~29mn
|
||||
if: branch = release
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make tsan-test-zstream
|
||||
- CC=clang-3.8 make tsan-fuzztest
|
||||
|
||||
- name: PPC64LE + Fuzz test # ~13mn
|
||||
if: branch = release
|
||||
arch: ppc64le
|
||||
script:
|
||||
- cat /proc/cpuinfo
|
||||
- make test
|
||||
|
||||
- name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?)
|
||||
dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
|
||||
if: branch = release
|
||||
script:
|
||||
- make ppcinstall
|
||||
- make ppc64fuzz
|
||||
|
||||
# note : we already have aarch64 tests on hardware
|
||||
- name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn
|
||||
if: branch = release
|
||||
dist: xenial
|
||||
script:
|
||||
- make arminstall
|
||||
- make aarch64fuzz
|
||||
|
||||
# meson dedicated test
|
||||
- name: Xenial (Meson + clang) # ~15mn
|
||||
if: branch = release
|
||||
dist: xenial
|
||||
language: cpp
|
||||
compiler: clang
|
||||
install:
|
||||
- sudo apt-get install -qq liblz4-dev valgrind tree
|
||||
- |
|
||||
travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
|
||||
unzip ~/ninja.zip -d ~/.local/bin
|
||||
- |
|
||||
travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/get-pip.py' &&
|
||||
python3 ~/get-pip.py --user &&
|
||||
pip3 install --user meson
|
||||
script:
|
||||
- |
|
||||
meson setup \
|
||||
--buildtype=debugoptimized \
|
||||
-Db_lundef=false \
|
||||
-Dauto_features=enabled \
|
||||
-Dbin_programs=true \
|
||||
-Dbin_tests=true \
|
||||
-Dbin_contrib=true \
|
||||
-Ddefault_library=both \
|
||||
build/meson builddir
|
||||
- pushd builddir
|
||||
- ninja
|
||||
- meson test --verbose --no-rebuild
|
||||
- DESTDIR=./staging ninja install
|
||||
- tree ./staging
|
||||
after_failure:
|
||||
- cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
|
||||
|
||||
allow_failures:
|
||||
- env: ALLOW_FAILURES=true
|
||||
script:
|
||||
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
||||
- echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
|
||||
- export FUZZERTEST=-T2mn;
|
||||
export ZSTREAM_TESTTIME=-T2mn;
|
||||
export DECODECORPUS_TESTTIME=-T1mn;
|
||||
sh -c "$Cmd" || travis_terminate 1;
|
||||
|
||||
+2
-366
@@ -5,7 +5,7 @@ possible.
|
||||
## Our Development Process
|
||||
New versions are being developed in the "dev" branch,
|
||||
or in their own feature branch.
|
||||
When they are deemed ready for a release, they are merged into "release".
|
||||
When they are deemed ready for a release, they are merged into "master".
|
||||
|
||||
As a consequences, all contributions must stage first through "dev"
|
||||
or their own feature branch.
|
||||
@@ -26,370 +26,6 @@ to do this once to work on any of Facebook's open source projects.
|
||||
|
||||
Complete your CLA here: <https://code.facebook.com/cla>
|
||||
|
||||
## Workflow
|
||||
Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
|
||||
will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
|
||||
related changes into a branch.
|
||||
|
||||
Our contribution process works in three main stages:
|
||||
1. Local development
|
||||
* Update:
|
||||
* Checkout your fork of zstd if you have not already
|
||||
```
|
||||
git checkout https://github.com/<username>/zstd
|
||||
cd zstd
|
||||
```
|
||||
* Update your local dev branch
|
||||
```
|
||||
git pull https://github.com/facebook/zstd dev
|
||||
git push origin dev
|
||||
```
|
||||
* Topic and development:
|
||||
* Make a new branch on your fork about the topic you're developing for
|
||||
```
|
||||
# branch names should be consise but sufficiently informative
|
||||
git checkout -b <branch-name>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Make commits and push
|
||||
```
|
||||
# make some changes =
|
||||
git add -u && git commit -m <message>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Note: run local tests to ensure that your changes didn't break existing functionality
|
||||
* Quick check
|
||||
```
|
||||
make shortest
|
||||
```
|
||||
* Longer check
|
||||
```
|
||||
make test
|
||||
```
|
||||
2. Code Review and CI tests
|
||||
* Ensure CI tests pass:
|
||||
* Before sharing anything to the community, make sure that all CI tests pass on your local fork.
|
||||
See our section on setting up your CI environment for more information on how to do this.
|
||||
* Ensure that static analysis passes on your development machine. See the Static Analysis section
|
||||
below to see how to do this.
|
||||
* Create a pull request:
|
||||
* When you are ready to share you changes to the community, create a pull request from your branch
|
||||
to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
|
||||
page.
|
||||
* From there, select the branch where you made changes as your source branch and facebook:dev
|
||||
as the destination.
|
||||
* Examine the diff presented between the two branches to make sure there is nothing unexpected.
|
||||
* Write a good pull request description:
|
||||
* While there is no strict template that our contributors follow, we would like them to
|
||||
sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
|
||||
at least indirectly, address the following points.
|
||||
* Is this pull request important and why?
|
||||
* Is it addressing an issue? If so, what issue? (provide links for convenience please)
|
||||
* Is this a new feature? If so, why is it useful and/or necessary?
|
||||
* Are there background references and documents that reviewers should be aware of to properly assess this change?
|
||||
* Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
|
||||
* Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
|
||||
* Submit the pull request and iterate with feedback.
|
||||
3. Merge and Release
|
||||
* Getting approval:
|
||||
* You will have to iterate on your changes with feedback from other collaborators to reach a point
|
||||
where your pull request can be safely merged.
|
||||
* To avoid too many comments on style and convention, make sure that you have a
|
||||
look at our style section below before creating a pull request.
|
||||
* Eventually, someone from the zstd team will approve your pull request and not long after merge it into
|
||||
the dev branch.
|
||||
* Housekeeping:
|
||||
* Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
|
||||
the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
|
||||
issue at hand, then please indicate this by requesting that an issue be closed by commenting.
|
||||
* Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
|
||||
that the change must make it to an official zstd release for it to be meaningful. We recommend
|
||||
that contributers track the activity on their pull request and corresponding issue(s) page(s) until
|
||||
their change makes it to the next release of zstd. Users will often discover bugs in your code or
|
||||
suggest ways to refine and improve your initial changes even after the pull request is merged.
|
||||
|
||||
## Static Analysis
|
||||
Static analysis is a process for examining the correctness or validity of a program without actually
|
||||
executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
|
||||
static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
|
||||
|
||||
Once installed, you can ensure that our static analysis tests pass on your local development machine
|
||||
by running:
|
||||
```
|
||||
make staticAnalyze
|
||||
```
|
||||
|
||||
In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
|
||||
just `contrib/largeNbDicts` and nothing else, you can run:
|
||||
|
||||
```
|
||||
scan-build make -C contrib/largeNbDicts largeNbDicts
|
||||
```
|
||||
|
||||
### Pitfalls of static analysis
|
||||
`scan-build` is part of our regular CI suite. Other static analyzers are not.
|
||||
|
||||
It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
|
||||
|
||||
- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
|
||||
- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
|
||||
- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
|
||||
+ torteous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
|
||||
+ sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
|
||||
- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
|
||||
|
||||
This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kind of reports can be useful, and are accepted.
|
||||
|
||||
## Performance
|
||||
Performance is extremely important for zstd and we only merge pull requests whose performance
|
||||
landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
|
||||
This high bar for performance means that every PR which has the potential to
|
||||
impact performance takes a very long time for us to properly review. That being said, we
|
||||
always welcome contributions to improve performance (or worsen performance for the trade-off of
|
||||
something else). Please keep the following in mind before submitting a performance related PR:
|
||||
|
||||
1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
|
||||
very well documented via past Github issues and pull requests. It may be the case that your
|
||||
particular performance optimization has already been considered in the past. Please take some
|
||||
time to search through old issues and pull requests using keywords specific to your
|
||||
would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
|
||||
on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
|
||||
it will be helpful for you to have context from that topic's history before contributing.
|
||||
2. The distinction between noise and actual performance gains can unfortunately be very subtle
|
||||
especially when microbenchmarking extremely small wins or losses. The only remedy to getting
|
||||
something subtle merged is extensive benchmarking. You will be doing us a great favor if you
|
||||
take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
|
||||
benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
|
||||
your changes on every single processor and os out there (and neither will we) but do that best
|
||||
you can:) We've adding some things to think about when benchmarking below in the Benchmarking
|
||||
Performance section which might be helpful for you.
|
||||
3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
|
||||
legitimate thing to do as long as it does not harm the overall performance health of Zstd.
|
||||
This is a hard balance to strike but please keep in mind other aspects of Zstd when
|
||||
submitting changes that are clang-specific, windows-specific, etc.
|
||||
|
||||
## Benchmarking Performance
|
||||
Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
|
||||
testing over theoretical speculation. This guide it not perfect but for most scenarios, it
|
||||
is a good place to start.
|
||||
|
||||
### Stability
|
||||
Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
|
||||
benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
|
||||
will typically not be stable enough to obtain reliable benchmark results. If you can get your
|
||||
hands on a desktop, this is usually a better scenario.
|
||||
|
||||
Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
|
||||
do a little more work to ensure that you are in fact measuring the changes you've made not and
|
||||
noise. Here are some things you can do to make your benchmarks more stable:
|
||||
|
||||
1. The most simple thing you can do to drastically improve the stability of your benchmark is
|
||||
to run it multiple times and then aggregate the results of those runs. As a general rule of
|
||||
thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
|
||||
you will have to aggregate over to get reliable results. Here are some additional things to keep in
|
||||
mind when running multiple trials:
|
||||
* How you aggregate your samples are important. You might be tempted to use the mean of your
|
||||
results. While this is certainly going to be a more stable number than a raw single sample
|
||||
benchmark number, you might have more luck by taking the median. The mean is not robust to
|
||||
outliers whereas the median is. Better still, you could simply take the fastest speed your
|
||||
benchmark achieved on each run since that is likely the fastest your process will be
|
||||
capable of running your code. In our experience, this (aggregating by just taking the sample
|
||||
with the fastest running time) has been the most stable approach.
|
||||
* The more samples you have, the more stable your benchmarks should be. You can verify
|
||||
your improved stability by looking at the size of your confidence intervals as you
|
||||
increase your sample count. These should get smaller and smaller. Eventually hopefully
|
||||
smaller than the performance win you are expecting.
|
||||
* Most processors will take some time to get `hot` when running anything. The observations
|
||||
you collect during that time period will very different from the true performance number. Having
|
||||
a very large number of sample will help alleviate this problem slightly but you can also
|
||||
address is directly by simply not including the first `n` iterations of your benchmark in
|
||||
your aggregations. You can determine `n` by simply looking at the results from each iteration
|
||||
and then hand picking a good threshold after which the variance in results seems to stabilize.
|
||||
2. You cannot really get reliable benchmarks if your host machine is simultaneously running
|
||||
another cpu/memory-intensive application in the background. If you are running benchmarks on your
|
||||
personal laptop for instance, you should close all applications (including your code editor and
|
||||
browser) before running your benchmarks. You might also have invisible background applications
|
||||
running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
|
||||
on Windows. You will get more stable benchmark results of you end those processes as well.
|
||||
* If you have multiple cores, you can even run your benchmark on a reserved core to prevent
|
||||
pollution from other OS and user processes. There are a number of ways to do this depending
|
||||
on your OS:
|
||||
* On linux boxes, you have use https://github.com/lpechacek/cpuset.
|
||||
* On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
|
||||
* On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
|
||||
3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
|
||||
Dynamically linking your library will introduce some added variation (not a large amount but
|
||||
definitely some). Statically linking libzstd will be more stable. Static libraries should
|
||||
be enabled by default when building zstd.
|
||||
4. Use a profiler with a good high resolution timer. See the section below on profiling for
|
||||
details on this.
|
||||
5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
|
||||
6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
|
||||
tmpfs avoids touching a real storage system, which can have a pretty big variability.
|
||||
|
||||
Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
|
||||
|
||||
### Zstd benchmark
|
||||
The fastest signal you can get regarding your performance changes is via the in-build zstd cli
|
||||
bench option. You can run Zstd as you typically would for your scenario using some set of options
|
||||
and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
|
||||
for that options you have just provided. If you want to look at the internals of how this
|
||||
benchmarking script works, you can check out programs/benchzstd.c
|
||||
|
||||
For example: say you have made a change that you believe improves the speed of zstd level 1. The
|
||||
very first thing you should use to asses whether you actually achieved any sort of improvement
|
||||
is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
|
||||
specify a running time for your benchmark in seconds (default is 3 seconds).
|
||||
Usually, the longer the running time, the more stable your results will be.
|
||||
|
||||
```
|
||||
$ git checkout <commit-before-your-change>
|
||||
$ make && cp zstd zstd-old
|
||||
$ git checkout <commit-after-your-change>
|
||||
$ make && cp zstd zstd-new
|
||||
$ zstd-old -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s
|
||||
$ zstd-new -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s
|
||||
```
|
||||
|
||||
Unless your performance win is large enough to be visible despite the intrinsic noise
|
||||
on your computer, benchzstd alone will likely not be enough to validate the impact of your
|
||||
changes. For example, the results of the example above indicate that effectively nothing
|
||||
changed but there could be a small <3% improvement that the noise on the host machine
|
||||
obscured. So unless you see a large performance win (10-15% consistently) using just
|
||||
this method of evaluation will not be sufficient.
|
||||
|
||||
### Profiling
|
||||
There are a number of great profilers out there. We're going to briefly mention how you can
|
||||
profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
|
||||
on windows.
|
||||
|
||||
Say you have an idea for a change that you think will provide some good performance gains
|
||||
for level 1 compression on Zstd. Typically this means, you have identified a section of
|
||||
code that you think can be made to run faster.
|
||||
|
||||
The first thing you will want to do is make sure that the piece of code is actually taking up
|
||||
a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
|
||||
0.0001% of the total running time. Luckily, there are tools to help with this.
|
||||
Profilers will let you see how much time your code spends inside a particular function.
|
||||
If your target code snippit is only part of a function, it might be worth trying to
|
||||
isolate that snippit by moving it to its own function (this is usually not necessary but
|
||||
might be).
|
||||
|
||||
Most profilers (including the profilers dicusssed below) will generate a call graph of
|
||||
functions for you. Your goal will be to find your function of interest in this call grapch
|
||||
and then inspect the time spent inside of it. You might also want to to look at the
|
||||
annotated assembly which most profilers will provide you with.
|
||||
|
||||
#### Instruments
|
||||
We will once again consider the scenario where you think you've identified a piece of code
|
||||
whose performance can be improved upon. Follow these steps to profile your code using
|
||||
Instruments.
|
||||
|
||||
1. Open Instruments
|
||||
2. Select `Time Profiler` from the list of standard templates
|
||||
3. Close all other applications except for your instruments window and your terminal
|
||||
4. Run your benchmarking script from your terminal window
|
||||
* You will want a benchmark that runs for at least a few seconds (5 seconds will
|
||||
usually be long enough). This way the profiler will have something to work with
|
||||
and you will have ample time to attach your profiler to this process:)
|
||||
* I will just use benchzstd as my bencharmking script for this example:
|
||||
```
|
||||
$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
|
||||
```
|
||||
5. Once you run your benchmarking script, switch back over to instruments and attach your
|
||||
process to the time profiler. You can do this by:
|
||||
* Clicking on the `All Processes` drop down in the top left of the toolbar.
|
||||
* Selecting your process from the dropdown. In my case, it is just going to be labled
|
||||
`zstd`
|
||||
* Hitting the bright red record circle button on the top left of the toolbar
|
||||
6. You profiler will now start collecting metrics from your bencharking script. Once
|
||||
you think you have collected enough samples (usually this is the case after 3 seconds of
|
||||
recording), stop your profiler.
|
||||
7. Make sure that in toolbar of the bottom window, `profile` is selected.
|
||||
8. You should be able to see your call graph.
|
||||
* If you don't see the call graph or an incomplete call graph, make sure you have compiled
|
||||
zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
|
||||
you will have to supply the `-g` flag alone with your build script. You might also
|
||||
have to provide the `-fno-omit-frame-pointer` flag
|
||||
9. Dig down the graph to find your function call and then inspect it by double clicking
|
||||
the list item. You will be able to see the annotated source code and the assembly side by
|
||||
side.
|
||||
|
||||
#### Perf
|
||||
|
||||
This wiki has a pretty detailed tutorial on getting started working with perf so we'll
|
||||
leave you to check that out of you're getting started:
|
||||
|
||||
https://perf.wiki.kernel.org/index.php/Tutorial
|
||||
|
||||
Some general notes on perf:
|
||||
* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
|
||||
counter statistics. Perf uses a high resolution timer and this is likely one
|
||||
of the first things your team will run when assessing your PR.
|
||||
* Perf has a long list of hardware counters that can be viewed with `perf --list`.
|
||||
When measuring optimizations, something worth trying is to make sure the handware
|
||||
counters you expect to be impacted by your change are in fact being so. For example,
|
||||
if you expect the L1 cache misses to decrease with your change, you can look at the
|
||||
counter `L1-dcache-load-misses`
|
||||
* Perf hardware counters will not work on a virtual machine.
|
||||
|
||||
#### Visual Studio
|
||||
|
||||
TODO
|
||||
|
||||
|
||||
## Setting up continuous integration (CI) on your fork
|
||||
Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
|
||||
are well tested before they make it to an official release. Specifically, we use the platforms
|
||||
travis-ci, circle-ci, and appveyor.
|
||||
|
||||
Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
|
||||
The easiest way to run these CI tests on your own before submitting a PR to our dev branch
|
||||
is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
|
||||
instructions for doing this.
|
||||
|
||||
### travis-ci
|
||||
Follow these steps to link travis-ci with your github fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://travis-ci.org/
|
||||
3. Click 'Sign in with Github' on the top right
|
||||
4. Click 'Authorize travis-ci'
|
||||
5. Click 'Activate all repositories using Github Apps'
|
||||
6. Select 'Only select repositories' and select your fork of zstd from the drop down
|
||||
7. Click 'Approve and Install'
|
||||
8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
|
||||
9. Click 'Authorize travis-pro'
|
||||
10. You should have travis set up on your fork now.
|
||||
|
||||
### circle-ci
|
||||
TODO
|
||||
|
||||
### appveyor
|
||||
Follow these steps to link circle-ci with your girhub fork of zstd
|
||||
|
||||
1. Make sure you are logged into your github account
|
||||
2. Go to https://www.appveyor.com/
|
||||
3. Click 'Sign in' on the top right
|
||||
4. Select 'Github' on the left panel
|
||||
5. Click 'Authorize appveyor'
|
||||
6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
|
||||
7. You should have appveyor set up on your fork now.
|
||||
|
||||
### General notes on CI
|
||||
CI tests run every time a pull request (PR) is created or updated. The exact tests
|
||||
that get run will depend on the destination branch you specify. Some tests take
|
||||
longer to run than others. Currently, our CI is set up to run a short
|
||||
series of tests when creating a PR to the dev branch and a longer series of tests
|
||||
when creating a PR to the release branch. You can look in the configuration files
|
||||
of the respective CI platform for more information on what gets run when.
|
||||
|
||||
Most people will just want to create a PR with the destination set to their local dev
|
||||
branch of zstd. You can then find the status of the tests on the PR's page. You can also
|
||||
re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
|
||||
|
||||
## Issues
|
||||
We use GitHub issues to track public bugs. Please ensure your description is
|
||||
clear and has sufficient instructions to be able to reproduce the issue.
|
||||
@@ -398,7 +34,7 @@ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
|
||||
disclosure of security bugs. In those cases, please go through the process
|
||||
outlined on that page and do not file a public issue.
|
||||
|
||||
## Coding Style
|
||||
## Coding Style
|
||||
* 4 spaces for indentation rather than tabs
|
||||
|
||||
## License
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
|
||||
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
# verbose mode (print commands) on V=1 or VERBOSE=1
|
||||
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
BUILDIR = build
|
||||
@@ -21,19 +17,10 @@ FUZZDIR = $(TESTDIR)/fuzz
|
||||
# Define nul output
|
||||
VOID = /dev/null
|
||||
|
||||
# When cross-compiling from linux to windows, you might
|
||||
# need to specify this as "Windows." Fedora build fails
|
||||
# without it.
|
||||
#
|
||||
# Note: mingw-w64 build from linux to windows does not
|
||||
# fail on other tested distros (ubuntu, debian) even
|
||||
# without manually specifying the TARGET_SYSTEM.
|
||||
TARGET_SYSTEM ?= $(OS)
|
||||
|
||||
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
|
||||
EXT =.exe
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
else
|
||||
EXT =
|
||||
EXT =
|
||||
endif
|
||||
|
||||
## default: Build lib-release and zstd-release
|
||||
@@ -49,27 +36,27 @@ allmost: allzstd zlibwrapper
|
||||
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||
.PHONY: allzstd
|
||||
allzstd: lib
|
||||
$(Q)$(MAKE) -C $(PRGDIR) all
|
||||
$(Q)$(MAKE) -C $(TESTDIR) all
|
||||
$(MAKE) -C $(PRGDIR) all
|
||||
$(MAKE) -C $(TESTDIR) all
|
||||
|
||||
.PHONY: all32
|
||||
all32:
|
||||
$(MAKE) -C $(PRGDIR) zstd32
|
||||
$(MAKE) -C $(TESTDIR) all32
|
||||
|
||||
.PHONY: lib lib-release
|
||||
.PHONY: lib lib-release libzstd.a
|
||||
lib lib-release :
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
|
||||
.PHONY: zstd zstd-release
|
||||
zstd zstd-release:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) .
|
||||
|
||||
.PHONY: zstdmt
|
||||
zstdmt:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
|
||||
.PHONY: zlibwrapper
|
||||
zlibwrapper: lib
|
||||
@@ -77,33 +64,24 @@ zlibwrapper: lib
|
||||
|
||||
## test: run long-duration tests
|
||||
.PHONY: test
|
||||
DEBUGLEVEL ?= 1
|
||||
test: MOREFLAGS += -g -Werror
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
||||
test:
|
||||
DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
|
||||
|
||||
## shortest: same as `make check`
|
||||
.PHONY: shortest
|
||||
shortest:
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
## check: run basic tests for `zstd` cli
|
||||
.PHONY: check
|
||||
check: shortest
|
||||
|
||||
.PHONY: automated_benchmarking
|
||||
automated_benchmarking:
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: benchmarking
|
||||
benchmarking: automated_benchmarking
|
||||
|
||||
## examples: build all examples in `examples/` directory
|
||||
## examples: build all examples in `/examples` directory
|
||||
.PHONY: examples
|
||||
examples: lib
|
||||
$(MAKE) -C examples all
|
||||
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
|
||||
|
||||
## manual: generate API documentation in html format
|
||||
.PHONY: manual
|
||||
@@ -120,10 +98,8 @@ man:
|
||||
contrib: lib
|
||||
$(MAKE) -C contrib/pzstd all
|
||||
$(MAKE) -C contrib/seekable_format/examples all
|
||||
$(MAKE) -C contrib/seekable_format/tests test
|
||||
$(MAKE) -C contrib/adaptive-compression all
|
||||
$(MAKE) -C contrib/largeNbDicts all
|
||||
cd contrib/single_file_libs/ ; ./build_decoder_test.sh
|
||||
cd contrib/single_file_libs/ ; ./build_library_test.sh
|
||||
|
||||
.PHONY: cleanTabs
|
||||
cleanTabs:
|
||||
@@ -131,18 +107,18 @@ cleanTabs:
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C examples/ $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
$(Q)$(RM) -r lz4
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C examples/ $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/pzstd $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/adaptive-compression $@ > $(VOID)
|
||||
@$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
@$(RM) -r lz4
|
||||
@echo Cleaning completed
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
@@ -153,12 +129,7 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
|
||||
HOST_OS = POSIX
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
EGREP = egrep --color=never
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
@@ -166,7 +137,7 @@ EGREP = egrep $(EGREP_OPTIONS)
|
||||
## list: Print all targets and their descriptions (if provided)
|
||||
.PHONY: list
|
||||
list:
|
||||
$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
|
||||
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
|
||||
{ \
|
||||
@@ -179,15 +150,15 @@ list:
|
||||
done \
|
||||
} | column -t -s $$'\t'
|
||||
|
||||
.PHONY: install armtest usan asan uasan
|
||||
.PHONY: install clangtest armtest usan asan uasan
|
||||
install:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: travis-install
|
||||
travis-install:
|
||||
@@ -211,7 +182,7 @@ gcc7build: clean
|
||||
.PHONY: clangbuild
|
||||
clangbuild: clean
|
||||
clang -v
|
||||
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
|
||||
CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
|
||||
|
||||
m32build: clean
|
||||
gcc -v
|
||||
@@ -224,10 +195,10 @@ aarch64build: clean
|
||||
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
ppcbuild: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
|
||||
ppc64build: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
|
||||
|
||||
armfuzz: clean
|
||||
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
@@ -255,6 +226,10 @@ gcc6test: clean
|
||||
gcc-6 -v
|
||||
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
|
||||
|
||||
clangtest: clean
|
||||
clang -v
|
||||
$(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
|
||||
@@ -286,11 +261,12 @@ uasanregressiontest:
|
||||
msanregressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
|
||||
|
||||
# run UBsan with -fsanitize-recover=pointer-overflow
|
||||
# this only works with recent compilers such as gcc 8+
|
||||
# run UBsan with -fsanitize-recover=signed-integer-overflow
|
||||
# due to a bug in UBsan when doing pointer subtraction
|
||||
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
|
||||
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
|
||||
@@ -308,10 +284,10 @@ asan32: clean
|
||||
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
|
||||
|
||||
uasan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
tsan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
|
||||
@@ -359,7 +335,7 @@ endif
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
endif
|
||||
|
||||
|
||||
@@ -371,15 +347,11 @@ cmakebuild:
|
||||
cmake --version
|
||||
$(RM) -r $(BUILDIR)/cmake/build
|
||||
mkdir $(BUILDIR)/cmake/build
|
||||
cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build -j4;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build install;
|
||||
$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
|
||||
cd $(BUILDIR)/cmake/build; ctest -V -L Medium
|
||||
cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||
|
||||
c89build: clean
|
||||
c90build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c89 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
CFLAGS="-std=c90 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long`
|
||||
|
||||
gnu90build: clean
|
||||
$(CC) -v
|
||||
|
||||
+50
-290
@@ -1,251 +1,11 @@
|
||||
v1.4.9 (Mar 1, 2021)
|
||||
bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
|
||||
bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
|
||||
api: Add (De)Compression Tracing Functionality (#2482, @terrelln)
|
||||
api: Support References to Multiple DDicts (#2446, @senhuang42)
|
||||
api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
|
||||
perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
|
||||
perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
|
||||
perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
|
||||
cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
|
||||
cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
|
||||
cli: Improve Help Message (#2500, @senhuang42)
|
||||
tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
|
||||
tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat)
|
||||
tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte)
|
||||
build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973)
|
||||
build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt)
|
||||
build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime)
|
||||
build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka)
|
||||
build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln)
|
||||
build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg)
|
||||
build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973)
|
||||
contrib: Add `recover_directory` Program (#2473, @terrelln)
|
||||
doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42)
|
||||
doc: Fix Typos (#2459, @ThomasWaldmann)
|
||||
|
||||
v1.4.8 (Dec 18, 2020)
|
||||
hotfix: wrong alignment of an internal buffer
|
||||
|
||||
v1.4.7 (Dec 16, 2020)
|
||||
perf: stronger --long mode at high compression levels, by @senhuang42
|
||||
perf: stronger --patch-from at high compression levels, thanks to --long improvements
|
||||
perf: faster dictionary compression at medium compression levels, by @felixhandte
|
||||
perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln
|
||||
perf: improved fast compression speeds with Visual Studio, by @animalize
|
||||
cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42
|
||||
cli : accept decompressing files with *.zstd suffix
|
||||
cli : provide a condensed summary by default when processing multiple files
|
||||
cli : fix : stdin input no longer confused as user prompt
|
||||
cli : improve accuracy of several error messages
|
||||
api : new sequence ingestion API, by @senhuang42
|
||||
api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin
|
||||
api : new ZSTD_getDictID_fromCDict(), by @LuAPi
|
||||
api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln
|
||||
api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli)
|
||||
api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli)
|
||||
build: fix cmake script when using path with spaces, by @terrelln
|
||||
build: improved compile-time detection of aarch64/neon platforms, by @bsdimp
|
||||
build: Fix building on AIX 5.1, by @likema
|
||||
build: compile paramgrill with cmake on Windows, requested by @mirh
|
||||
doc : clarify repcode updates in format specification, by @felixhandte
|
||||
|
||||
v1.4.6
|
||||
fix : Always return dstSize_tooSmall when that is the case
|
||||
fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary
|
||||
perf: Improve small block decompression speed by 20%+, by @terrelln
|
||||
perf: Reduce compression stack usage by 1 KB, by @terrelln
|
||||
perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256)
|
||||
perf: Improve histogram construction, by @cyan4973 (#2253)
|
||||
cli : Add --output-dir-mirror option, by @xxie24 (#2219)
|
||||
cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279)
|
||||
cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283)
|
||||
cli : Call stat less often, by @felixhandte (#2262)
|
||||
cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250)
|
||||
cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206)
|
||||
api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166).
|
||||
api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291)
|
||||
api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228).
|
||||
build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183)
|
||||
build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221)
|
||||
build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186)
|
||||
build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions
|
||||
build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics
|
||||
build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258)
|
||||
build: Fix -Wcomma warnings, by @cwoffenden
|
||||
build: Remove distutils requirement for meson build, by @neheb (#2197)
|
||||
build: Fix cli compilation with uclibc
|
||||
build: Fix cli compilation without st_mtime, by @ffontaine (#2246)
|
||||
build: Fix shadowing warnings in library
|
||||
build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227)
|
||||
misc: Improve single file library and include dictBuilder, by @cwoffenden
|
||||
misc: Allow compression dictionaries with missing symbols
|
||||
misc: Add freestanding translation script in contrib/freestanding_lib
|
||||
misc: Collect all of zstd's libc dependencies into zstd_deps.h
|
||||
doc : Add ZSTD_versionString() to manual, by @animalize
|
||||
doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270)
|
||||
|
||||
v1.4.5 (May 22, 2020)
|
||||
fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
|
||||
perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
|
||||
perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
|
||||
perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
|
||||
perf: Small level 1 compression speed gains (depending on compiler)
|
||||
cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
|
||||
cli : New --filelist= : Provide a list of files to operate upon from a file
|
||||
cli : -b -d command can now benchmark decompression on multiple files
|
||||
cli : New --no-content-size command
|
||||
cli : New --show-default-cparams information command
|
||||
api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
|
||||
api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
|
||||
build: Generate a single-file libzstd library (#2065, by @cwoffenden)
|
||||
build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
|
||||
build: zstd now compiles cleanly under -pedantic (#2099)
|
||||
build: zstd now compiles with make-4.3
|
||||
build: Support mingw cross-compilation from Linux, by @Ericson2314
|
||||
build: Meson multi-thread build fix on windows
|
||||
build: Some misc icc fixes backed by new ci test on travis
|
||||
misc: bitflip analyzer tool, by @felixhandte
|
||||
misc: Extend largeNbDicts benchmark to compression
|
||||
misc: Edit-distance match finder in contrib/
|
||||
doc : Improved beginner CONTRIBUTING.md docs
|
||||
doc : New issue templates for zstd
|
||||
|
||||
v1.4.4 (Nov 6, 2019)
|
||||
perf: Improved decompression speed, by > 10%, by @terrelln
|
||||
perf: Better compression speed when re-using a context, by @felixhandte
|
||||
perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
|
||||
perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
|
||||
perf: minor generic speed optimization, by @davidbolvansky
|
||||
api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
|
||||
api: fixed decoding of magic-less frames, by @terrelln
|
||||
api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
|
||||
cli: Named pipes support, by @bimbashrestha
|
||||
cli: short tar's extension support, by @stokito
|
||||
cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
|
||||
cli: commands --stream-size=# and --size-hint=#, by @nmagerko
|
||||
cli: command --exclude-compressed, by @shashank0791
|
||||
cli: faster `-t` test mode
|
||||
cli: improved some error messages, by @vangyzen
|
||||
cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
|
||||
cli: fix rare deadlock condition within dictionary builder, by @terrelln
|
||||
build: single-file decoder with emscripten compilation script, by @cwoffenden
|
||||
build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
|
||||
build: fixed deprecation warning for certain gcc version, reported by @jasonma163
|
||||
build: fix compilation on old gcc versions, by @cemeyer
|
||||
build: improved installation directories for cmake script, by Dmitri Shubin
|
||||
pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
|
||||
misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
|
||||
misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
|
||||
|
||||
v1.4.3 (Aug 20, 2019)
|
||||
bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
|
||||
bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
|
||||
build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
|
||||
|
||||
v1.4.2 (Jul 26, 2019)
|
||||
bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
|
||||
bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
|
||||
misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
|
||||
misc: Restructure source files by @ephiepark (#1679)
|
||||
|
||||
v1.4.1 (Jul 20, 2019)
|
||||
bug: Fix data corruption in niche use cases by @terrelln (#1659)
|
||||
bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
|
||||
bug: Fix out of bounds read by @terrelln (#1590)
|
||||
perf: Improve decode speed by ~7% @mgrice (#1668)
|
||||
perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
|
||||
perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
|
||||
perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
|
||||
perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
|
||||
api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
|
||||
cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
|
||||
cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
|
||||
cli: Restrict read permissions on destination files by @chungy (#1644)
|
||||
cli: zstdgrep: handle -f flag by @felixhandte (#1618)
|
||||
cli: zstdcat: follow symlinks by @vejnar (#1604)
|
||||
doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
|
||||
doc: Fix typo by @yk-tanigawa (#1633)
|
||||
doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
|
||||
build: CMake: support building with LZ4 @leeyoung624 (#1626)
|
||||
build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
|
||||
build: CMake: respect existing uninstall target by @j301scott (#1619)
|
||||
build: Make: skip multithread tests when built without support by @michaelforney (#1620)
|
||||
build: Make: Fix examples/ test target by @sjnam (#1603)
|
||||
build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
|
||||
build: Meson: fix build by @lzutao (#1602)
|
||||
build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
|
||||
build: Visual Studio: fix linking by @absotively (#1639)
|
||||
build: Fix MinGW-W64 build by @myzhang1029 (#1600)
|
||||
misc: Expand decodecorpus coverage by @ephiepark (#1664)
|
||||
|
||||
v1.4.0 (Apr 17, 2019)
|
||||
perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
|
||||
api: Move the advanced API, including all functions in the staging section, to the stable section
|
||||
api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
|
||||
api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
|
||||
api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
|
||||
api: Don't export ZSTDMT functions from the shared library by default
|
||||
api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
|
||||
api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
|
||||
api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
|
||||
api: Move ZSTD_getDictID_*() functions to the stable section
|
||||
api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
|
||||
api: Allow compression parameters to be set when a dictionary is used
|
||||
api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
|
||||
api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
|
||||
api: Setting ZSTD_d_maxWindowLog to 0 means use the default
|
||||
cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
|
||||
cli: Add --[no-]compress-literals flag to enable or disable literal compression
|
||||
doc: Update the examples to use the advanced API
|
||||
doc: Explain how to transition from old streaming functions to the advanced API in the header
|
||||
build: Improve the Windows release packages
|
||||
build: Improve CMake build by @hjmjohnson
|
||||
build: Build fixes for FreeBSD by @lwhsu
|
||||
build: Remove redundant warnings by @thatsafunnyname
|
||||
build: Fix tests on OpenBSD by @bket
|
||||
build: Extend fuzzer build system to work with the new clang engine
|
||||
build: CMake now creates the libzstd.so.1 symlink
|
||||
build: Improve Menson build by @lzutao
|
||||
misc: Fix symbolic link detection on FreeBSD
|
||||
misc: Use physical core count for -T0 on FreeBSD by @cemeyer
|
||||
misc: Fix zstd --list on truncated files by @kostmo
|
||||
misc: Improve logging in debug mode by @felixhandte
|
||||
misc: Add CirrusCI tests by @lwhsu
|
||||
misc: Optimize dictionary memory usage in corner cases
|
||||
misc: Improve the dictionary builder on small or homogeneous data
|
||||
misc: Fix spelling across the repo by @jsoref
|
||||
|
||||
v1.3.8 (Dec 28, 2018)
|
||||
perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
|
||||
perf: slightly better compression ratio at high compression modes
|
||||
api : finalized advanced API, last stage before "stable" status
|
||||
api : new --rsyncable mode, by @terrelln
|
||||
api : support decompression of empty frames into NULL (used to be an error) (#1385)
|
||||
build: new set of macros to build a minimal size decoder, by @felixhandte
|
||||
build: fix compilation on MIPS32, reported by @clbr (#1441)
|
||||
build: fix compilation with multiple -arch flags, by @ryandesign
|
||||
build: highly upgraded meson build, by @lzutao
|
||||
build: improved buck support, by @obelisk
|
||||
build: fix cmake script : can create debug build, by @pitrou
|
||||
build: Makefile : grep works on both colored consoles and systems without color support
|
||||
build: fixed zstd-pgo, by @bmwiedemann
|
||||
cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
|
||||
cli : --no-progress flag, preserving final summary (#1371), by @terrelln
|
||||
cli : ensure destination file is not source file (#1422)
|
||||
cli : clearer error messages, especially when input file not present
|
||||
doc : clarified zstd_compression_format.md, by @ulikunitz
|
||||
misc: fixed zstdgrep, returns 1 on failure, by @lzutao
|
||||
misc: NEWS renamed as CHANGELOG, in accordance with fboss
|
||||
|
||||
v1.3.7 (Oct 20, 2018)
|
||||
v1.3.7
|
||||
perf: slightly better decompression speed on clang (depending on hardware target)
|
||||
fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
|
||||
build: no longer build backtrace by default in release mode; restrict further automatic mode
|
||||
build: control backtrace support through build macro BACKTRACE
|
||||
misc: added man pages for zstdless and zstdgrep, by @samrussell
|
||||
|
||||
v1.3.6 (Oct 6, 2018)
|
||||
v1.3.6
|
||||
perf: much faster dictionary builder, by @jenniferliu
|
||||
perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
|
||||
perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
|
||||
@@ -259,7 +19,7 @@ build: Read Legacy format is limited to v0.5+ by default. Can be changed at comp
|
||||
doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
|
||||
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
|
||||
|
||||
v1.3.5 (Jun 29, 2018)
|
||||
v1.3.5
|
||||
perf: much faster dictionary compression, by @felixhandte
|
||||
perf: small quality improvement for dictionary generation, by @terrelln
|
||||
perf: slightly improved high compression levels (notably level 19)
|
||||
@@ -274,7 +34,7 @@ build: make and make all are compatible with -j
|
||||
doc : clarify zstd_compression_format.md, updated for IETF RFC process
|
||||
misc: pzstd compatible with reproducible compilation, by @lamby
|
||||
|
||||
v1.3.4 (Mar 27, 2018)
|
||||
v1.3.4
|
||||
perf: faster speed (especially decoding speed) on recent cpus (haswell+)
|
||||
perf: much better performance associating --long with multi-threading, by @terrelln
|
||||
perf: better compression at levels 13-15
|
||||
@@ -292,7 +52,7 @@ build: VS2017 scripts, by @HaydnTrigg
|
||||
misc: all /contrib projects fixed
|
||||
misc: added /contrib/docker script by @gyscos
|
||||
|
||||
v1.3.3 (Dec 21, 2017)
|
||||
v1.3.3
|
||||
perf: faster zstd_opt strategy (levels 16-19)
|
||||
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
|
||||
cli : fix : content size written in header by default
|
||||
@@ -304,7 +64,7 @@ api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` mac
|
||||
build: fix : compilation under rhel6 and centos6, reported by @pixelb
|
||||
build: added `check` target
|
||||
|
||||
v1.3.2 (Oct 10, 2017)
|
||||
v1.3.2
|
||||
new : long range mode, using --long command, by Stella Lau (@stellamplau)
|
||||
new : ability to generate and decode magicless frames (#591)
|
||||
changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
|
||||
@@ -327,7 +87,7 @@ example : added streaming_memory_usage
|
||||
license : changed /examples license to BSD + GPLv2
|
||||
license : fix a few header files to reflect new license (#825)
|
||||
|
||||
v1.3.1 (Aug 21, 2017)
|
||||
v1.3.1
|
||||
New license : BSD + GPLv2
|
||||
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
|
||||
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
|
||||
@@ -342,7 +102,7 @@ new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cru
|
||||
new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
|
||||
updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
|
||||
|
||||
v1.3.0 (Jul 6, 2017)
|
||||
v1.3.0
|
||||
cli : new : `--list` command, by Paul Cruz
|
||||
cli : changed : xz/lzma support enabled by default
|
||||
cli : changed : `-t *` continue processing list after a decompression error
|
||||
@@ -357,7 +117,7 @@ tools : decodecorpus can generate random dictionary-compressed samples, by Paul
|
||||
new : contrib/seekable_format, demo and API, by Sean Purcell
|
||||
changed : contrib/linux-kernel, updated version and license, by Nick Terrell
|
||||
|
||||
v1.2.0 (May 5, 2017)
|
||||
v1.2.0
|
||||
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
|
||||
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
|
||||
cli : new : zstdmt symlink hardwired to `zstd -T0`
|
||||
@@ -379,7 +139,7 @@ build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
|
||||
tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
|
||||
new : contrib/linux-kernel version, by Nick Terrell
|
||||
|
||||
v1.1.4 (Mar 18, 2017)
|
||||
v1.1.4
|
||||
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
||||
cli : new : advanced benchmark command --priority=rt
|
||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||
@@ -395,7 +155,7 @@ build : improved cmake script, by @Majlen
|
||||
build : added -Wformat-security flag, as recommended by Padraig Brady
|
||||
doc : new : educational decoder, by Sean Purcell
|
||||
|
||||
v1.1.3 (Feb 7, 2017)
|
||||
v1.1.3
|
||||
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
|
||||
cli : new : experimental target `make zstdmt`, with multi-threading support
|
||||
cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
|
||||
@@ -411,7 +171,7 @@ API : fix : all symbols properly exposed in libzstd, by Nick Terrell
|
||||
build : support for Solaris target, by Przemyslaw Skibinski
|
||||
doc : clarified specification, by Sean Purcell
|
||||
|
||||
v1.1.2 (Dec 15, 2016)
|
||||
v1.1.2
|
||||
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
|
||||
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
|
||||
API : zbuff : changed : prototypes now generate deprecation warnings
|
||||
@@ -428,7 +188,7 @@ zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
|
||||
install : better compatibility with FreeBSD, by Dimitry Andric
|
||||
source tree : changed : zbuff source files moved to lib/deprecated
|
||||
|
||||
v1.1.1 (Nov 2, 2016)
|
||||
v1.1.1
|
||||
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
|
||||
New : doc/zstd_manual.html, by Przemyslaw Skibinski
|
||||
Improved : slightly better compression ratio at --ultra levels (>= 20)
|
||||
@@ -439,7 +199,7 @@ Changed : zstd_errors.h is now installed within /include (and replaces errors_pu
|
||||
Updated man page
|
||||
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
|
||||
|
||||
v1.1.0 (Sep 28, 2016)
|
||||
v1.1.0
|
||||
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
||||
added : NetBSD install target (#338)
|
||||
Improved : speed for batches of small files
|
||||
@@ -453,16 +213,16 @@ Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (
|
||||
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
|
||||
Fixed : zstd-pgo, reported by octoploid (#329)
|
||||
|
||||
v1.0.0 (Sep 1, 2016)
|
||||
v1.0.0
|
||||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.1 (Aug 18, 2016)
|
||||
v0.8.1
|
||||
New streaming API
|
||||
Changed : --ultra now enables levels beyond 19
|
||||
Changed : -i# now selects benchmark time in second
|
||||
@@ -471,7 +231,7 @@ Fixed : speed regression on specific patterns (#272)
|
||||
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
|
||||
Fixed : ICC compilation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.0 (Aug 2, 2016)
|
||||
v0.8.0
|
||||
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
|
||||
New : Build on FreeBSD and DragonFly, thanks to JrMarino
|
||||
Changed : modified API : ZSTD_compressEnd()
|
||||
@@ -484,17 +244,17 @@ Modified : minor compression level adaptations
|
||||
Updated : compression format specification to v0.2.0
|
||||
changed : zstd.h moved to /lib directory
|
||||
|
||||
v0.7.5 (Aug 1, 2016)
|
||||
v0.7.5
|
||||
Transition version, supporting decoding of v0.8.x
|
||||
|
||||
v0.7.4 (Jul 17, 2016)
|
||||
v0.7.4
|
||||
Added : homebrew for Mac, by Daniel Cade
|
||||
Added : more examples
|
||||
Fixed : segfault when using small dictionaries, reported by Felix Handte
|
||||
Modified : default compression level for CLI is now 3
|
||||
Updated : specification, to v0.1.1
|
||||
|
||||
v0.7.3 (Jul 9, 2016)
|
||||
v0.7.3
|
||||
New : compression format specification
|
||||
New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
|
||||
New : `ZSTD_getDecompressedSize()`
|
||||
@@ -506,18 +266,18 @@ fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by
|
||||
modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
|
||||
modified : legacy functions no longer need magic number
|
||||
|
||||
v0.7.2 (Jul 4, 2016)
|
||||
v0.7.2
|
||||
fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
|
||||
fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
|
||||
fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
|
||||
|
||||
v0.7.1 (Jun 23, 2016)
|
||||
v0.7.1
|
||||
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
|
||||
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
|
||||
fixed : corruption issue, reported by cj
|
||||
modified : checksum enabled by default in command line mode
|
||||
|
||||
v0.7.0 (Jun 17, 2016)
|
||||
v0.7.0
|
||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||
New : Command `--rm`, to remove source file after successful de/compression
|
||||
New : Visual build scripts, by Christophe Chevalier
|
||||
@@ -530,7 +290,7 @@ API : support for custom malloc/free functions
|
||||
New : controllable Dictionary ID
|
||||
New : Support for skippable frames
|
||||
|
||||
v0.6.1 (May 13, 2016)
|
||||
v0.6.1
|
||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||
New : Ability to compile compressor / decompressor separately
|
||||
Changed : new lib directory structure
|
||||
@@ -540,103 +300,103 @@ Fixed : null-string roundtrip (#176)
|
||||
New : benchmark mode can select directory as input
|
||||
Experimental : midipix support, VMS support
|
||||
|
||||
v0.6.0 (Apr 13, 2016)
|
||||
v0.6.0
|
||||
Stronger high compression modes, thanks to Przemyslaw Skibinski
|
||||
API : ZSTD_getFrameParams() provides size of decompressed content
|
||||
New : highest compression modes require `--ultra` command to fully unleash their capacity
|
||||
Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
|
||||
|
||||
v0.5.1 (Feb 18, 2016)
|
||||
v0.5.1
|
||||
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
|
||||
Changed : Dictionary builder integrated into libzstd and zstd cli
|
||||
Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
|
||||
Fix : high compression modes for big-endian platforms
|
||||
New : zstd cli : `-t` | `--test` command
|
||||
|
||||
v0.5.0 (Feb 5, 2016)
|
||||
v0.5.0
|
||||
New : dictionary builder utility
|
||||
Changed : streaming & dictionary API
|
||||
Improved : better compression of small data
|
||||
|
||||
v0.4.7 (Jan 22, 2016)
|
||||
v0.4.7
|
||||
Improved : small compression speed improvement in HC mode
|
||||
Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
|
||||
fix : bt search bug
|
||||
|
||||
v0.4.6 (Jan 13, 2016)
|
||||
v0.4.6
|
||||
fix : fast compression mode on Windows
|
||||
New : cmake configuration file, thanks to Artyom Dymchenko
|
||||
Improved : high compression mode on repetitive data
|
||||
New : block-level API
|
||||
New : ZSTD_duplicateCCtx()
|
||||
|
||||
v0.4.5 (Dec 18, 2015)
|
||||
v0.4.5
|
||||
new : -m/--multiple : compress/decompress multiple files
|
||||
|
||||
v0.4.4 (Dec 14, 2015)
|
||||
v0.4.4
|
||||
Fixed : high compression modes for Windows 32 bits
|
||||
new : external dictionary API extended to buffered mode and accessible through command line
|
||||
new : windows DLL project, thanks to Christophe Chevalier
|
||||
|
||||
v0.4.3 (Dec 7, 2015)
|
||||
v0.4.3 :
|
||||
new : external dictionary API
|
||||
new : zstd-frugal
|
||||
|
||||
v0.4.2 (Dec 2, 2015)
|
||||
v0.4.2 :
|
||||
Generic minor improvements for small blocks
|
||||
Fixed : big-endian compatibility, by Peter Harris (#85)
|
||||
|
||||
v0.4.1 (Dec 1, 2015)
|
||||
v0.4.1
|
||||
Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
|
||||
removed `zstd.c`
|
||||
|
||||
v0.4.0 (Nov 29, 2015)
|
||||
v0.4.0
|
||||
Command line utility compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
|
||||
v0.3.6 (Nov 10, 2015)
|
||||
v0.3.6
|
||||
small blocks params
|
||||
|
||||
v0.3.5 (Nov 9, 2015)
|
||||
v0.3.5
|
||||
minor generic compression improvements
|
||||
|
||||
v0.3.4 (Nov 6, 2015)
|
||||
v0.3.4
|
||||
Faster fast cLevels
|
||||
|
||||
v0.3.3 (Nov 5, 2015)
|
||||
v0.3.3
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3.2 (Nov 2, 2015)
|
||||
v0.3.2
|
||||
Fixed Visual Studio
|
||||
|
||||
v0.3.1 (Nov 2, 2015)
|
||||
v0.3.1 :
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3 (Oct 30, 2015)
|
||||
v0.3
|
||||
HC mode : compression levels 2-26
|
||||
|
||||
v0.2.2 (Oct 28, 2015)
|
||||
v0.2.2
|
||||
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
|
||||
|
||||
v0.2.1 (Oct 24, 2015)
|
||||
v0.2.1
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
|
||||
v0.2.0 (Oct 22, 2015)
|
||||
v0.2.0
|
||||
**Breaking format change**
|
||||
Faster decompression speed
|
||||
Can still decode v0.1 format
|
||||
|
||||
v0.1.3 (Oct 15, 2015)
|
||||
v0.1.3
|
||||
fix uninitialization warning, reported by Evan Nemerson
|
||||
|
||||
v0.1.2 (Sep 11, 2015)
|
||||
v0.1.2
|
||||
frame concatenation support
|
||||
|
||||
v0.1.1 (Aug 27, 2015)
|
||||
v0.1.1
|
||||
fix compression bug
|
||||
detects write-flush errors
|
||||
|
||||
v0.1.0 (Aug 25, 2015)
|
||||
v0.1.0
|
||||
first release
|
||||
@@ -9,13 +9,7 @@ and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4
|
||||
Should your project require another programming language,
|
||||
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
**Development branch status:**
|
||||
|
||||
[![Build Status][travisDevBadge]][travisLink]
|
||||
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
||||
[![Build status][CircleDevBadge]][CircleLink]
|
||||
[![Build status][CirrusDevBadge]][CirrusLink]
|
||||
[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
|
||||
Development branch status : [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
|
||||
|
||||
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
||||
[travisLink]: https://travis-ci.org/facebook/zstd
|
||||
@@ -23,18 +17,14 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
|
||||
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
|
||||
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
||||
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
||||
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
|
||||
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
|
||||
[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
|
||||
[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
|
||||
|
||||
## Benchmarks
|
||||
### Benchmarks
|
||||
|
||||
For reference, several fast compression algorithms were tested and compared
|
||||
on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
|
||||
with a Core i9-9900K CPU @ 5.0GHz,
|
||||
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
|
||||
with a Core i7-6700K CPU @ 4.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
compiled with [gcc] 9.3.0,
|
||||
compiled with [gcc] 7.3.0,
|
||||
on the [Silesia compression corpus].
|
||||
|
||||
[lzbench]: https://github.com/inikep/lzbench
|
||||
@@ -43,26 +33,18 @@ on the [Silesia compression corpus].
|
||||
|
||||
| Compressor name | Ratio | Compression| Decompress.|
|
||||
| --------------- | ------| -----------| ---------- |
|
||||
| **zstd 1.4.5 -1** | 2.884 | 500 MB/s | 1660 MB/s |
|
||||
| zlib 1.2.11 -1 | 2.743 | 90 MB/s | 400 MB/s |
|
||||
| brotli 1.0.7 -0 | 2.703 | 400 MB/s | 450 MB/s |
|
||||
| **zstd 1.4.5 --fast=1** | 2.434 | 570 MB/s | 2200 MB/s |
|
||||
| **zstd 1.4.5 --fast=3** | 2.312 | 640 MB/s | 2300 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 560 MB/s | 710 MB/s |
|
||||
| **zstd 1.4.5 --fast=5** | 2.178 | 700 MB/s | 2420 MB/s |
|
||||
| lzo1x 2.10 -1 | 2.106 | 690 MB/s | 820 MB/s |
|
||||
| lz4 1.9.2 | 2.101 | 740 MB/s | 4530 MB/s |
|
||||
| **zstd 1.4.5 --fast=7** | 2.096 | 750 MB/s | 2480 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 410 MB/s | 860 MB/s |
|
||||
| snappy 1.1.8 | 2.073 | 560 MB/s | 1790 MB/s |
|
||||
| **zstd 1.3.4 -1** | 2.877 | 470 MB/s | 1380 MB/s |
|
||||
| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 400 MB/s |
|
||||
| brotli 1.0.2 -0 | 2.701 | 410 MB/s | 430 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s |
|
||||
| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s |
|
||||
| lz4 1.8.1 | 2.101 | 750 MB/s | 3700 MB/s |
|
||||
| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
|
||||
|
||||
[zlib]: http://www.zlib.net/
|
||||
[zlib]:http://www.zlib.net/
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
The negative compression levels, specified with `--fast=#`,
|
||||
offer faster compression and decompression speed in exchange for some loss in
|
||||
compression ratio compared to level 1, as seen in the table above.
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increments.
|
||||
Decompression speed is preserved and remains roughly the same at all settings,
|
||||
@@ -83,7 +65,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
|
||||
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
||||
|
||||
|
||||
## The case for Small Data compression
|
||||
### The case for Small Data compression
|
||||
|
||||
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
||||
|
||||
@@ -107,24 +89,24 @@ Training works if there is some correlation in a family of small data samples. T
|
||||
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
||||
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
### Dictionary compression How To:
|
||||
#### Dictionary compression How To:
|
||||
|
||||
1. Create the dictionary
|
||||
1) Create the dictionary
|
||||
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
|
||||
2. Compress with dictionary
|
||||
2) Compress with dictionary
|
||||
|
||||
`zstd -D dictionaryName FILE`
|
||||
`zstd -D dictionaryName FILE`
|
||||
|
||||
3. Decompress with dictionary
|
||||
3) Decompress with dictionary
|
||||
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
|
||||
|
||||
## Build instructions
|
||||
### Build instructions
|
||||
|
||||
### Makefile
|
||||
#### Makefile
|
||||
|
||||
If your system is compatible with standard `make` (or `gmake`),
|
||||
invoking `make` in root directory will generate `zstd` cli in root directory.
|
||||
@@ -133,7 +115,7 @@ Other available options include:
|
||||
- `make install` : create and install zstd cli, library and man pages
|
||||
- `make check` : create and run `zstd`, tests its behavior on local platform
|
||||
|
||||
### cmake
|
||||
#### cmake
|
||||
|
||||
A `cmake` project generator is provided within `build/cmake`.
|
||||
It can generate Makefiles or other build scripts
|
||||
@@ -141,29 +123,11 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||
|
||||
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||
|
||||
### Meson
|
||||
#### Meson
|
||||
|
||||
A Meson project is provided within [`build/meson`](build/meson). Follow
|
||||
build instructions in that directory.
|
||||
A Meson project is provided within `contrib/meson`.
|
||||
|
||||
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
||||
example about how Meson is used to build this project.
|
||||
|
||||
Note that default build type is **release**.
|
||||
|
||||
### VCPKG
|
||||
You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
|
||||
|
||||
git clone https://github.com/Microsoft/vcpkg.git
|
||||
cd vcpkg
|
||||
./bootstrap-vcpkg.sh
|
||||
./vcpkg integrate install
|
||||
./vcpkg install zstd
|
||||
|
||||
The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
|
||||
If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
|
||||
|
||||
### Visual Studio (Windows)
|
||||
#### Visual Studio (Windows)
|
||||
|
||||
Going into `build` directory, you will find additional possibilities:
|
||||
- Projects for Visual Studio 2005, 2008 and 2010.
|
||||
@@ -171,29 +135,19 @@ Going into `build` directory, you will find additional possibilities:
|
||||
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
||||
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
||||
|
||||
### Buck
|
||||
|
||||
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
||||
The output binary will be in `buck-out/gen/programs/`.
|
||||
|
||||
## Testing
|
||||
|
||||
You can run quick local smoke tests by executing the `playTest.sh` script from the `src/tests` directory.
|
||||
Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the zstd and datagen binary.
|
||||
For information on CI testing, please refer to TESTING.md
|
||||
|
||||
## Status
|
||||
### Status
|
||||
|
||||
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
||||
Zstandard is considered safe for production environments.
|
||||
|
||||
## License
|
||||
### License
|
||||
|
||||
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
||||
|
||||
## Contributing
|
||||
### Contributing
|
||||
|
||||
The `dev` branch is the one where all contributions are merged before reaching `release`.
|
||||
If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
|
||||
Direct commit to `release` are not permitted.
|
||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
||||
Direct commit to "master" are not permitted.
|
||||
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@ They consist of the following tests:
|
||||
- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
|
||||
- Compilation on various versions of gcc, clang, and g++
|
||||
- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
|
||||
|
||||
Medium Tests
|
||||
------------
|
||||
@@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
|
||||
They consist of the following tests:
|
||||
- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
|
||||
Msan on x86_64
|
||||
- `tests/playTests.sh --test-large-data`
|
||||
- `tests/playTests.sh --test-long-data`
|
||||
- Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
|
||||
- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
|
||||
- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
|
||||
@@ -27,7 +27,7 @@ They consist of the following tests:
|
||||
|
||||
Long Tests
|
||||
----------
|
||||
Long tests run on all commits to `release` branch,
|
||||
Long tests run on all commits to `master` branch,
|
||||
and once a day on the current version of `dev` branch,
|
||||
on TravisCI.
|
||||
They consist of the following tests:
|
||||
|
||||
+9
-52
@@ -1,20 +1,14 @@
|
||||
# Following tests are run _only_ on `release` branch
|
||||
# and on selected feature branch named `appveyorTest` or `visual*`
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
branches:
|
||||
only:
|
||||
- release
|
||||
- master
|
||||
- /appveyor*/
|
||||
- /visual*/
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static"
|
||||
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib"
|
||||
ARTIFACT: "true"
|
||||
BUILD: "true"
|
||||
- COMPILER: "gcc"
|
||||
@@ -26,7 +20,7 @@
|
||||
- COMPILER: "clang"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
|
||||
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
|
||||
BUILD: "true"
|
||||
|
||||
- COMPILER: "gcc"
|
||||
@@ -92,9 +86,9 @@
|
||||
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
||||
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
||||
cp zstd.exe ..\bin\zstd.exe &&
|
||||
git clone --depth 1 --branch release https://github.com/facebook/zstd &&
|
||||
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
||||
cd zstd &&
|
||||
git archive --format=tar release -o zstd-src.tar &&
|
||||
git archive --format=tar master -o zstd-src.tar &&
|
||||
..\zstd -19 zstd-src.tar &&
|
||||
appveyor PushArtifact zstd-src.tar.zst &&
|
||||
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
||||
@@ -162,8 +156,6 @@
|
||||
- if [%TEST%]==[cmake] (
|
||||
mkdir build\cmake\build &&
|
||||
cd build\cmake\build &&
|
||||
SET FUZZERTEST=-T2mn &&
|
||||
SET ZSTREAM_TESTTIME=-T2mn &&
|
||||
cmake -G "Visual Studio 14 2015 Win64" .. &&
|
||||
cd ..\..\.. &&
|
||||
make clean
|
||||
@@ -171,32 +163,25 @@
|
||||
- SET "FUZZERTEST=-T30s"
|
||||
- if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||
CD tests &&
|
||||
SET ZSTD_BIN=./zstd.exe&&
|
||||
SET DATAGEN_BIN=./datagen.exe&&
|
||||
SET ZSTD=./zstd.exe &&
|
||||
sh -e playTests.sh --test-large-data &&
|
||||
fullbench.exe -i1 &&
|
||||
fullbench.exe -i1 -P0 &&
|
||||
fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
)
|
||||
|
||||
|
||||
# The following tests are for regular pushes
|
||||
# into `dev` or some feature branch
|
||||
# There run less tests, for shorter feedback loop
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
HOST: "cygwin"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
||||
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
||||
- COMPILER: "gcc"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x86"
|
||||
@@ -204,7 +189,7 @@
|
||||
- COMPILER: "clang"
|
||||
HOST: "mingw"
|
||||
PLATFORM: "x64"
|
||||
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
|
||||
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
||||
|
||||
- COMPILER: "visual"
|
||||
HOST: "visual"
|
||||
@@ -226,14 +211,6 @@
|
||||
install:
|
||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET PATH_ORIGINAL=%PATH%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
ECHO Installing Cygwin Packages &&
|
||||
C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
|
||||
gcc-g++,^
|
||||
gcc,^
|
||||
cmake,^
|
||||
make
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
|
||||
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
|
||||
@@ -246,17 +223,6 @@
|
||||
|
||||
build_script:
|
||||
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[cygwin] (
|
||||
set CHERE_INVOKING=yes &&
|
||||
set CC=%COMPILER% &&
|
||||
C:\cygwin64\bin\bash --login -c "
|
||||
set -e;
|
||||
cd build/cmake;
|
||||
CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
|
||||
make -j4;
|
||||
ctest -V -L Medium;
|
||||
"
|
||||
)
|
||||
- if [%HOST%]==[mingw] (
|
||||
( if [%PLATFORM%]==[x64] (
|
||||
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
|
||||
@@ -281,12 +247,3 @@
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
|
||||
)
|
||||
|
||||
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[mingw] (
|
||||
set "CC=%COMPILER%" &&
|
||||
make clean &&
|
||||
make check
|
||||
)
|
||||
|
||||
@@ -29,5 +29,3 @@ compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
build
|
||||
lib
|
||||
!cmake/lib
|
||||
!meson/lib
|
||||
|
||||
@@ -364,18 +364,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
|
||||
>
|
||||
@@ -392,14 +380,6 @@
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
@@ -409,21 +389,13 @@
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
RelativePath="..\..\..\programs\bench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\tests\fullbench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -506,22 +478,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -121,7 +121,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -195,7 +195,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
@@ -272,7 +272,7 @@
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
|
||||
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
@@ -328,14 +328,6 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
@@ -412,30 +404,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -542,22 +514,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -333,19 +333,7 @@
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\util.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\timefn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchfn.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\benchzstd.c"
|
||||
RelativePath="..\..\..\programs\bench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
@@ -424,30 +412,10 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -480,10 +448,6 @@
|
||||
RelativePath="..\..\..\programs\zstdcli.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\zstdcli_trace.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
|
||||
>
|
||||
@@ -626,22 +590,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -396,18 +396,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.c"
|
||||
>
|
||||
@@ -432,14 +420,6 @@
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress_block.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
@@ -554,22 +534,6 @@
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_literals.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_cwksp.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress_superblock.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_fast.h"
|
||||
>
|
||||
|
||||
@@ -155,7 +155,6 @@
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\datagencli.c" />
|
||||
</ItemGroup>
|
||||
@@ -165,4 +164,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -166,16 +166,14 @@
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -186,4 +184,4 @@
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
@@ -167,9 +167,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
@@ -178,12 +175,8 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\tests\fullbench.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -196,21 +189,16 @@
|
||||
<ClInclude Include="..\..\..\lib\common\threading.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchfn.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
|
||||
@@ -90,7 +90,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -105,7 +105,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
@@ -122,7 +122,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -142,7 +142,7 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
@@ -167,9 +167,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -178,14 +175,10 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\tests\fuzzer.c" />
|
||||
</ItemGroup>
|
||||
@@ -199,17 +192,12 @@
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||
|
||||
@@ -31,9 +31,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,8 +39,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
@@ -81,17 +76,12 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ResourceCompile Include="libzstd-dll.rc" />
|
||||
@@ -216,6 +206,7 @@
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -240,6 +231,7 @@
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -31,9 +31,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,8 +39,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
@@ -81,42 +76,42 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>libzstd</RootNamespace>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
|
||||
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
@@ -136,36 +131,35 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>libzstd_static</TargetName>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<ProgramDataBaseFileName>$(OutDir)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -185,7 +179,7 @@
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
@@ -205,10 +199,11 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -226,13 +221,14 @@
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
||||
@@ -32,9 +32,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
|
||||
@@ -42,8 +39,6 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
@@ -55,15 +50,11 @@
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
|
||||
<ClCompile Include="..\..\..\programs\util.c" />
|
||||
<ClCompile Include="..\..\..\programs\timefn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchfn.c" />
|
||||
<ClCompile Include="..\..\..\programs\benchzstd.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\dibio.c" />
|
||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||
<ClCompile Include="..\..\..\programs\zstdcli.c" />
|
||||
<ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\common\pool.h" />
|
||||
@@ -79,16 +70,11 @@
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_cwksp.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_compress_superblock.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_double_fast.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
|
||||
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
|
||||
@@ -97,7 +83,7 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\programs\benchzstd.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\dibio.h" />
|
||||
<ClInclude Include="..\..\..\programs\fileio.h" />
|
||||
|
||||
+55
-148
@@ -7,197 +7,104 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
|
||||
|
||||
# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
|
||||
# Set and use the newest cmake policies that are validated to work
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
|
||||
set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
|
||||
if("${CMAKE_MAJOR_VERSION}" LESS 3)
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
|
||||
"${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
|
||||
else()
|
||||
set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
|
||||
endif()
|
||||
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
|
||||
PROJECT(zstd)
|
||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
|
||||
SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
|
||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
|
||||
# Ensure Release build even if not invoked via Makefile
|
||||
SET(CMAKE_BUILD_TYPE "Release")
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
# Parse version
|
||||
include(GetZstdLibraryVersion)
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
|
||||
|
||||
if( CMAKE_MAJOR_VERSION LESS 3 )
|
||||
## Provide cmake 3+ behavior for older versions of cmake
|
||||
project(zstd)
|
||||
set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
|
||||
set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
|
||||
set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
|
||||
set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
|
||||
enable_language(C) # Main library is in C
|
||||
enable_language(CXX) # Testing contributed code also utilizes CXX
|
||||
else()
|
||||
project(zstd
|
||||
VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
|
||||
LANGUAGES C # Main library is in C
|
||||
CXX # Testing contributed code also utilizes CXX
|
||||
)
|
||||
endif()
|
||||
message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
|
||||
set(zstd_HOMEPAGE_URL "http://www.zstd.net")
|
||||
set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
|
||||
|
||||
# Set a default build type if none was specified
|
||||
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
||||
message(STATUS "Setting build type to 'Release' as none was specified.")
|
||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
|
||||
# Set the possible values of build type for cmake-gui
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
|
||||
endif()
|
||||
|
||||
include(GNUInstallDirs)
|
||||
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||
INCLUDE(GNUInstallDirs)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add extra compilation flags
|
||||
#-----------------------------------------------------------------------------
|
||||
include(AddZstdCompilationFlags)
|
||||
INCLUDE(AddZstdCompilationFlags)
|
||||
ADD_ZSTD_COMPILATION_FLAGS()
|
||||
|
||||
# Always hide XXHash symbols
|
||||
add_definitions(-DXXH_NAMESPACE=ZSTD_)
|
||||
ADD_DEFINITIONS(-DXXH_NAMESPACE=ZSTD_)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Installation variables
|
||||
#-----------------------------------------------------------------------------
|
||||
message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
|
||||
message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
|
||||
MESSAGE(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
|
||||
MESSAGE(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Options
|
||||
#-----------------------------------------------------------------------------
|
||||
|
||||
# Legacy support
|
||||
option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=5)
|
||||
else ()
|
||||
message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
add_definitions(-DZSTD_LEGACY_SUPPORT=0)
|
||||
endif ()
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
|
||||
ELSE (ZSTD_LEGACY_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
# Multi-threading support
|
||||
option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
|
||||
OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
|
||||
else ()
|
||||
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
|
||||
endif ()
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
|
||||
ELSE (ZSTD_MULTITHREAD_SUPPORT)
|
||||
MESSAGE(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
|
||||
ENDIF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
|
||||
option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
|
||||
option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
|
||||
|
||||
# Respect the conventional CMake option for enabling tests if it was specified on the first configure
|
||||
if (BUILD_TESTING)
|
||||
set(ZSTD_BUILD_TESTS_default ON)
|
||||
else()
|
||||
set(ZSTD_BUILD_TESTS_default OFF)
|
||||
endif()
|
||||
option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
|
||||
OPTION(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
|
||||
OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
|
||||
OPTION(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
|
||||
if (MSVC)
|
||||
option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
|
||||
OPTION(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
|
||||
endif ()
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# External dependencies
|
||||
#-----------------------------------------------------------------------------
|
||||
if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if(CMAKE_USE_PTHREADS_INIT)
|
||||
set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
endif ()
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
|
||||
SET(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
FIND_PACKAGE(Threads REQUIRED)
|
||||
IF(CMAKE_USE_PTHREADS_INIT)
|
||||
SET(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
|
||||
ELSE()
|
||||
MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
ENDIF()
|
||||
ENDIF (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add source directories
|
||||
#-----------------------------------------------------------------------------
|
||||
add_subdirectory(lib)
|
||||
ADD_SUBDIRECTORY(lib)
|
||||
|
||||
option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
|
||||
IF (ZSTD_BUILD_PROGRAMS)
|
||||
IF (NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
ENDIF (NOT ZSTD_BUILD_STATIC)
|
||||
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build shared library to build zstd CLI")
|
||||
endif ()
|
||||
ADD_SUBDIRECTORY(programs)
|
||||
ENDIF (ZSTD_BUILD_PROGRAMS)
|
||||
|
||||
add_subdirectory(programs)
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_TESTS)
|
||||
IF (NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build static library to build tests")
|
||||
ENDIF (NOT ZSTD_BUILD_STATIC)
|
||||
|
||||
if (ZSTD_BUILD_TESTS)
|
||||
enable_testing()
|
||||
if (NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build static library to build tests")
|
||||
endif ()
|
||||
ADD_SUBDIRECTORY(tests)
|
||||
ENDIF (ZSTD_BUILD_TESTS)
|
||||
|
||||
add_subdirectory(tests)
|
||||
endif ()
|
||||
|
||||
if (ZSTD_BUILD_CONTRIB)
|
||||
add_subdirectory(contrib)
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_CONTRIB)
|
||||
ADD_SUBDIRECTORY(contrib)
|
||||
ENDIF (ZSTD_BUILD_CONTRIB)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Add clean-all target
|
||||
#-----------------------------------------------------------------------------
|
||||
add_custom_target(clean-all
|
||||
ADD_CUSTOM_TARGET(clean-all
|
||||
COMMAND ${CMAKE_BUILD_TOOL} clean
|
||||
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
|
||||
)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# Generate Package Config files
|
||||
#
|
||||
# This section is based on the boiler plate code from:
|
||||
# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
|
||||
#-----------------------------------------------------------------------------
|
||||
include(CMakePackageConfigHelpers)
|
||||
write_basic_package_version_file(
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
VERSION ${zstd_VERSION}
|
||||
COMPATIBILITY SameMajorVersion
|
||||
)
|
||||
|
||||
# A Package Config file that works from the build directory
|
||||
export(EXPORT zstdExports
|
||||
FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
|
||||
NAMESPACE zstd::
|
||||
)
|
||||
configure_file(zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
|
||||
COPYONLY
|
||||
)
|
||||
|
||||
# A Package Config file that works from the installation directory
|
||||
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
|
||||
install(EXPORT zstdExports
|
||||
FILE zstdTargets.cmake
|
||||
NAMESPACE zstd::
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
install(FILES
|
||||
zstdConfig.cmake
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
|
||||
DESTINATION ${ConfigPackageLocation}
|
||||
)
|
||||
|
||||
@@ -20,7 +20,7 @@ function(EnableCompilerFlag _flag _C _CXX)
|
||||
endif ()
|
||||
endfunction()
|
||||
|
||||
macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
MACRO(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
|
||||
#Set c++11 by default
|
||||
EnableCompilerFlag("-std=c++11" false true)
|
||||
@@ -33,10 +33,6 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
EnableCompilerFlag("-Wcast-align" true true)
|
||||
EnableCompilerFlag("-Wcast-qual" true true)
|
||||
EnableCompilerFlag("-Wstrict-prototypes" true false)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("-DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
elseif (MSVC) # Add specific compilation flags for Windows Visual
|
||||
|
||||
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
|
||||
@@ -47,32 +43,25 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
# UNICODE SUPPORT
|
||||
EnableCompilerFlag("/D_UNICODE" true true)
|
||||
EnableCompilerFlag("/DUNICODE" true true)
|
||||
# Enable asserts in Debug mode
|
||||
if (CMAKE_BUILD_TYPE MATCHES "Debug")
|
||||
EnableCompilerFlag("/DDEBUGLEVEL=1" true true)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Remove duplicates compilation flags
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if( ${flag_var} )
|
||||
separate_arguments(${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
separate_arguments(${flag_var})
|
||||
list(REMOVE_DUPLICATES ${flag_var})
|
||||
string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
|
||||
ENDFOREACH (flag_var)
|
||||
|
||||
if (MSVC AND ZSTD_USE_STATIC_RUNTIME)
|
||||
foreach (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
FOREACH (flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
|
||||
CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if ( ${flag_var} )
|
||||
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach ()
|
||||
STRING(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
ENDFOREACH (flag_var)
|
||||
endif ()
|
||||
|
||||
endmacro()
|
||||
ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
# Find LibLZ4
|
||||
#
|
||||
# Find LibLZ4 headers and library
|
||||
#
|
||||
# Result Variables
|
||||
#
|
||||
# LIBLZ4_FOUND - True if lz4 is found
|
||||
# LIBLZ4_INCLUDE_DIRS - lz4 headers directories
|
||||
# LIBLZ4_LIBRARIES - lz4 libraries
|
||||
# LIBLZ4_VERSION_MAJOR - The major version of lz4
|
||||
# LIBLZ4_VERSION_MINOR - The minor version of lz4
|
||||
# LIBLZ4_VERSION_RELEASE - The release version of lz4
|
||||
# LIBLZ4_VERSION_STRING - version number string (e.g. 1.8.3)
|
||||
#
|
||||
# Hints
|
||||
#
|
||||
# Set ``LZ4_ROOT_DIR`` to the directory of lz4.h and lz4 library
|
||||
|
||||
set(_LIBLZ4_ROOT_HINTS
|
||||
ENV LZ4_ROOT_DIR)
|
||||
|
||||
find_path( LIBLZ4_INCLUDE_DIR lz4.h
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
find_library( LIBLZ4_LIBRARY NAMES lz4 liblz4 liblz4_static
|
||||
HINTS ${_LIBLZ4_ROOT_HINTS})
|
||||
|
||||
if(LIBLZ4_INCLUDE_DIR)
|
||||
file(STRINGS "${LIBLZ4_INCLUDE_DIR}/lz4.h" LIBLZ4_HEADER_CONTENT REGEX "#define LZ4_VERSION_[A-Z]+ +[0-9]+")
|
||||
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MAJOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MAJOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_MINOR +([0-9]+).*" "\\1" LIBLZ4_VERSION_MINOR "${LIBLZ4_HEADER_CONTENT}")
|
||||
string(REGEX REPLACE ".*#define LZ4_VERSION_RELEASE +([0-9]+).*" "\\1" LIBLZ4_VERSION_RELEASE "${LIBLZ4_HEADER_CONTENT}")
|
||||
|
||||
set(LIBLZ4_VERSION_STRING "${LIBLZ4_VERSION_MAJOR}.${LIBLZ4_VERSION_MINOR}.${LIBLZ4_VERSION_RELEASE}")
|
||||
unset(LIBLZ4_HEADER_CONTENT)
|
||||
endif()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZ4 REQUIRED_VARS LIBLZ4_INCLUDE_DIR
|
||||
LIBLZ4_LIBRARY
|
||||
VERSION_VAR LIBLZ4_VERSION_STRING
|
||||
FAIL_MESSAGE "Could NOT find LZ4, try to set the paths to lz4.h and lz4 library in environment variable LZ4_ROOT_DIR")
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
set(LIBLZ4_LIBRARIES ${LIBLZ4_LIBRARY})
|
||||
set(LIBLZ4_INCLUDE_DIRS ${LIBLZ4_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
mark_as_advanced( LIBLZ4_INCLUDE_DIR LIBLZ4_LIBRARY )
|
||||
@@ -1,10 +1,9 @@
|
||||
function(GetZstdLibraryVersion _header _major _minor _patch)
|
||||
function(GetZstdLibraryVersion _header _major _minor _release)
|
||||
# Read file content
|
||||
file(READ ${_header} CONTENT)
|
||||
FILE(READ ${_header} CONTENT)
|
||||
|
||||
string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}")
|
||||
set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
SET(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
SET(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE)
|
||||
SET(${_release} ${CMAKE_MATCH_3} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# Cmake contributions
|
||||
|
||||
Contributions to the cmake build configurations are welcome. Please
|
||||
use case sensitivity that matches modern (ie. cmake version 2.6 and above)
|
||||
conventions of using lower-case for commands, and upper-case for
|
||||
variables.
|
||||
|
||||
## How to build
|
||||
|
||||
As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
|
||||
To do this, you can create a new directory and build in it:
|
||||
```sh
|
||||
cd build/cmake
|
||||
mkdir builddir
|
||||
cd builddir
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
Then you can clean all cmake caches by simply delete the new directory:
|
||||
```sh
|
||||
rm -rf build/cmake/builddir
|
||||
```
|
||||
|
||||
And of course, you can directly build in build/cmake:
|
||||
```sh
|
||||
cd build/cmake
|
||||
cmake
|
||||
make
|
||||
```
|
||||
|
||||
To show cmake build options, you can:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -LH ..
|
||||
```
|
||||
|
||||
Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
|
||||
make
|
||||
```
|
||||
|
||||
### referring
|
||||
[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
|
||||
|
||||
## CMake Style Recommendations
|
||||
|
||||
### Indent all code correctly, i.e. the body of
|
||||
|
||||
* if/else/endif
|
||||
* foreach/endforeach
|
||||
* while/endwhile
|
||||
* macro/endmacro
|
||||
* function/endfunction
|
||||
|
||||
Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
|
||||
spaces for indenting as is used in the rest of the file. Do not use tabs.
|
||||
|
||||
### Upper/lower casing
|
||||
|
||||
Most important: use consistent upper- or lowercasing within one file !
|
||||
|
||||
In general, the all-lowercase style is preferred.
|
||||
|
||||
So, this is recommended:
|
||||
|
||||
```
|
||||
add_executable(foo foo.c)
|
||||
```
|
||||
|
||||
These forms are discouraged
|
||||
|
||||
```
|
||||
ADD_EXECUTABLE(bar bar.c)
|
||||
Add_Executable(hello hello.c)
|
||||
aDd_ExEcUtAbLe(blub blub.c)
|
||||
```
|
||||
|
||||
### End commands
|
||||
To make the code easier to read, use empty commands for endforeach(), endif(),
|
||||
endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
|
||||
|
||||
For example, do this:
|
||||
|
||||
```
|
||||
if(FOOVAR)
|
||||
some_command(...)
|
||||
else()
|
||||
another_command(...)
|
||||
endif()
|
||||
```
|
||||
|
||||
and not this:
|
||||
|
||||
```
|
||||
if(BARVAR)
|
||||
some_other_command(...)
|
||||
endif(BARVAR)
|
||||
```
|
||||
|
||||
### Other resources for best practices
|
||||
|
||||
https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules
|
||||
@@ -7,7 +7,7 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(contrib)
|
||||
PROJECT(contrib)
|
||||
|
||||
add_subdirectory(pzstd)
|
||||
add_subdirectory(gen_html)
|
||||
ADD_SUBDIRECTORY(pzstd)
|
||||
ADD_SUBDIRECTORY(gen_html)
|
||||
|
||||
@@ -7,24 +7,24 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(gen_html)
|
||||
include(GetZstdLibraryVersion)
|
||||
PROJECT(gen_html)
|
||||
INCLUDE(GetZstdLibraryVersion)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
|
||||
set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html)
|
||||
SET(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX})
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR})
|
||||
|
||||
add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp)
|
||||
ADD_EXECUTABLE(gen_html ${GENHTML_DIR}/gen_html.cpp)
|
||||
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
|
||||
set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
|
||||
add_custom_target(zstd_manual.html ALL
|
||||
SET(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
|
||||
ADD_CUSTOM_TARGET(zstd_manual.html ALL
|
||||
${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
|
||||
DEPENDS gen_html COMMENT "Update zstd manual")
|
||||
|
||||
install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
|
||||
INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
|
||||
|
||||
@@ -7,26 +7,26 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(pzstd)
|
||||
PROJECT(pzstd)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
|
||||
|
||||
add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
|
||||
set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
|
||||
ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
|
||||
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
|
||||
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
if (CMAKE_USE_PTHREADS_INIT)
|
||||
target_link_libraries(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
|
||||
else()
|
||||
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
endif()
|
||||
SET(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
FIND_PACKAGE(Threads REQUIRED)
|
||||
IF (CMAKE_USE_PTHREADS_INIT)
|
||||
TARGET_LINK_LIBRARIES(pzstd libzstd_shared ${CMAKE_THREAD_LIBS_INIT})
|
||||
ELSE()
|
||||
MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
|
||||
ENDIF()
|
||||
|
||||
install(TARGETS pzstd RUNTIME DESTINATION "bin")
|
||||
INSTALL(TARGETS pzstd RUNTIME DESTINATION "bin")
|
||||
|
||||
+143
-126
@@ -7,51 +7,82 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(libzstd C)
|
||||
PROJECT(libzstd)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
||||
option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
||||
OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||
|
||||
if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||
message(SEND_ERROR "You need to build at least one flavor of libzstd")
|
||||
endif()
|
||||
IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||
MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd")
|
||||
ENDIF()
|
||||
|
||||
# Define library directory, where sources and header files are located
|
||||
include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
|
||||
|
||||
file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
|
||||
file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
|
||||
file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
|
||||
file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
|
||||
file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
|
||||
# Parse version
|
||||
INCLUDE(GetZstdLibraryVersion)
|
||||
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
|
||||
MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
|
||||
set(Sources
|
||||
${CommonSources}
|
||||
${CompressSources}
|
||||
${DecompressSources}
|
||||
${DictBuilderSources}
|
||||
${DeprecatedSources})
|
||||
SET(Sources
|
||||
${LIBRARY_DIR}/common/entropy_common.c
|
||||
${LIBRARY_DIR}/common/fse_decompress.c
|
||||
${LIBRARY_DIR}/common/threading.c
|
||||
${LIBRARY_DIR}/common/pool.c
|
||||
${LIBRARY_DIR}/common/zstd_common.c
|
||||
${LIBRARY_DIR}/common/error_private.c
|
||||
${LIBRARY_DIR}/common/xxhash.c
|
||||
${LIBRARY_DIR}/compress/hist.c
|
||||
${LIBRARY_DIR}/compress/fse_compress.c
|
||||
${LIBRARY_DIR}/compress/huf_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_compress.c
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_fast.c
|
||||
${LIBRARY_DIR}/compress/zstd_double_fast.c
|
||||
${LIBRARY_DIR}/compress/zstd_lazy.c
|
||||
${LIBRARY_DIR}/compress/zstd_opt.c
|
||||
${LIBRARY_DIR}/compress/zstd_ldm.c
|
||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/cover.c
|
||||
${LIBRARY_DIR}/dictBuilder/fastcover.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_compress.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
|
||||
|
||||
file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
|
||||
file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
|
||||
file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
|
||||
file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
|
||||
file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
|
||||
|
||||
set(Headers
|
||||
SET(Headers
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${CommonHeaders}
|
||||
${CompressHeaders}
|
||||
${DecompressHeaders}
|
||||
${DictBuilderHeaders}
|
||||
${DeprecatedHeaders})
|
||||
${LIBRARY_DIR}/common/debug.h
|
||||
${LIBRARY_DIR}/common/pool.h
|
||||
${LIBRARY_DIR}/common/threading.h
|
||||
${LIBRARY_DIR}/common/bitstream.h
|
||||
${LIBRARY_DIR}/common/error_private.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
${LIBRARY_DIR}/common/fse.h
|
||||
${LIBRARY_DIR}/common/huf.h
|
||||
${LIBRARY_DIR}/common/mem.h
|
||||
${LIBRARY_DIR}/common/zstd_internal.h
|
||||
${LIBRARY_DIR}/compress/hist.h
|
||||
${LIBRARY_DIR}/compress/zstd_compress_internal.h
|
||||
${LIBRARY_DIR}/compress/zstd_fast.h
|
||||
${LIBRARY_DIR}/compress/zstd_double_fast.h
|
||||
${LIBRARY_DIR}/compress/zstd_lazy.h
|
||||
${LIBRARY_DIR}/compress/zstd_opt.h
|
||||
${LIBRARY_DIR}/compress/zstd_ldm.h
|
||||
${LIBRARY_DIR}/compress/zstdmt_compress.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/dictBuilder/cover.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
include_directories(${LIBRARY_LEGACY_DIR})
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
|
||||
|
||||
set(Sources ${Sources}
|
||||
SET(Sources ${Sources}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
||||
@@ -60,7 +91,7 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.c)
|
||||
|
||||
set(Headers ${Headers}
|
||||
SET(Headers ${Headers}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v01.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v02.h
|
||||
@@ -69,118 +100,104 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v05.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.h)
|
||||
endif ()
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Split project to static and shared libraries build
|
||||
set(library_targets)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
list(APPEND library_targets libzstd_shared)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_shared ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif()
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
add_library(libzstd_static STATIC ${Sources} ${Headers})
|
||||
list(APPEND library_targets libzstd_static)
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
if (UNIX)
|
||||
target_link_libraries(libzstd_static ${THREADS_LIBS})
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
IF (UNIX)
|
||||
TARGET_LINK_LIBRARIES(libzstd_shared ${THREADS_LIBS})
|
||||
ENDIF ()
|
||||
ENDIF()
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
IF (UNIX)
|
||||
TARGET_LINK_LIBRARIES(libzstd_static ${THREADS_LIBS})
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
# Add specific compile definitions for MSVC project
|
||||
if (MSVC)
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# With MSVC static library needs to be renamed to avoid conflict with import library
|
||||
if (MSVC)
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd_static)
|
||||
else ()
|
||||
set(STATIC_LIBRARY_BASE_NAME zstd)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(STATIC_LIBRARY_BASE_NAME zstd_static)
|
||||
ELSE ()
|
||||
SET(STATIC_LIBRARY_BASE_NAME zstd)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Define static and shared library names
|
||||
if (ZSTD_BUILD_SHARED)
|
||||
set_target_properties(
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_shared
|
||||
PROPERTIES
|
||||
OUTPUT_NAME zstd
|
||||
VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
|
||||
SOVERSION ${zstd_VERSION_MAJOR})
|
||||
endif ()
|
||||
SOVERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE})
|
||||
ENDIF (ZSTD_BUILD_SHARED)
|
||||
|
||||
if (ZSTD_BUILD_STATIC)
|
||||
set_target_properties(
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
SET_TARGET_PROPERTIES(
|
||||
libzstd_static
|
||||
PROPERTIES
|
||||
POSITION_INDEPENDENT_CODE On
|
||||
OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
|
||||
endif ()
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
if (UNIX OR MINGW)
|
||||
IF (UNIX)
|
||||
# pkg-config
|
||||
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
set(EXEC_PREFIX "\${prefix}")
|
||||
set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
|
||||
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
|
||||
set(VERSION "${zstd_VERSION}")
|
||||
SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
|
||||
SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
|
||||
SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
ADD_CUSTOM_TARGET(libzstd.pc ALL
|
||||
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
|
||||
-DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
|
||||
COMMENT "Creating pkg-config file")
|
||||
|
||||
string(LENGTH "${PREFIX}" PREFIX_LENGTH)
|
||||
string(SUBSTRING "${LIBDIR}" 0 ${PREFIX_LENGTH} LIBDIR_PREFIX)
|
||||
string(SUBSTRING "${LIBDIR}" ${PREFIX_LENGTH} -1 LIBDIR_SUFFIX)
|
||||
string(SUBSTRING "${INCLUDEDIR}" 0 ${PREFIX_LENGTH} INCLUDEDIR_PREFIX)
|
||||
string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
|
||||
|
||||
if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
|
||||
set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
|
||||
endif()
|
||||
if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
|
||||
set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
|
||||
endif()
|
||||
|
||||
configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
|
||||
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
|
||||
endif ()
|
||||
INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${LIBDIR}/pkgconfig")
|
||||
ENDIF (UNIX)
|
||||
|
||||
# install target
|
||||
install(FILES
|
||||
"${LIBRARY_DIR}/zstd.h"
|
||||
"${LIBRARY_DIR}/dictBuilder/zdict.h"
|
||||
"${LIBRARY_DIR}/common/zstd_errors.h"
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
INSTALL(FILES
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/dictBuilder/cover.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
DESTINATION "include")
|
||||
|
||||
install(TARGETS ${library_targets}
|
||||
EXPORT zstdExports
|
||||
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
||||
)
|
||||
IF (ZSTD_BUILD_SHARED)
|
||||
INSTALL(TARGETS libzstd_shared RUNTIME DESTINATION "bin"
|
||||
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
|
||||
ENDIF()
|
||||
IF (ZSTD_BUILD_STATIC)
|
||||
INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
|
||||
ENDIF (ZSTD_BUILD_STATIC)
|
||||
|
||||
# uninstall target
|
||||
if (NOT TARGET uninstall)
|
||||
configure_file(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
IMMEDIATE @ONLY)
|
||||
CONFIGURE_FILE(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
|
||||
IMMEDIATE @ONLY)
|
||||
|
||||
add_custom_target(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
endif ()
|
||||
ADD_CUSTOM_TARGET(uninstall
|
||||
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
endif()
|
||||
endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
|
||||
|
||||
file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
|
||||
string(REGEX REPLACE "\n" ";" files "${files}")
|
||||
@@ -15,8 +15,8 @@ foreach(file ${files})
|
||||
)
|
||||
if(NOT "${rm_retval}" STREQUAL 0)
|
||||
message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
|
||||
endif()
|
||||
else()
|
||||
endif(NOT "${rm_retval}" STREQUAL 0)
|
||||
else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
|
||||
endif()
|
||||
endforeach()
|
||||
endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
|
||||
endforeach(file)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)
|
||||
@@ -7,129 +7,104 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
project(programs C)
|
||||
PROJECT(programs)
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
if (ZSTD_LEGACY_SUPPORT)
|
||||
set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
endif ()
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
if (ZSTD_PROGRAMS_LINK_SHARED)
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
|
||||
else ()
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
|
||||
endif ()
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
endif ()
|
||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
||||
IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
TARGET_LINK_LIBRARIES(zstd rt)
|
||||
ENDIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
|
||||
|
||||
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
|
||||
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
target_link_libraries(zstd rt)
|
||||
endif ()
|
||||
install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
IF (UNIX)
|
||||
ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
|
||||
ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
|
||||
INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
|
||||
|
||||
if (UNIX)
|
||||
add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
|
||||
add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
|
||||
add_custom_target(zstd.1 ALL
|
||||
ADD_CUSTOM_TARGET(zstd.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
|
||||
COMMENT "Copying manpage zstd.1")
|
||||
add_custom_target(zstdgrep.1 ALL
|
||||
ADD_CUSTOM_TARGET(zstdgrep.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
|
||||
COMMENT "Copying manpage zstdgrep.1")
|
||||
add_custom_target(zstdless.1 ALL
|
||||
ADD_CUSTOM_TARGET(zstdless.1 ALL
|
||||
${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
|
||||
COMMENT "Copying manpage zstdless.1")
|
||||
add_custom_target(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
|
||||
add_custom_target(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
|
||||
ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
|
||||
ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
|
||||
|
||||
# Define MAN_INSTALL_DIR if necessary
|
||||
if (MAN_INSTALL_DIR)
|
||||
else ()
|
||||
set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1)
|
||||
endif ()
|
||||
IF (MAN_INSTALL_DIR)
|
||||
ELSE (MAN_INSTALL_DIR)
|
||||
SET(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1)
|
||||
ENDIF (MAN_INSTALL_DIR)
|
||||
|
||||
install(FILES
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/unzstd.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1
|
||||
${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
|
||||
DESTINATION "${MAN_INSTALL_DIR}")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
|
||||
|
||||
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
|
||||
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
|
||||
endif ()
|
||||
ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
|
||||
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
||||
SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
ENDIF (UNIX)
|
||||
|
||||
# Add multi-threading support definitions
|
||||
|
||||
if (ZSTD_MULTITHREAD_SUPPORT)
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
IF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
|
||||
|
||||
if (UNIX)
|
||||
target_link_libraries(zstd ${THREADS_LIBS})
|
||||
IF (UNIX)
|
||||
TARGET_LINK_LIBRARIES(zstd ${THREADS_LIBS})
|
||||
|
||||
add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
||||
endif ()
|
||||
endif ()
|
||||
ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
|
||||
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
|
||||
ENDIF (UNIX)
|
||||
ENDIF (ZSTD_MULTITHREAD_SUPPORT)
|
||||
|
||||
option(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
|
||||
option(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
|
||||
option(ZSTD_LZ4_SUPPORT "LZ4 SUPPORT" OFF)
|
||||
OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
|
||||
OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
|
||||
|
||||
# Add gzip support
|
||||
if (ZSTD_ZLIB_SUPPORT)
|
||||
find_package(ZLIB REQUIRED)
|
||||
IF (ZSTD_ZLIB_SUPPORT)
|
||||
FIND_PACKAGE(ZLIB REQUIRED)
|
||||
|
||||
if (ZLIB_FOUND)
|
||||
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${ZLIB_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "zlib library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (ZLIB_FOUND)
|
||||
INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
|
||||
TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
|
||||
ELSE ()
|
||||
MESSAGE(SEND_ERROR "zlib library is missing")
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
# Add lzma support
|
||||
if (ZSTD_LZMA_SUPPORT)
|
||||
find_package(LibLZMA REQUIRED)
|
||||
IF (ZSTD_LZMA_SUPPORT)
|
||||
FIND_PACKAGE(LibLZMA REQUIRED)
|
||||
|
||||
if (LIBLZMA_FOUND)
|
||||
include_directories(${LIBLZMA_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZMA_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lzma library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Add lz4 support
|
||||
if (ZSTD_LZ4_SUPPORT)
|
||||
find_package(LibLZ4 REQUIRED)
|
||||
|
||||
if (LIBLZ4_FOUND)
|
||||
include_directories(${LIBLZ4_INCLUDE_DIRS})
|
||||
target_link_libraries(zstd ${LIBLZ4_LIBRARIES})
|
||||
set_property(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZ4COMPRESS;ZSTD_LZ4DECOMPRESS")
|
||||
else ()
|
||||
message(SEND_ERROR "lz4 library is missing")
|
||||
endif ()
|
||||
endif ()
|
||||
IF (LIBLZMA_FOUND)
|
||||
INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
|
||||
TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
|
||||
SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
|
||||
ELSE ()
|
||||
MESSAGE(SEND_ERROR "lzma library is missing")
|
||||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
@@ -30,79 +30,26 @@
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
project(tests)
|
||||
PROJECT(tests)
|
||||
|
||||
# name: Cache variable name. The value is expected to be a semicolon-separated
|
||||
# list of command line flags
|
||||
# default_value: Value to initialize the option with. Can be space separated.
|
||||
function(AddTestFlagsOption name default_value doc)
|
||||
string(STRIP "${default_value}" default_value)
|
||||
string(REGEX REPLACE " +" ";" default_value "${default_value}")
|
||||
set(${name} ${default_value} CACHE STRING "${doc}")
|
||||
mark_as_advanced(${name})
|
||||
endfunction()
|
||||
|
||||
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
|
||||
include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
|
||||
SET(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
|
||||
INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
|
||||
|
||||
add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
target_link_libraries(datagen libzstd_static)
|
||||
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
|
||||
TARGET_LINK_LIBRARIES(datagen libzstd_static)
|
||||
|
||||
#
|
||||
# fullbench
|
||||
#
|
||||
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
|
||||
target_link_libraries(fullbench libzstd_static)
|
||||
add_test(NAME fullbench COMMAND fullbench)
|
||||
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
|
||||
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
||||
|
||||
#
|
||||
# fuzzer
|
||||
#
|
||||
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
|
||||
target_link_libraries(fuzzer libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
|
||||
add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
|
||||
# Disable the timeout since the run time is too long for the default timeout of
|
||||
# 1500 seconds and varies considerably between low-end and high-end CPUs.
|
||||
# set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
|
||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
|
||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||
|
||||
#
|
||||
# zstreamtest
|
||||
#
|
||||
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
|
||||
target_link_libraries(zstreamtest libzstd_static)
|
||||
AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
|
||||
add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
|
||||
|
||||
#
|
||||
# playTests.sh
|
||||
#
|
||||
AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "$ENV{PLAYTESTS_FLAGS}"
|
||||
"Semicolon-separated list of flags to pass to the playTests.sh test")
|
||||
add_test(NAME playTests COMMAND sh -c "\"${TESTS_DIR}/playTests.sh\" ${ZSTD_PLAYTESTS_FLAGS}")
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
|
||||
"ZSTD_BIN=$<TARGET_FILE:zstd>"
|
||||
"DATAGEN_BIN=$<TARGET_FILE:datagen>"
|
||||
)
|
||||
else()
|
||||
message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
|
||||
set_tests_properties(playTests PROPERTIES DISABLED YES)
|
||||
endif()
|
||||
|
||||
# Label the "Medium" set of tests (see TESTING.md)
|
||||
set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
|
||||
|
||||
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
|
||||
if (UNIX)
|
||||
target_link_libraries(paramgrill libzstd_static m) #m is math library
|
||||
else()
|
||||
target_link_libraries(paramgrill libzstd_static)
|
||||
endif ()
|
||||
IF (UNIX)
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
|
||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||
ENDIF (UNIX)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import re
|
||||
|
||||
|
||||
def find_version_tuple(filepath):
|
||||
version_file_data = None
|
||||
with open(filepath) as fd:
|
||||
version_file_data = fd.read()
|
||||
|
||||
patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
|
||||
#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
|
||||
"""
|
||||
regex = re.compile(patterns, re.MULTILINE)
|
||||
version_match = regex.search(version_file_data)
|
||||
if version_match:
|
||||
return version_match.groups()
|
||||
raise Exception("Unable to find version string")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
|
||||
parser.add_argument('file', help='path to lib/zstd.h')
|
||||
args = parser.parse_args()
|
||||
version_tuple = find_version_tuple(args.file)
|
||||
print('.'.join(version_tuple))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
# This file should be synced with https://github.com/lzutao/meson-symlink
|
||||
|
||||
import os
|
||||
import pathlib # since Python 3.4
|
||||
|
||||
|
||||
def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
|
||||
if not install_dir.exists():
|
||||
install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
|
||||
if not install_dir.is_dir():
|
||||
raise NotADirectoryError(install_dir)
|
||||
|
||||
new_dst = install_dir.joinpath(dst)
|
||||
if new_dst.is_symlink() and os.readlink(new_dst) == src:
|
||||
print('File exists: {!r} -> {!r}'.format(new_dst, src))
|
||||
return
|
||||
print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
|
||||
new_dst.symlink_to(src, target_is_directory=dst_is_dir)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Install a symlink',
|
||||
usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
|
||||
'example:\n'
|
||||
' {0} dash sh /bin'.format(pathlib.Path(__file__).name))
|
||||
parser.add_argument('source', help='target to link')
|
||||
parser.add_argument('dest', help='link name')
|
||||
parser.add_argument('install_dir', help='installation directory')
|
||||
parser.add_argument('-d', '--isdir',
|
||||
action='store_true',
|
||||
help='dest is a directory')
|
||||
parser.add_argument('-m', '--mode',
|
||||
help='directory mode on creating if not exist',
|
||||
default='0o755')
|
||||
args = parser.parse_args()
|
||||
|
||||
dir_mode = int(args.mode, 8)
|
||||
|
||||
meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
|
||||
install_dir = pathlib.Path(meson_destdir, args.install_dir)
|
||||
install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
Meson build system for zstandard
|
||||
================================
|
||||
|
||||
Meson is a build system designed to optimize programmer productivity.
|
||||
It aims to do this by providing simple, out-of-the-box support for
|
||||
modern software development tools and practices, such as unit tests,
|
||||
coverage reports, Valgrind, CCache and the like.
|
||||
|
||||
This Meson build system is provided with no guarantee and maintained
|
||||
by Dima Krasner \<dima@dimakrasner.com\>.
|
||||
|
||||
It outputs one `libzstd`, either shared or static, depending on
|
||||
`default_library` option.
|
||||
|
||||
## How to build
|
||||
|
||||
`cd` to this meson directory (`build/meson`)
|
||||
|
||||
```sh
|
||||
meson setup -Dbin_programs=true -Dbin_contrib=true builddir
|
||||
cd builddir
|
||||
ninja # to build
|
||||
ninja install # to install
|
||||
```
|
||||
|
||||
You might want to install it in staging directory:
|
||||
|
||||
```sh
|
||||
DESTDIR=./staging ninja install
|
||||
```
|
||||
|
||||
To configure build options, use:
|
||||
|
||||
```sh
|
||||
meson configure
|
||||
```
|
||||
|
||||
See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).
|
||||
@@ -1,30 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html'))
|
||||
|
||||
gen_html = executable('gen_html',
|
||||
join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
|
||||
include_directories: gen_html_includes,
|
||||
native: true,
|
||||
install: false)
|
||||
|
||||
# Update zstd manual
|
||||
zstd_manual_html = custom_target('zstd_manual.html',
|
||||
output : 'zstd_manual.html',
|
||||
command : [gen_html,
|
||||
zstd_version,
|
||||
join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
|
||||
'@OUTPUT@'],
|
||||
install : false)
|
||||
@@ -1,12 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
subdir('pzstd')
|
||||
subdir('gen_html')
|
||||
@@ -1,24 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../../..'
|
||||
|
||||
pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd'))
|
||||
pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
|
||||
join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
|
||||
pzstd = executable('pzstd',
|
||||
pzstd_sources,
|
||||
cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
|
||||
include_directories: pzstd_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: true)
|
||||
@@ -1,132 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
|
||||
join_paths(zstd_rootdir, 'lib/common'),
|
||||
join_paths(zstd_rootdir, 'lib/compress'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated'))]
|
||||
|
||||
libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_trace.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/hist.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
|
||||
join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
|
||||
join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
|
||||
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
|
||||
|
||||
# Explicit define legacy support
|
||||
add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
|
||||
language: 'c')
|
||||
|
||||
if legacy_level == 0
|
||||
message('Legacy support: DISABLED')
|
||||
else
|
||||
# See ZSTD_LEGACY_SUPPORT of lib/README.md
|
||||
message('Enable legacy support back to version 0.@0@'.format(legacy_level))
|
||||
|
||||
libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
|
||||
foreach i : [1, 2, 3, 4, 5, 6, 7]
|
||||
if legacy_level <= i
|
||||
libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
|
||||
endif
|
||||
endforeach
|
||||
endif
|
||||
|
||||
libzstd_deps = []
|
||||
if use_multi_thread
|
||||
message('Enable multi-threading support')
|
||||
add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
|
||||
libzstd_deps = [ thread_dep ]
|
||||
endif
|
||||
|
||||
libzstd_c_args = []
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
libzstd_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
|
||||
libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
|
||||
'-DZSTD_HEAPMODE=0',
|
||||
'-D_CONSOLE',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
else
|
||||
libzstd_c_args += ['-DZSTD_HEAPMODE=0',
|
||||
'-D_CRT_SECURE_NO_WARNINGS']
|
||||
endif
|
||||
endif
|
||||
|
||||
mingw_ansi_stdio_flags = []
|
||||
if host_machine_os == os_windows and cc_id == compiler_gcc
|
||||
mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
|
||||
endif
|
||||
libzstd_c_args += mingw_ansi_stdio_flags
|
||||
|
||||
libzstd_debug_cflags = []
|
||||
if use_debug
|
||||
libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
|
||||
if cc_id == compiler_gcc or cc_id == compiler_clang
|
||||
libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
|
||||
'-Wdeclaration-after-statement', '-Wstrict-prototypes',
|
||||
'-Wundef', '-Wpointer-arith', '-Wvla',
|
||||
'-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
|
||||
'-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
|
||||
endif
|
||||
endif
|
||||
libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
|
||||
|
||||
libzstd = library('zstd',
|
||||
libzstd_sources,
|
||||
include_directories: libzstd_includes,
|
||||
c_args: libzstd_c_args,
|
||||
dependencies: libzstd_deps,
|
||||
install: true,
|
||||
version: zstd_libversion)
|
||||
|
||||
libzstd_dep = declare_dependency(link_with: libzstd,
|
||||
include_directories: libzstd_includes)
|
||||
|
||||
pkgconfig.generate(libzstd,
|
||||
name: 'libzstd',
|
||||
filebase: 'libzstd',
|
||||
description: 'fast lossless compression algorithm library',
|
||||
version: zstd_libversion,
|
||||
url: 'http://www.zstd.net/')
|
||||
|
||||
install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
|
||||
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))
|
||||
@@ -1,146 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
project('zstd',
|
||||
['c', 'cpp'],
|
||||
license: ['BSD', 'GPLv2'],
|
||||
default_options : [
|
||||
'c_std=gnu99',
|
||||
'cpp_std=c++11',
|
||||
'buildtype=release'
|
||||
],
|
||||
version: 'DUMMY',
|
||||
meson_version: '>=0.47.0')
|
||||
|
||||
cc = meson.get_compiler('c')
|
||||
cxx = meson.get_compiler('cpp')
|
||||
pkgconfig = import('pkgconfig')
|
||||
windows_mod = import('windows')
|
||||
|
||||
host_machine_os = host_machine.system()
|
||||
os_windows = 'windows'
|
||||
os_linux = 'linux'
|
||||
os_darwin = 'darwin'
|
||||
os_freebsd = 'freebsd'
|
||||
os_sun = 'sunos'
|
||||
|
||||
cc_id = cc.get_id()
|
||||
compiler_gcc = 'gcc'
|
||||
compiler_clang = 'clang'
|
||||
compiler_msvc = 'msvc'
|
||||
|
||||
zstd_version = meson.project_version()
|
||||
|
||||
zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
|
||||
GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
|
||||
r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
|
||||
if r.returncode() == 0
|
||||
zstd_version = r.stdout().strip()
|
||||
message('Project version is now: @0@'.format(zstd_version))
|
||||
else
|
||||
error('Cannot find project version in @0@'.format(zstd_h_file))
|
||||
endif
|
||||
|
||||
zstd_libversion = zstd_version
|
||||
|
||||
# =============================================================================
|
||||
# Installation directories
|
||||
# =============================================================================
|
||||
|
||||
zstd_prefix = get_option('prefix')
|
||||
zstd_bindir = get_option('bindir')
|
||||
zstd_datadir = get_option('datadir')
|
||||
zstd_mandir = get_option('mandir')
|
||||
zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
|
||||
|
||||
# =============================================================================
|
||||
# Project options
|
||||
# =============================================================================
|
||||
|
||||
# Built-in options
|
||||
use_debug = get_option('debug')
|
||||
buildtype = get_option('buildtype')
|
||||
default_library_type = get_option('default_library')
|
||||
|
||||
# Custom options
|
||||
debug_level = get_option('debug_level')
|
||||
legacy_level = get_option('legacy_level')
|
||||
use_backtrace = get_option('backtrace')
|
||||
use_static_runtime = get_option('static_runtime')
|
||||
|
||||
bin_programs = get_option('bin_programs')
|
||||
bin_contrib = get_option('bin_contrib')
|
||||
bin_tests = get_option('bin_tests')
|
||||
|
||||
feature_multi_thread = get_option('multi_thread')
|
||||
feature_zlib = get_option('zlib')
|
||||
feature_lzma = get_option('lzma')
|
||||
feature_lz4 = get_option('lz4')
|
||||
|
||||
# =============================================================================
|
||||
# Dependencies
|
||||
# =============================================================================
|
||||
|
||||
libm_dep = cc.find_library('m', required: bin_tests)
|
||||
thread_dep = dependency('threads', required: feature_multi_thread)
|
||||
use_multi_thread = thread_dep.found()
|
||||
# Arguments in dependency should be equivalent to those passed to pkg-config
|
||||
zlib_dep = dependency('zlib', required: feature_zlib)
|
||||
use_zlib = zlib_dep.found()
|
||||
lzma_dep = dependency('liblzma', required: feature_lzma)
|
||||
use_lzma = lzma_dep.found()
|
||||
lz4_dep = dependency('liblz4', required: feature_lz4)
|
||||
use_lz4 = lz4_dep.found()
|
||||
|
||||
# =============================================================================
|
||||
# Compiler flags
|
||||
# =============================================================================
|
||||
|
||||
add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
|
||||
|
||||
if [compiler_gcc, compiler_clang].contains(cc_id)
|
||||
common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
|
||||
if cc_id == compiler_clang
|
||||
# Should use Meson's own --werror build option
|
||||
#common_warning_flags += '-Werror'
|
||||
common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
|
||||
endif
|
||||
cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
|
||||
cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
|
||||
add_project_arguments(cc_compile_flags, language : 'c')
|
||||
add_project_arguments(cxx_compile_flags, language : 'cpp')
|
||||
elif cc_id == compiler_msvc
|
||||
msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
|
||||
if use_multi_thread
|
||||
msvc_compile_flags += '/MP'
|
||||
endif
|
||||
if use_static_runtime
|
||||
msvc_compile_flags += '/MT'
|
||||
endif
|
||||
add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
|
||||
endif
|
||||
|
||||
# =============================================================================
|
||||
# Subdirs
|
||||
# =============================================================================
|
||||
|
||||
subdir('lib')
|
||||
|
||||
if bin_programs
|
||||
subdir('programs')
|
||||
endif
|
||||
|
||||
if bin_tests
|
||||
subdir('tests')
|
||||
endif
|
||||
|
||||
if bin_contrib
|
||||
subdir('contrib')
|
||||
endif
|
||||
@@ -1,36 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
|
||||
|
||||
option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
|
||||
description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
|
||||
option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
|
||||
description: 'Enable run-time debug. See lib/common/debug.h')
|
||||
option('backtrace', type: 'boolean', value: false,
|
||||
description: 'Display a stack backtrace when execution generates a runtime exception')
|
||||
option('static_runtime', type: 'boolean', value: false,
|
||||
description: 'Link to static run-time libraries on MSVC')
|
||||
|
||||
option('bin_programs', type: 'boolean', value: true,
|
||||
description: 'Enable programs build')
|
||||
option('bin_tests', type: 'boolean', value: false,
|
||||
description: 'Enable tests build')
|
||||
option('bin_contrib', type: 'boolean', value: false,
|
||||
description: 'Enable contrib build')
|
||||
|
||||
option('multi_thread', type: 'feature', value: 'enabled',
|
||||
description: 'Enable multi-threading when pthread is detected')
|
||||
option('zlib', type: 'feature', value: 'auto',
|
||||
description: 'Enable zlib support')
|
||||
option('lzma', type: 'feature', value: 'auto',
|
||||
description: 'Enable lzma support')
|
||||
option('lz4', type: 'feature', value: 'auto',
|
||||
description: 'Enable lz4 support')
|
||||
@@ -1,105 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/dibio.c'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
|
||||
|
||||
zstd_c_args = libzstd_debug_cflags
|
||||
if use_multi_thread
|
||||
zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
|
||||
endif
|
||||
|
||||
zstd_deps = [ libzstd_dep ]
|
||||
if use_zlib
|
||||
zstd_deps += [ zlib_dep ]
|
||||
zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lzma
|
||||
zstd_deps += [ lzma_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
|
||||
endif
|
||||
|
||||
if use_lz4
|
||||
zstd_deps += [ lz4_dep ]
|
||||
zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
|
||||
endif
|
||||
|
||||
export_dynamic_on_windows = false
|
||||
# explicit backtrace enable/disable for Linux & Darwin
|
||||
if not use_backtrace
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=0'
|
||||
elif use_debug and host_machine_os == os_windows # MinGW target
|
||||
zstd_c_args += '-DBACKTRACE_ENABLE=1'
|
||||
export_dynamic_on_windows = true
|
||||
endif
|
||||
|
||||
if cc_id == compiler_msvc
|
||||
if default_library_type != 'static'
|
||||
zstd_programs_sources += [windows_mod.compile_resources(
|
||||
join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
|
||||
endif
|
||||
endif
|
||||
|
||||
zstd = executable('zstd',
|
||||
zstd_programs_sources,
|
||||
c_args: zstd_c_args,
|
||||
dependencies: zstd_deps,
|
||||
export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
|
||||
install: true)
|
||||
|
||||
zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/fileio.c')]
|
||||
|
||||
# Minimal target, with only zstd compression and decompression.
|
||||
# No bench. No legacy.
|
||||
executable('zstd-frugal',
|
||||
zstd_frugal_sources,
|
||||
dependencies: libzstd_dep,
|
||||
c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT', '-DZSTD_NOTRACE' ],
|
||||
install: true)
|
||||
|
||||
install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless'),
|
||||
install_dir: zstd_bindir)
|
||||
|
||||
# =============================================================================
|
||||
# Programs and manpages installing
|
||||
# =============================================================================
|
||||
|
||||
install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
|
||||
join_paths(zstd_rootdir, 'programs/zstdless.1'))
|
||||
|
||||
InstallSymlink_py = '../InstallSymlink.py'
|
||||
zstd_man1_dir = join_paths(zstd_mandir, 'man1')
|
||||
bin_EXT = host_machine_os == os_windows ? '.exe' : ''
|
||||
man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
|
||||
|
||||
foreach f : ['zstdcat', 'unzstd']
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, f + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
|
||||
endforeach
|
||||
|
||||
if use_multi_thread
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + bin_EXT, 'zstdmt' + bin_EXT, zstd_bindir)
|
||||
meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
|
||||
endif
|
||||
@@ -1,206 +0,0 @@
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present Dima Krasner <dima@dimakrasner.com>
|
||||
# lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
|
||||
zstd_rootdir = '../../..'
|
||||
|
||||
tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
|
||||
os_freebsd, 'netbsd', 'dragonfly', os_sun]
|
||||
|
||||
# =============================================================================
|
||||
# Test flags
|
||||
# =============================================================================
|
||||
|
||||
FUZZER_FLAGS = ['--no-big-tests']
|
||||
FUZZERTEST = '-T200s'
|
||||
ZSTREAM_TESTTIME = '-T90s'
|
||||
DECODECORPUS_TESTTIME = '-T30'
|
||||
ZSTDRTTEST = ['--test-large-data']
|
||||
|
||||
# =============================================================================
|
||||
# Executables
|
||||
# =============================================================================
|
||||
|
||||
test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
|
||||
|
||||
datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/datagencli.c')]
|
||||
datagen = executable('datagen',
|
||||
datagen_sources,
|
||||
c_args: [ '-DNDEBUG' ],
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fullbench.c')]
|
||||
fullbench = executable('fullbench',
|
||||
fullbench_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/fuzzer.c')]
|
||||
fuzzer = executable('fuzzer',
|
||||
fuzzer_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/seqgen.c'),
|
||||
join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
|
||||
zstreamtest = executable('zstreamtest',
|
||||
zstreamtest_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: libzstd_dep,
|
||||
install: false)
|
||||
|
||||
paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'programs/benchzstd.c'),
|
||||
join_paths(zstd_rootdir, 'programs/datagen.c'),
|
||||
join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'tests/paramgrill.c')]
|
||||
paramgrill = executable('paramgrill',
|
||||
paramgrill_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
|
||||
roundTripCrash = executable('roundTripCrash',
|
||||
roundTripCrash_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
|
||||
longmatch = executable('longmatch',
|
||||
longmatch_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
|
||||
invalidDictionaries = executable('invalidDictionaries',
|
||||
invalidDictionaries_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
|
||||
legacy = executable('legacy',
|
||||
legacy_sources,
|
||||
# Use -Dlegacy_level build option to control it
|
||||
#c_args: '-DZSTD_LEGACY_SUPPORT=4',
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
endif
|
||||
|
||||
decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
|
||||
decodecorpus = executable('decodecorpus',
|
||||
decodecorpus_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, libm_dep ],
|
||||
install: false)
|
||||
|
||||
poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
|
||||
join_paths(zstd_rootdir, 'programs/timefn.c'),
|
||||
join_paths(zstd_rootdir, 'tests/poolTests.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/pool.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/threading.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
|
||||
join_paths(zstd_rootdir, 'lib/common/error_private.c')]
|
||||
poolTests = executable('poolTests',
|
||||
poolTests_sources,
|
||||
include_directories: test_includes,
|
||||
dependencies: [ libzstd_dep, thread_dep ],
|
||||
install: false)
|
||||
|
||||
checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
|
||||
checkTag = executable('checkTag',
|
||||
checkTag_sources,
|
||||
dependencies: [ libzstd_dep ],
|
||||
install: false)
|
||||
|
||||
# =============================================================================
|
||||
# Tests (Use "meson test --list" to list all tests)
|
||||
# =============================================================================
|
||||
|
||||
if tests_supported_oses.contains(host_machine_os)
|
||||
valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
|
||||
valgrindTest_py = files('valgrindTest.py')
|
||||
test('valgrindTest',
|
||||
valgrindTest_py,
|
||||
args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
|
||||
depends: [zstd, datagen, fuzzer, fullbench],
|
||||
timeout: 600) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
if host_machine_os != os_windows
|
||||
playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
|
||||
test('test-zstd',
|
||||
playTests_sh,
|
||||
args: ZSTDRTTEST,
|
||||
env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
|
||||
depends: [datagen],
|
||||
workdir: meson.current_build_dir(),
|
||||
timeout: 2800) # Timeout should work on HDD drive
|
||||
endif
|
||||
|
||||
test('test-fullbench-1',
|
||||
fullbench,
|
||||
args: ['-i1'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
test('test-fullbench-2',
|
||||
fullbench,
|
||||
args: ['-i1', '-P0'],
|
||||
depends: [datagen],
|
||||
timeout: 60)
|
||||
|
||||
if use_zlib
|
||||
test('test-fuzzer',
|
||||
fuzzer,
|
||||
args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
|
||||
timeout: 480)
|
||||
endif
|
||||
|
||||
test('test-zstream-1',
|
||||
zstreamtest,
|
||||
args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 240)
|
||||
test('test-zstream-2',
|
||||
zstreamtest,
|
||||
args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-zstream-3',
|
||||
zstreamtest,
|
||||
args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
|
||||
timeout: 120)
|
||||
test('test-longmatch', longmatch, timeout: 36)
|
||||
test('test-invalidDictionaries', invalidDictionaries) # should be fast
|
||||
if 0 < legacy_level and legacy_level <= 4
|
||||
test('test-legacy', legacy) # should be fast
|
||||
endif
|
||||
test('test-decodecorpus',
|
||||
decodecorpus,
|
||||
args: ['-t', DECODECORPUS_TESTTIME],
|
||||
timeout: 60)
|
||||
test('test-poolTests', poolTests) # should be fast
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# #############################################################################
|
||||
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# #############################################################################
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
|
||||
VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
|
||||
|
||||
print('\n ---- valgrind tests : memory analyzer ----')
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
|
||||
|
||||
if subprocess.call([*VALGRIND_ARGS, zstd],
|
||||
stdout=subprocess.DEVNULL) == 0:
|
||||
raise subprocess.CalledProcessError('zstd without argument should have failed')
|
||||
|
||||
with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmp_fd:
|
||||
with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
|
||||
stdin=p1.stdout) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
|
||||
with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
|
||||
subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
|
||||
stdin=p1.stdout,
|
||||
stdout=subprocess.DEVNULL) as p2:
|
||||
p1.stdout.close()
|
||||
p2.communicate()
|
||||
if p2.returncode != 0:
|
||||
raise subprocess.CalledProcessError()
|
||||
|
||||
subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
|
||||
subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
|
||||
parser.add_argument('valgrind', help='valgrind path')
|
||||
parser.add_argument('zstd', help='zstd path')
|
||||
parser.add_argument('datagen', help='datagen path')
|
||||
parser.add_argument('fuzzer', help='fuzzer path')
|
||||
parser.add_argument('fullbench', help='fullbench path')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
valgrind = args.valgrind
|
||||
zstd = args.zstd
|
||||
datagen = args.datagen
|
||||
fuzzer = args.fuzzer
|
||||
fullbench = args.fullbench
|
||||
|
||||
valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
# binaries generated
|
||||
adapt
|
||||
datagen
|
||||
@@ -0,0 +1,76 @@
|
||||
|
||||
ZSTDDIR = ../../lib
|
||||
PRGDIR = ../../programs
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
|
||||
MULTITHREAD_LDFLAGS = -pthread
|
||||
DEBUGFLAGS= -g -DZSTD_DEBUG=1
|
||||
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
|
||||
|
||||
all: adapt datagen
|
||||
|
||||
adapt: $(ZSTD_FILES) adapt.c
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
adapt-debug: $(ZSTD_FILES) adapt.c
|
||||
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
test-adapt-correctness: datagen adapt
|
||||
@./test-correctness.sh
|
||||
@echo "test correctness complete"
|
||||
|
||||
test-adapt-performance: datagen adapt
|
||||
@./test-performance.sh
|
||||
@echo "test performance complete"
|
||||
|
||||
clean:
|
||||
@$(RM) -f adapt datagen
|
||||
@$(RM) -rf *.dSYM
|
||||
@$(RM) -f tmp*
|
||||
@$(RM) -f tests/*.zst
|
||||
@$(RM) -f tests/tmp*
|
||||
@echo "finished cleaning"
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
PREFIX ?= /usr/local
|
||||
DESTDIR ?=
|
||||
BINDIR ?= $(PREFIX)/bin
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL) -m 755
|
||||
|
||||
install: adapt
|
||||
@echo Installing binaries
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
|
||||
@$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive installation completed
|
||||
|
||||
uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive programs successfully uninstalled
|
||||
endif
|
||||
@@ -0,0 +1,91 @@
|
||||
### Summary
|
||||
|
||||
`adapt` is a new compression tool targeted at optimizing performance across network connections and pipelines. The tool is aimed at sensing network speeds and adapting compression level based on network or pipe speeds.
|
||||
In situations where the compression level does not appropriately match the network/pipe speed, compression may be bottlenecking the entire pipeline or the files may not be compressed as much as they potentially could be, therefore losing efficiency. It also becomes quite impractical to manually measure and set an optimalcompression level (which could potentially change over time).
|
||||
|
||||
### Using `adapt`
|
||||
|
||||
In order to build and use the tool, you can simply run `make adapt` in the `adaptive-compression` directory under `contrib`. This will generate an executable available for use. Another possible method of installation is running `make install`, which will create and install the binary as the command `zstd-adaptive`.
|
||||
|
||||
Similar to many other compression utilities, `zstd-adaptive` can be invoked by using the following format:
|
||||
|
||||
`zstd-adaptive [options] [file(s)]`
|
||||
|
||||
Supported options for the above format are described below.
|
||||
|
||||
`zstd-adaptive` also supports reading from `stdin` and writing to `stdout`, which is potentially more useful. By default, if no files are given, `zstd-adaptive` reads from and writes to standard I/O. Therefore, you can simply insert it within a pipeline like so:
|
||||
|
||||
`cat FILE | zstd-adaptive | ssh "cat - > tmp.zst"`
|
||||
|
||||
If a file is provided, it is also possible to force writing to stdout using the `-c` flag like so:
|
||||
|
||||
`zstd-adaptive -c FILE | ssh "cat - > tmp.zst"`
|
||||
|
||||
Several options described below can be used to control the behavior of `zstd-adaptive`. More specifically, using the `-l#` and `-u#` flags will will set upper and lower bounds so that the compression level will always be within that range. The `-i#` flag can also be used to change the initial compression level. If an initial compression level is not provided, the initial compression level will be chosen such that it is within the appropriate range (it becomes equal to the lower bound).
|
||||
|
||||
### Options
|
||||
`-oFILE` : write output to `FILE`
|
||||
|
||||
`-i#` : provide initial compression level (must within the appropriate bounds)
|
||||
|
||||
`-h` : display help/information
|
||||
|
||||
`-f` : force the compression level to stay constant
|
||||
|
||||
`-c` : force write to `stdout`
|
||||
|
||||
`-p` : hide progress bar
|
||||
|
||||
`-q` : quiet mode -- do not show progress bar or other information
|
||||
|
||||
`-l#` : set a lower bound on the compression level (default is 1)
|
||||
|
||||
`-u#` : set an upper bound on the compression level (default is 22)
|
||||
### Benchmarking / Test results
|
||||
#### Artificial Tests
|
||||
These artificial tests were run by using the `pv` command line utility in order to limit pipe speeds (25 MB/s read and 5 MB/s write limits were chosen to mimic severe throughput constraints). A 40 GB backup file was sent through a pipeline, compressed, and written out to a file. Compression time, size, and ratio were computed. Data for `zstd -15` was excluded from these tests because the test runs quite long.
|
||||
|
||||
<table>
|
||||
<tr><th> 25 MB/s read limit </th></tr>
|
||||
<tr><td>
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.108 | 20.718 GB | 29m 48.530s |
|
||||
| zstd-adaptive | 2.230 | 19.581 GB | 29m 48.798s |
|
||||
|
||||
</td><tr>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<tr><th> 5 MB/s write limit </th></tr>
|
||||
<tr><td>
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.108 | 20.718 GB | 1h 10m 43.076s |
|
||||
| zstd-adaptive | 2.249 | 19.412 GB | 1h 06m 15.577s |
|
||||
|
||||
</td></tr>
|
||||
</table>
|
||||
|
||||
The commands used for this test generally followed the form:
|
||||
|
||||
`cat FILE | pv -L 25m -q | COMPRESSION | pv -q > tmp.zst # impose 25 MB/s read limit`
|
||||
|
||||
`cat FILE | pv -q | COMPRESSION | pv -L 5m -q > tmp.zst # impose 5 MB/s write limit`
|
||||
|
||||
#### SSH Tests
|
||||
|
||||
The following tests were performed by piping a relatively large backup file (approximately 80 GB) through compression and over SSH to be stored on a server. The test data includes statistics for time and compressed size on `zstd` at several compression levels, as well as `zstd-adaptive`. The data highlights the potential advantages that `zstd-adaptive` has over using a low static compression level and the negative imapcts that using an excessively high static compression level can have on
|
||||
pipe throughput.
|
||||
|
||||
| Compressor Name | Ratio | Compressed Size | Compression Time |
|
||||
|:----------------|------:|----------------:|-----------------:|
|
||||
| zstd -3 | 2.212 | 32.426 GB | 1h 17m 59.756s |
|
||||
| zstd -15 | 2.374 | 30.213 GB | 2h 56m 59.441s |
|
||||
| zstd-adaptive | 2.315 | 30.993 GB | 1h 18m 52.860s |
|
||||
|
||||
The commands used for this test generally followed the form:
|
||||
|
||||
`cat FILE | COMPRESSION | ssh dev "cat - > tmp.zst"`
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*/
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Dependencies
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options */
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define SIZE_DEFAULT ((64 KB) + 1)
|
||||
#define SEED_DEFAULT 0
|
||||
#define COMPRESSIBILITY_DEFAULT 50
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n",
|
||||
COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
|
||||
double litProba = 0.0;
|
||||
U64 size = SIZE_DEFAULT;
|
||||
U32 seed = SEED_DEFAULT;
|
||||
const char* const programName = argv[0];
|
||||
|
||||
int argNb;
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (*argument=='-') {
|
||||
argument++;
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
size *= 10, size += *argument++ - '0';
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
seed *= 10, seed += *argument++ - '0';
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
probaU32 = 0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
probaU32 *= 10, probaU32 += *argument++ - '0';
|
||||
if (probaU32>100) probaU32 = 100;
|
||||
break;
|
||||
case 'L': /* hidden argument : Literal distribution probability */
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
litProba *= 10, litProba += *argument++ - '0';
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default:
|
||||
return usage(programName);
|
||||
}
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
DISPLAYLEVEL(4, "Compressible data Generator \n");
|
||||
if (probaU32!=COMPRESSIBILITY_DEFAULT)
|
||||
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
|
||||
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
echo "correctness tests -- general"
|
||||
./datagen -s1 -g1GB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s2 -g500MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s3 -g250MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s4 -g125MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s5 -g50MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s6 -g25MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s7 -g10MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s8 -g5MB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s9 -g500KB > tmp
|
||||
./adapt -otmp.zst tmp
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- streaming"
|
||||
./datagen -s10 -g1GB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s11 -g100MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s12 -g10MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s13 -g1MB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s14 -g100KB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s15 -g10KB > tmp
|
||||
cat tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- read limit"
|
||||
./datagen -s16 -g1GB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s17 -g100MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s18 -g10MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s19 -g1MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s20 -g100KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s21 -g10KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- write limit"
|
||||
./datagen -s22 -g1GB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s23 -g100MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s24 -g10MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s25 -g1MB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s26 -g100KB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s27 -g10KB > tmp
|
||||
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- read and write limits"
|
||||
./datagen -s28 -g1GB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s29 -g100MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s30 -g10MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s31 -g1MB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s32 -g100KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s33 -g10KB > tmp
|
||||
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- forced compression level"
|
||||
./datagen -s34 -g1GB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s35 -g100MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s36 -g10MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s37 -g1MB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s38 -g100KB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
./datagen -s39 -g10KB > tmp
|
||||
./adapt tmp -otmp.zst -i11 -f
|
||||
zstd -d tmp.zst -o tmp2
|
||||
diff -s -q tmp tmp2
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- window size test"
|
||||
./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
|
||||
zstd -d tmp.zst
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- testing bounds"
|
||||
./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
|
||||
rm tmp*
|
||||
make clean
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
echo "testing time -- no limits set"
|
||||
./datagen -s1 -g1GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
./datagen -s2 -g2GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
./datagen -s3 -g4GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ntesting compression ratio -- no limits set"
|
||||
./datagen -s4 -g1GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s5 -g2GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s6 -g4GB > tmp
|
||||
time ./adapt -otmp1.zst tmp
|
||||
time zstd -1 -o tmp2.zst tmp
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
echo e "\ntesting performance at various compression levels -- no limits set"
|
||||
./datagen -s7 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i5 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -5 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s8 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i10 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -10 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s9 -g1GB > tmp
|
||||
echo "adapt"
|
||||
time ./adapt -i15 -f tmp -otmp1.zst
|
||||
echo "zstdcli"
|
||||
time zstd -15 tmp -o tmp2.zst
|
||||
ls -l tmp1.zst tmp2.zst
|
||||
rm tmp*
|
||||
@@ -1 +0,0 @@
|
||||
check_flipped_bits
|
||||
@@ -1,35 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2019-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
.PHONY: all
|
||||
all: check_flipped_bits
|
||||
|
||||
ZSTDLIBDIR ?= ../../lib
|
||||
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \
|
||||
-I$(ZSTDLIBDIR)/decompress
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls -Wmissing-prototypes
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
.PHONY: $(ZSTDLIBDIR)/libzstd.a
|
||||
$(ZSTDLIBDIR)/libzstd.a:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a
|
||||
$(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f check_flipped_bits
|
||||
@@ -1,400 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2019-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
typedef struct {
|
||||
char *input;
|
||||
size_t input_size;
|
||||
|
||||
char *perturbed; /* same size as input */
|
||||
|
||||
char *output;
|
||||
size_t output_size;
|
||||
|
||||
const char *dict_file_name;
|
||||
const char *dict_file_dir_name;
|
||||
int32_t dict_id;
|
||||
char *dict;
|
||||
size_t dict_size;
|
||||
ZSTD_DDict* ddict;
|
||||
|
||||
ZSTD_DCtx* dctx;
|
||||
|
||||
int success_count;
|
||||
int error_counts[ZSTD_error_maxCode];
|
||||
} stuff_t;
|
||||
|
||||
static void free_stuff(stuff_t* stuff) {
|
||||
free(stuff->input);
|
||||
free(stuff->output);
|
||||
ZSTD_freeDDict(stuff->ddict);
|
||||
free(stuff->dict);
|
||||
ZSTD_freeDCtx(stuff->dctx);
|
||||
}
|
||||
|
||||
static void usage(void) {
|
||||
fprintf(stderr, "check_flipped_bits input_filename [-d dict] [-D dict_dir]\n");
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "Arguments:\n");
|
||||
fprintf(stderr, " -d file: path to a dictionary file to use.\n");
|
||||
fprintf(stderr, " -D dir : path to a directory, with files containing dictionaries, of the\n"
|
||||
" form DICTID.zstd-dict, e.g., 12345.zstd-dict.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void print_summary(stuff_t* stuff) {
|
||||
int error_code;
|
||||
fprintf(stderr, "%9d successful decompressions\n", stuff->success_count);
|
||||
for (error_code = 0; error_code < ZSTD_error_maxCode; error_code++) {
|
||||
int count = stuff->error_counts[error_code];
|
||||
if (count) {
|
||||
fprintf(
|
||||
stderr, "%9d failed decompressions with message: %s\n",
|
||||
count, ZSTD_getErrorString(error_code));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static char* readFile(const char* filename, size_t* size) {
|
||||
struct stat statbuf;
|
||||
int ret;
|
||||
FILE* f;
|
||||
char *buf;
|
||||
size_t bytes_read;
|
||||
|
||||
ret = stat(filename, &statbuf);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "stat failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
if ((statbuf.st_mode & S_IFREG) != S_IFREG) {
|
||||
fprintf(stderr, "Input must be regular file\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*size = statbuf.st_size;
|
||||
|
||||
f = fopen(filename, "r");
|
||||
if (f == NULL) {
|
||||
fprintf(stderr, "fopen failed: %m\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
buf = malloc(*size);
|
||||
if (buf == NULL) {
|
||||
fprintf(stderr, "malloc failed\n");
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bytes_read = fread(buf, 1, *size, f);
|
||||
if (bytes_read != *size) {
|
||||
fprintf(stderr, "failed to read whole file\n");
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = fclose(f);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "fclose failed: %m\n");
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDict(const char* filename, char **buf, size_t* size, int32_t* dict_id) {
|
||||
ZSTD_DDict* ddict;
|
||||
*buf = readFile(filename, size);
|
||||
if (*buf == NULL) {
|
||||
fprintf(stderr, "Opening dictionary file '%s' failed\n", filename);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ddict = ZSTD_createDDict_advanced(*buf, *size, ZSTD_dlm_byRef, ZSTD_dct_auto, ZSTD_defaultCMem);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict.\n");
|
||||
return NULL;
|
||||
}
|
||||
if (dict_id != NULL) {
|
||||
*dict_id = ZSTD_getDictID_fromDDict(ddict);
|
||||
}
|
||||
return ddict;
|
||||
}
|
||||
|
||||
static ZSTD_DDict* readDictByID(stuff_t *stuff, int32_t dict_id, char **buf, size_t* size) {
|
||||
if (stuff->dict_file_dir_name == NULL) {
|
||||
return NULL;
|
||||
} else {
|
||||
size_t dir_name_len = strlen(stuff->dict_file_dir_name);
|
||||
int dir_needs_separator = 0;
|
||||
size_t dict_file_name_alloc_size = dir_name_len + 1 /* '/' */ + 10 /* max int32_t len */ + strlen(".zstd-dict") + 1 /* '\0' */;
|
||||
char *dict_file_name = malloc(dict_file_name_alloc_size);
|
||||
ZSTD_DDict* ddict;
|
||||
int32_t read_dict_id;
|
||||
if (dict_file_name == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (dir_name_len > 0 && stuff->dict_file_dir_name[dir_name_len - 1] != '/') {
|
||||
dir_needs_separator = 1;
|
||||
}
|
||||
|
||||
snprintf(
|
||||
dict_file_name,
|
||||
dict_file_name_alloc_size,
|
||||
"%s%s%u.zstd-dict",
|
||||
stuff->dict_file_dir_name,
|
||||
dir_needs_separator ? "/" : "",
|
||||
dict_id);
|
||||
|
||||
/* fprintf(stderr, "Loading dict %u from '%s'.\n", dict_id, dict_file_name); */
|
||||
|
||||
ddict = readDict(dict_file_name, buf, size, &read_dict_id);
|
||||
if (ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", dict_file_name);
|
||||
free(dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
if (read_dict_id != dict_id) {
|
||||
fprintf(stderr, "Read dictID (%u) does not match expected (%u).\n", read_dict_id, dict_id);
|
||||
free(dict_file_name);
|
||||
ZSTD_freeDDict(ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
free(dict_file_name);
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
static int init_stuff(stuff_t* stuff, int argc, char *argv[]) {
|
||||
const char* input_filename;
|
||||
|
||||
if (argc < 2) {
|
||||
usage();
|
||||
}
|
||||
|
||||
input_filename = argv[1];
|
||||
stuff->input_size = 0;
|
||||
stuff->input = readFile(input_filename, &stuff->input_size);
|
||||
if (stuff->input == NULL) {
|
||||
fprintf(stderr, "Failed to read input file.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->perturbed = malloc(stuff->input_size);
|
||||
if (stuff->perturbed == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
memcpy(stuff->perturbed, stuff->input, stuff->input_size);
|
||||
|
||||
stuff->output_size = ZSTD_DStreamOutSize();
|
||||
stuff->output = malloc(stuff->output_size);
|
||||
if (stuff->output == NULL) {
|
||||
fprintf(stderr, "malloc failed.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->dict_file_name = NULL;
|
||||
stuff->dict_file_dir_name = NULL;
|
||||
stuff->dict_id = 0;
|
||||
stuff->dict = NULL;
|
||||
stuff->dict_size = 0;
|
||||
stuff->ddict = NULL;
|
||||
|
||||
if (argc > 2) {
|
||||
if (!strcmp(argv[2], "-d")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else
|
||||
if (!strcmp(argv[2], "-D")) {
|
||||
if (argc > 3) {
|
||||
stuff->dict_file_dir_name = argv[3];
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
} else {
|
||||
usage();
|
||||
}
|
||||
}
|
||||
|
||||
if (stuff->dict_file_dir_name) {
|
||||
int32_t dict_id = ZSTD_getDictID_fromFrame(stuff->input, stuff->input_size);
|
||||
if (dict_id != 0) {
|
||||
stuff->ddict = readDictByID(stuff, dict_id, &stuff->dict, &stuff->dict_size);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create cached ddict.\n");
|
||||
return 0;
|
||||
}
|
||||
stuff->dict_id = dict_id;
|
||||
}
|
||||
} else
|
||||
if (stuff->dict_file_name) {
|
||||
stuff->ddict = readDict(stuff->dict_file_name, &stuff->dict, &stuff->dict_size, &stuff->dict_id);
|
||||
if (stuff->ddict == NULL) {
|
||||
fprintf(stderr, "Failed to create ddict from '%s'.\n", stuff->dict_file_name);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->dctx = ZSTD_createDCtx();
|
||||
if (stuff->dctx == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
stuff->success_count = 0;
|
||||
memset(stuff->error_counts, 0, sizeof(stuff->error_counts));
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int test_decompress(stuff_t* stuff) {
|
||||
size_t ret;
|
||||
ZSTD_inBuffer in = {stuff->perturbed, stuff->input_size, 0};
|
||||
ZSTD_outBuffer out = {stuff->output, stuff->output_size, 0};
|
||||
ZSTD_DCtx* dctx = stuff->dctx;
|
||||
int32_t custom_dict_id = ZSTD_getDictID_fromFrame(in.src, in.size);
|
||||
char *custom_dict = NULL;
|
||||
size_t custom_dict_size = 0;
|
||||
ZSTD_DDict* custom_ddict = NULL;
|
||||
|
||||
if (custom_dict_id != 0 && custom_dict_id != stuff->dict_id) {
|
||||
/* fprintf(stderr, "Instead of dict %u, this perturbed blob wants dict %u.\n", stuff->dict_id, custom_dict_id); */
|
||||
custom_ddict = readDictByID(stuff, custom_dict_id, &custom_dict, &custom_dict_size);
|
||||
}
|
||||
|
||||
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_DCtx_refDDict(dctx, custom_ddict);
|
||||
} else {
|
||||
ZSTD_DCtx_refDDict(dctx, stuff->ddict);
|
||||
}
|
||||
|
||||
while (in.pos != in.size) {
|
||||
out.pos = 0;
|
||||
ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||
|
||||
if (ZSTD_isError(ret)) {
|
||||
unsigned int code = ZSTD_getErrorCode(ret);
|
||||
if (code >= ZSTD_error_maxCode) {
|
||||
fprintf(stderr, "Received unexpected error code!\n");
|
||||
exit(1);
|
||||
}
|
||||
stuff->error_counts[code]++;
|
||||
/*
|
||||
fprintf(
|
||||
stderr, "Decompression failed: %s\n", ZSTD_getErrorName(ret));
|
||||
*/
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
stuff->success_count++;
|
||||
|
||||
if (custom_ddict != NULL) {
|
||||
ZSTD_freeDDict(custom_ddict);
|
||||
free(custom_dict);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bits(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t bit;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (bit = 0; bit < 8; bit++) {
|
||||
unsigned char new_val = old_val ^ (1 << bit);
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Flipping byte %zu bit %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, bit, old_val, new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int perturb_bytes(stuff_t* stuff) {
|
||||
size_t pos;
|
||||
size_t new_val;
|
||||
for (pos = 0; pos < stuff->input_size; pos++) {
|
||||
unsigned char old_val = stuff->input[pos];
|
||||
if (pos % 1000 == 0) {
|
||||
fprintf(stderr, "Perturbing byte %zu / %zu\n", pos, stuff->input_size);
|
||||
}
|
||||
for (new_val = 0; new_val < 256; new_val++) {
|
||||
stuff->perturbed[pos] = new_val;
|
||||
if (test_decompress(stuff)) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Changing byte %zu (0x%02x -> 0x%02x) "
|
||||
"produced a successful decompression!\n",
|
||||
pos, old_val, (unsigned char)new_val);
|
||||
}
|
||||
}
|
||||
stuff->perturbed[pos] = old_val;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
stuff_t stuff;
|
||||
|
||||
if(!init_stuff(&stuff, argc, argv)) {
|
||||
fprintf(stderr, "Failed to init.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (test_decompress(&stuff)) {
|
||||
fprintf(stderr, "Blob already decompresses successfully!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
perturb_bits(&stuff);
|
||||
|
||||
perturb_bytes(&stuff);
|
||||
|
||||
print_summary(&stuff);
|
||||
|
||||
free_stuff(&stuff);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -5,7 +5,7 @@ The `Dockerfile` script requires a version of `docker` >= 17.05
|
||||
|
||||
## Installing docker
|
||||
|
||||
The official docker install docs use a ppa with a modern version available:
|
||||
The officiel docker install docs use a ppa with a modern version available:
|
||||
https://docs.docker.com/install/linux/docker-ce/ubuntu/
|
||||
|
||||
## How to run
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
ARG :=
|
||||
|
||||
CC ?= gcc
|
||||
CFLAGS ?= -O3
|
||||
INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
|
||||
|
||||
RANDOM_FILE := ../randomDictBuilder/random.c
|
||||
IO_FILE := ../randomDictBuilder/io.c
|
||||
|
||||
all: run clean
|
||||
|
||||
.PHONY: run
|
||||
run: benchmark
|
||||
echo "Benchmarking with $(ARG)"
|
||||
./benchmark $(ARG)
|
||||
|
||||
.PHONY: test
|
||||
test: benchmarkTest clean
|
||||
|
||||
.PHONY: benchmarkTest
|
||||
benchmarkTest: benchmark test.sh
|
||||
sh test.sh
|
||||
|
||||
benchmark: benchmark.o io.o random.o libzstd.a
|
||||
$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark
|
||||
|
||||
benchmark.o: benchmark.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
|
||||
|
||||
random.o: $(RANDOM_FILE)
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
|
||||
|
||||
io.o: $(IO_FILE)
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
|
||||
|
||||
libzstd.a:
|
||||
$(MAKE) -C ../../../lib libzstd.a
|
||||
mv ../../../lib/libzstd.a .
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f *.o benchmark libzstd.a
|
||||
$(MAKE) -C ../../../lib clean
|
||||
echo "Cleaning is completed"
|
||||
@@ -0,0 +1,849 @@
|
||||
Benchmarking Dictionary Builder
|
||||
|
||||
### Permitted Argument:
|
||||
Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
|
||||
|
||||
###Running Test:
|
||||
make test
|
||||
|
||||
###Usage:
|
||||
Benchmark given input files: make ARG= followed by permitted arguments
|
||||
|
||||
### Examples:
|
||||
make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
|
||||
|
||||
###Benchmarking Result:
|
||||
- First Cover is optimize cover, second Cover uses optimized d and k from first one.
|
||||
- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. This is run for accel values from 1 to 10.
|
||||
- Fourth column is chosen d and fifth column is chosen k
|
||||
|
||||
github:
|
||||
NODICT 0.000004 2.999642
|
||||
RANDOM 0.024560 8.791189
|
||||
LEGACY 0.727109 8.173529
|
||||
COVER 40.565676 10.652243 8 1298
|
||||
COVER 3.608284 10.652243 8 1298
|
||||
FAST f=15 a=1 4.181024 10.570882 8 1154
|
||||
FAST f=15 a=1 0.040788 10.570882 8 1154
|
||||
FAST f=15 a=2 3.548352 10.574287 6 1970
|
||||
FAST f=15 a=2 0.035535 10.574287 6 1970
|
||||
FAST f=15 a=3 3.287364 10.613950 6 1010
|
||||
FAST f=15 a=3 0.032182 10.613950 6 1010
|
||||
FAST f=15 a=4 3.184976 10.573883 6 1058
|
||||
FAST f=15 a=4 0.029878 10.573883 6 1058
|
||||
FAST f=15 a=5 3.045513 10.580640 8 1154
|
||||
FAST f=15 a=5 0.022162 10.580640 8 1154
|
||||
FAST f=15 a=6 3.003296 10.583677 6 1010
|
||||
FAST f=15 a=6 0.028091 10.583677 6 1010
|
||||
FAST f=15 a=7 2.952655 10.622551 6 1106
|
||||
FAST f=15 a=7 0.02724 10.622551 6 1106
|
||||
FAST f=15 a=8 2.945674 10.614657 6 1010
|
||||
FAST f=15 a=8 0.027264 10.614657 6 1010
|
||||
FAST f=15 a=9 3.153439 10.564018 8 1154
|
||||
FAST f=15 a=9 0.020635 10.564018 8 1154
|
||||
FAST f=15 a=10 2.950416 10.511454 6 1010
|
||||
FAST f=15 a=10 0.026606 10.511454 6 1010
|
||||
FAST f=16 a=1 3.970029 10.681035 8 1154
|
||||
FAST f=16 a=1 0.038188 10.681035 8 1154
|
||||
FAST f=16 a=2 3.422892 10.484978 6 1874
|
||||
FAST f=16 a=2 0.034702 10.484978 6 1874
|
||||
FAST f=16 a=3 3.215836 10.632631 8 1154
|
||||
FAST f=16 a=3 0.026084 10.632631 8 1154
|
||||
FAST f=16 a=4 3.081353 10.626533 6 1106
|
||||
FAST f=16 a=4 0.030032 10.626533 6 1106
|
||||
FAST f=16 a=5 3.041241 10.545027 8 1922
|
||||
FAST f=16 a=5 0.022882 10.545027 8 1922
|
||||
FAST f=16 a=6 2.989390 10.638284 6 1874
|
||||
FAST f=16 a=6 0.028308 10.638284 6 1874
|
||||
FAST f=16 a=7 3.001581 10.797136 6 1106
|
||||
FAST f=16 a=7 0.027479 10.797136 6 1106
|
||||
FAST f=16 a=8 2.984107 10.658356 8 1058
|
||||
FAST f=16 a=8 0.021099 10.658356 8 1058
|
||||
FAST f=16 a=9 2.925788 10.523869 6 1010
|
||||
FAST f=16 a=9 0.026905 10.523869 6 1010
|
||||
FAST f=16 a=10 2.889605 10.745841 6 1874
|
||||
FAST f=16 a=10 0.026846 10.745841 6 1874
|
||||
FAST f=17 a=1 4.031953 10.672080 8 1202
|
||||
FAST f=17 a=1 0.040658 10.672080 8 1202
|
||||
FAST f=17 a=2 3.458107 10.589352 8 1106
|
||||
FAST f=17 a=2 0.02926 10.589352 8 1106
|
||||
FAST f=17 a=3 3.291189 10.662714 8 1154
|
||||
FAST f=17 a=3 0.026531 10.662714 8 1154
|
||||
FAST f=17 a=4 3.154950 10.549456 8 1346
|
||||
FAST f=17 a=4 0.024991 10.549456 8 1346
|
||||
FAST f=17 a=5 3.092271 10.541670 6 1202
|
||||
FAST f=17 a=5 0.038285 10.541670 6 1202
|
||||
FAST f=17 a=6 3.166146 10.729112 6 1874
|
||||
FAST f=17 a=6 0.038217 10.729112 6 1874
|
||||
FAST f=17 a=7 3.035467 10.810485 6 1106
|
||||
FAST f=17 a=7 0.036655 10.810485 6 1106
|
||||
FAST f=17 a=8 3.035668 10.530532 6 1058
|
||||
FAST f=17 a=8 0.037715 10.530532 6 1058
|
||||
FAST f=17 a=9 2.987917 10.589802 8 1922
|
||||
FAST f=17 a=9 0.02217 10.589802 8 1922
|
||||
FAST f=17 a=10 2.981647 10.722579 8 1106
|
||||
FAST f=17 a=10 0.021948 10.722579 8 1106
|
||||
FAST f=18 a=1 4.067144 10.634943 8 1154
|
||||
FAST f=18 a=1 0.041386 10.634943 8 1154
|
||||
FAST f=18 a=2 3.507377 10.546230 6 1970
|
||||
FAST f=18 a=2 0.037572 10.546230 6 1970
|
||||
FAST f=18 a=3 3.323015 10.648061 8 1154
|
||||
FAST f=18 a=3 0.028306 10.648061 8 1154
|
||||
FAST f=18 a=4 3.216735 10.705402 6 1010
|
||||
FAST f=18 a=4 0.030755 10.705402 6 1010
|
||||
FAST f=18 a=5 3.175794 10.588154 8 1874
|
||||
FAST f=18 a=5 0.025315 10.588154 8 1874
|
||||
FAST f=18 a=6 3.127459 10.751104 8 1106
|
||||
FAST f=18 a=6 0.023897 10.751104 8 1106
|
||||
FAST f=18 a=7 3.083017 10.780402 6 1106
|
||||
FAST f=18 a=7 0.029158 10.780402 6 1106
|
||||
FAST f=18 a=8 3.069700 10.547226 8 1346
|
||||
FAST f=18 a=8 0.024046 10.547226 8 1346
|
||||
FAST f=18 a=9 3.056591 10.674759 6 1010
|
||||
FAST f=18 a=9 0.028496 10.674759 6 1010
|
||||
FAST f=18 a=10 3.063588 10.737578 8 1106
|
||||
FAST f=18 a=10 0.023033 10.737578 8 1106
|
||||
FAST f=19 a=1 4.164041 10.650333 8 1154
|
||||
FAST f=19 a=1 0.042906 10.650333 8 1154
|
||||
FAST f=19 a=2 3.585409 10.577066 6 1058
|
||||
FAST f=19 a=2 0.038994 10.577066 6 1058
|
||||
FAST f=19 a=3 3.439643 10.639403 8 1154
|
||||
FAST f=19 a=3 0.028427 10.639403 8 1154
|
||||
FAST f=19 a=4 3.268869 10.554410 8 1298
|
||||
FAST f=19 a=4 0.026866 10.554410 8 1298
|
||||
FAST f=19 a=5 3.238225 10.615109 6 1010
|
||||
FAST f=19 a=5 0.03078 10.615109 6 1010
|
||||
FAST f=19 a=6 3.199558 10.609782 6 1874
|
||||
FAST f=19 a=6 0.030099 10.609782 6 1874
|
||||
FAST f=19 a=7 3.132395 10.794753 6 1106
|
||||
FAST f=19 a=7 0.028964 10.794753 6 1106
|
||||
FAST f=19 a=8 3.148446 10.554842 8 1298
|
||||
FAST f=19 a=8 0.024277 10.554842 8 1298
|
||||
FAST f=19 a=9 3.108324 10.668763 6 1010
|
||||
FAST f=19 a=9 0.02896 10.668763 6 1010
|
||||
FAST f=19 a=10 3.159863 10.757347 8 1106
|
||||
FAST f=19 a=10 0.023351 10.757347 8 1106
|
||||
FAST f=20 a=1 4.462698 10.661788 8 1154
|
||||
FAST f=20 a=1 0.047174 10.661788 8 1154
|
||||
FAST f=20 a=2 3.820269 10.678612 6 1106
|
||||
FAST f=20 a=2 0.040807 10.678612 6 1106
|
||||
FAST f=20 a=3 3.644955 10.648424 8 1154
|
||||
FAST f=20 a=3 0.031398 10.648424 8 1154
|
||||
FAST f=20 a=4 3.546257 10.559756 8 1298
|
||||
FAST f=20 a=4 0.029856 10.559756 8 1298
|
||||
FAST f=20 a=5 3.485248 10.646637 6 1010
|
||||
FAST f=20 a=5 0.033756 10.646637 6 1010
|
||||
FAST f=20 a=6 3.490438 10.775824 8 1106
|
||||
FAST f=20 a=6 0.028338 10.775824 8 1106
|
||||
FAST f=20 a=7 3.631289 10.801795 6 1106
|
||||
FAST f=20 a=7 0.035228 10.801795 6 1106
|
||||
FAST f=20 a=8 3.758936 10.545116 8 1346
|
||||
FAST f=20 a=8 0.027495 10.545116 8 1346
|
||||
FAST f=20 a=9 3.707024 10.677454 6 1010
|
||||
FAST f=20 a=9 0.031326 10.677454 6 1010
|
||||
FAST f=20 a=10 3.586593 10.756017 8 1106
|
||||
FAST f=20 a=10 0.027122 10.756017 8 1106
|
||||
FAST f=21 a=1 5.701396 10.655398 8 1154
|
||||
FAST f=21 a=1 0.067744 10.655398 8 1154
|
||||
FAST f=21 a=2 5.270542 10.650743 6 1106
|
||||
FAST f=21 a=2 0.052999 10.650743 6 1106
|
||||
FAST f=21 a=3 4.945294 10.652380 8 1154
|
||||
FAST f=21 a=3 0.052678 10.652380 8 1154
|
||||
FAST f=21 a=4 4.894079 10.543185 8 1298
|
||||
FAST f=21 a=4 0.04997 10.543185 8 1298
|
||||
FAST f=21 a=5 4.785417 10.630321 6 1010
|
||||
FAST f=21 a=5 0.045294 10.630321 6 1010
|
||||
FAST f=21 a=6 4.789381 10.664477 6 1874
|
||||
FAST f=21 a=6 0.046578 10.664477 6 1874
|
||||
FAST f=21 a=7 4.302955 10.805179 6 1106
|
||||
FAST f=21 a=7 0.041205 10.805179 6 1106
|
||||
FAST f=21 a=8 4.034630 10.551211 8 1298
|
||||
FAST f=21 a=8 0.040121 10.551211 8 1298
|
||||
FAST f=21 a=9 4.523868 10.799114 6 1010
|
||||
FAST f=21 a=9 0.043592 10.799114 6 1010
|
||||
FAST f=21 a=10 4.760736 10.750255 8 1106
|
||||
FAST f=21 a=10 0.043483 10.750255 8 1106
|
||||
FAST f=22 a=1 6.743064 10.640537 8 1154
|
||||
FAST f=22 a=1 0.086967 10.640537 8 1154
|
||||
FAST f=22 a=2 6.121739 10.626638 6 1970
|
||||
FAST f=22 a=2 0.066337 10.626638 6 1970
|
||||
FAST f=22 a=3 5.248851 10.640688 8 1154
|
||||
FAST f=22 a=3 0.054935 10.640688 8 1154
|
||||
FAST f=22 a=4 5.436579 10.588333 8 1298
|
||||
FAST f=22 a=4 0.064113 10.588333 8 1298
|
||||
FAST f=22 a=5 5.812815 10.652653 6 1010
|
||||
FAST f=22 a=5 0.058189 10.652653 6 1010
|
||||
FAST f=22 a=6 5.745472 10.666437 6 1874
|
||||
FAST f=22 a=6 0.057188 10.666437 6 1874
|
||||
FAST f=22 a=7 5.716393 10.806911 6 1106
|
||||
FAST f=22 a=7 0.056 10.806911 6 1106
|
||||
FAST f=22 a=8 5.698799 10.530784 8 1298
|
||||
FAST f=22 a=8 0.0583 10.530784 8 1298
|
||||
FAST f=22 a=9 5.710533 10.777391 6 1010
|
||||
FAST f=22 a=9 0.054945 10.777391 6 1010
|
||||
FAST f=22 a=10 5.685395 10.745023 8 1106
|
||||
FAST f=22 a=10 0.056526 10.745023 8 1106
|
||||
FAST f=23 a=1 7.836923 10.638828 8 1154
|
||||
FAST f=23 a=1 0.099522 10.638828 8 1154
|
||||
FAST f=23 a=2 6.627834 10.631061 6 1970
|
||||
FAST f=23 a=2 0.066769 10.631061 6 1970
|
||||
FAST f=23 a=3 5.602533 10.647288 8 1154
|
||||
FAST f=23 a=3 0.064513 10.647288 8 1154
|
||||
FAST f=23 a=4 6.005580 10.568747 8 1298
|
||||
FAST f=23 a=4 0.062022 10.568747 8 1298
|
||||
FAST f=23 a=5 5.481816 10.676921 6 1010
|
||||
FAST f=23 a=5 0.058959 10.676921 6 1010
|
||||
FAST f=23 a=6 5.460444 10.666194 6 1874
|
||||
FAST f=23 a=6 0.057687 10.666194 6 1874
|
||||
FAST f=23 a=7 5.659822 10.800377 6 1106
|
||||
FAST f=23 a=7 0.06783 10.800377 6 1106
|
||||
FAST f=23 a=8 6.826940 10.522167 8 1298
|
||||
FAST f=23 a=8 0.070533 10.522167 8 1298
|
||||
FAST f=23 a=9 6.804757 10.577799 8 1682
|
||||
FAST f=23 a=9 0.069949 10.577799 8 1682
|
||||
FAST f=23 a=10 6.774933 10.742093 8 1106
|
||||
FAST f=23 a=10 0.068395 10.742093 8 1106
|
||||
FAST f=24 a=1 8.444110 10.632783 8 1154
|
||||
FAST f=24 a=1 0.094357 10.632783 8 1154
|
||||
FAST f=24 a=2 7.289578 10.631061 6 1970
|
||||
FAST f=24 a=2 0.098515 10.631061 6 1970
|
||||
FAST f=24 a=3 8.619780 10.646289 8 1154
|
||||
FAST f=24 a=3 0.098041 10.646289 8 1154
|
||||
FAST f=24 a=4 8.508455 10.555199 8 1298
|
||||
FAST f=24 a=4 0.093885 10.555199 8 1298
|
||||
FAST f=24 a=5 8.471145 10.674363 6 1010
|
||||
FAST f=24 a=5 0.088676 10.674363 6 1010
|
||||
FAST f=24 a=6 8.426727 10.667228 6 1874
|
||||
FAST f=24 a=6 0.087247 10.667228 6 1874
|
||||
FAST f=24 a=7 8.356826 10.803027 6 1106
|
||||
FAST f=24 a=7 0.085835 10.803027 6 1106
|
||||
FAST f=24 a=8 6.756811 10.522049 8 1298
|
||||
FAST f=24 a=8 0.07107 10.522049 8 1298
|
||||
FAST f=24 a=9 6.548169 10.571882 8 1682
|
||||
FAST f=24 a=9 0.0713 10.571882 8 1682
|
||||
FAST f=24 a=10 8.238079 10.736453 8 1106
|
||||
FAST f=24 a=10 0.07004 10.736453 8 1106
|
||||
|
||||
|
||||
hg-commands:
|
||||
NODICT 0.000005 2.425276
|
||||
RANDOM 0.046332 3.490331
|
||||
LEGACY 0.720351 3.911682
|
||||
COVER 45.507731 4.132653 8 386
|
||||
COVER 1.868810 4.132653 8 386
|
||||
FAST f=15 a=1 4.561427 3.866894 8 1202
|
||||
FAST f=15 a=1 0.048946 3.866894 8 1202
|
||||
FAST f=15 a=2 3.574462 3.892119 8 1538
|
||||
FAST f=15 a=2 0.033677 3.892119 8 1538
|
||||
FAST f=15 a=3 3.230227 3.888791 6 1346
|
||||
FAST f=15 a=3 0.034312 3.888791 6 1346
|
||||
FAST f=15 a=4 3.042388 3.899739 8 1010
|
||||
FAST f=15 a=4 0.024307 3.899739 8 1010
|
||||
FAST f=15 a=5 2.800148 3.896220 8 818
|
||||
FAST f=15 a=5 0.022331 3.896220 8 818
|
||||
FAST f=15 a=6 2.706518 3.882039 8 578
|
||||
FAST f=15 a=6 0.020955 3.882039 8 578
|
||||
FAST f=15 a=7 2.701820 3.885430 6 866
|
||||
FAST f=15 a=7 0.026074 3.885430 6 866
|
||||
FAST f=15 a=8 2.604445 3.906932 8 1826
|
||||
FAST f=15 a=8 0.021789 3.906932 8 1826
|
||||
FAST f=15 a=9 2.598568 3.870324 6 1682
|
||||
FAST f=15 a=9 0.026004 3.870324 6 1682
|
||||
FAST f=15 a=10 2.575920 3.920783 8 1442
|
||||
FAST f=15 a=10 0.020228 3.920783 8 1442
|
||||
FAST f=16 a=1 4.630623 4.001430 8 770
|
||||
FAST f=16 a=1 0.047497 4.001430 8 770
|
||||
FAST f=16 a=2 3.674721 3.974431 8 1874
|
||||
FAST f=16 a=2 0.035761 3.974431 8 1874
|
||||
FAST f=16 a=3 3.338384 3.978703 8 1010
|
||||
FAST f=16 a=3 0.029436 3.978703 8 1010
|
||||
FAST f=16 a=4 3.004412 3.983035 8 1010
|
||||
FAST f=16 a=4 0.025744 3.983035 8 1010
|
||||
FAST f=16 a=5 2.881892 3.987710 8 770
|
||||
FAST f=16 a=5 0.023211 3.987710 8 770
|
||||
FAST f=16 a=6 2.807410 3.952717 8 1298
|
||||
FAST f=16 a=6 0.023199 3.952717 8 1298
|
||||
FAST f=16 a=7 2.819623 3.994627 8 770
|
||||
FAST f=16 a=7 0.021806 3.994627 8 770
|
||||
FAST f=16 a=8 2.740092 3.954032 8 1826
|
||||
FAST f=16 a=8 0.0226 3.954032 8 1826
|
||||
FAST f=16 a=9 2.682564 3.969879 6 1442
|
||||
FAST f=16 a=9 0.026324 3.969879 6 1442
|
||||
FAST f=16 a=10 2.657959 3.969755 8 674
|
||||
FAST f=16 a=10 0.020413 3.969755 8 674
|
||||
FAST f=17 a=1 4.729228 4.046000 8 530
|
||||
FAST f=17 a=1 0.049703 4.046000 8 530
|
||||
FAST f=17 a=2 3.764510 3.991519 8 1970
|
||||
FAST f=17 a=2 0.038195 3.991519 8 1970
|
||||
FAST f=17 a=3 3.416992 4.006296 6 914
|
||||
FAST f=17 a=3 0.036244 4.006296 6 914
|
||||
FAST f=17 a=4 3.145626 3.979182 8 1970
|
||||
FAST f=17 a=4 0.028676 3.979182 8 1970
|
||||
FAST f=17 a=5 2.995070 4.050070 8 770
|
||||
FAST f=17 a=5 0.025707 4.050070 8 770
|
||||
FAST f=17 a=6 2.911833 4.040024 8 770
|
||||
FAST f=17 a=6 0.02453 4.040024 8 770
|
||||
FAST f=17 a=7 2.894796 4.015884 8 818
|
||||
FAST f=17 a=7 0.023956 4.015884 8 818
|
||||
FAST f=17 a=8 2.789962 4.039303 8 530
|
||||
FAST f=17 a=8 0.023219 4.039303 8 530
|
||||
FAST f=17 a=9 2.787625 3.996762 8 1634
|
||||
FAST f=17 a=9 0.023651 3.996762 8 1634
|
||||
FAST f=17 a=10 2.754796 4.005059 8 1058
|
||||
FAST f=17 a=10 0.022537 4.005059 8 1058
|
||||
FAST f=18 a=1 4.779117 4.038214 8 242
|
||||
FAST f=18 a=1 0.048814 4.038214 8 242
|
||||
FAST f=18 a=2 3.829753 4.045768 8 722
|
||||
FAST f=18 a=2 0.036541 4.045768 8 722
|
||||
FAST f=18 a=3 3.495053 4.021497 8 770
|
||||
FAST f=18 a=3 0.032648 4.021497 8 770
|
||||
FAST f=18 a=4 3.221395 4.039623 8 770
|
||||
FAST f=18 a=4 0.027818 4.039623 8 770
|
||||
FAST f=18 a=5 3.059369 4.050414 8 530
|
||||
FAST f=18 a=5 0.026296 4.050414 8 530
|
||||
FAST f=18 a=6 3.019292 4.010714 6 962
|
||||
FAST f=18 a=6 0.031104 4.010714 6 962
|
||||
FAST f=18 a=7 2.949322 4.031439 6 770
|
||||
FAST f=18 a=7 0.030745 4.031439 6 770
|
||||
FAST f=18 a=8 2.876425 4.032088 6 386
|
||||
FAST f=18 a=8 0.027407 4.032088 6 386
|
||||
FAST f=18 a=9 2.850958 4.053372 8 674
|
||||
FAST f=18 a=9 0.023799 4.053372 8 674
|
||||
FAST f=18 a=10 2.884352 4.020148 8 1730
|
||||
FAST f=18 a=10 0.024401 4.020148 8 1730
|
||||
FAST f=19 a=1 4.815669 4.061203 8 674
|
||||
FAST f=19 a=1 0.051425 4.061203 8 674
|
||||
FAST f=19 a=2 3.951356 4.013822 8 1442
|
||||
FAST f=19 a=2 0.039968 4.013822 8 1442
|
||||
FAST f=19 a=3 3.554682 4.050425 8 722
|
||||
FAST f=19 a=3 0.032725 4.050425 8 722
|
||||
FAST f=19 a=4 3.242585 4.054677 8 722
|
||||
FAST f=19 a=4 0.028194 4.054677 8 722
|
||||
FAST f=19 a=5 3.105909 4.064524 8 818
|
||||
FAST f=19 a=5 0.02675 4.064524 8 818
|
||||
FAST f=19 a=6 3.059901 4.036857 8 1250
|
||||
FAST f=19 a=6 0.026396 4.036857 8 1250
|
||||
FAST f=19 a=7 3.016151 4.068234 6 770
|
||||
FAST f=19 a=7 0.031501 4.068234 6 770
|
||||
FAST f=19 a=8 2.962902 4.077509 8 530
|
||||
FAST f=19 a=8 0.023333 4.077509 8 530
|
||||
FAST f=19 a=9 2.899607 4.067328 8 530
|
||||
FAST f=19 a=9 0.024553 4.067328 8 530
|
||||
FAST f=19 a=10 2.950978 4.059901 8 434
|
||||
FAST f=19 a=10 0.023852 4.059901 8 434
|
||||
FAST f=20 a=1 5.259834 4.027579 8 1634
|
||||
FAST f=20 a=1 0.061123 4.027579 8 1634
|
||||
FAST f=20 a=2 4.382150 4.025093 8 1634
|
||||
FAST f=20 a=2 0.048009 4.025093 8 1634
|
||||
FAST f=20 a=3 4.104323 4.060842 8 530
|
||||
FAST f=20 a=3 0.040965 4.060842 8 530
|
||||
FAST f=20 a=4 3.853340 4.023504 6 914
|
||||
FAST f=20 a=4 0.041072 4.023504 6 914
|
||||
FAST f=20 a=5 3.728841 4.018089 6 1634
|
||||
FAST f=20 a=5 0.037469 4.018089 6 1634
|
||||
FAST f=20 a=6 3.683045 4.069138 8 578
|
||||
FAST f=20 a=6 0.028011 4.069138 8 578
|
||||
FAST f=20 a=7 3.726973 4.063160 8 722
|
||||
FAST f=20 a=7 0.028437 4.063160 8 722
|
||||
FAST f=20 a=8 3.555073 4.057690 8 386
|
||||
FAST f=20 a=8 0.027588 4.057690 8 386
|
||||
FAST f=20 a=9 3.551095 4.067253 8 482
|
||||
FAST f=20 a=9 0.025976 4.067253 8 482
|
||||
FAST f=20 a=10 3.490127 4.068518 8 530
|
||||
FAST f=20 a=10 0.025971 4.068518 8 530
|
||||
FAST f=21 a=1 7.343816 4.064945 8 770
|
||||
FAST f=21 a=1 0.085035 4.064945 8 770
|
||||
FAST f=21 a=2 5.930894 4.048206 8 386
|
||||
FAST f=21 a=2 0.067349 4.048206 8 386
|
||||
FAST f=21 a=3 6.770775 4.063417 8 578
|
||||
FAST f=21 a=3 0.077104 4.063417 8 578
|
||||
FAST f=21 a=4 6.889409 4.066761 8 626
|
||||
FAST f=21 a=4 0.0717 4.066761 8 626
|
||||
FAST f=21 a=5 6.714896 4.051813 8 914
|
||||
FAST f=21 a=5 0.071026 4.051813 8 914
|
||||
FAST f=21 a=6 6.539890 4.047263 8 1922
|
||||
FAST f=21 a=6 0.07127 4.047263 8 1922
|
||||
FAST f=21 a=7 6.511052 4.068373 8 482
|
||||
FAST f=21 a=7 0.065467 4.068373 8 482
|
||||
FAST f=21 a=8 6.458788 4.071597 8 482
|
||||
FAST f=21 a=8 0.063817 4.071597 8 482
|
||||
FAST f=21 a=9 6.377591 4.052905 8 434
|
||||
FAST f=21 a=9 0.063112 4.052905 8 434
|
||||
FAST f=21 a=10 6.360752 4.047773 8 530
|
||||
FAST f=21 a=10 0.063606 4.047773 8 530
|
||||
FAST f=22 a=1 10.523471 4.040812 8 962
|
||||
FAST f=22 a=1 0.14214 4.040812 8 962
|
||||
FAST f=22 a=2 9.454758 4.059396 8 914
|
||||
FAST f=22 a=2 0.118343 4.059396 8 914
|
||||
FAST f=22 a=3 9.043197 4.043019 8 1922
|
||||
FAST f=22 a=3 0.109798 4.043019 8 1922
|
||||
FAST f=22 a=4 8.716261 4.044819 8 770
|
||||
FAST f=22 a=4 0.099687 4.044819 8 770
|
||||
FAST f=22 a=5 8.529472 4.070576 8 530
|
||||
FAST f=22 a=5 0.093127 4.070576 8 530
|
||||
FAST f=22 a=6 8.424241 4.070565 8 722
|
||||
FAST f=22 a=6 0.093703 4.070565 8 722
|
||||
FAST f=22 a=7 8.403391 4.070591 8 578
|
||||
FAST f=22 a=7 0.089763 4.070591 8 578
|
||||
FAST f=22 a=8 8.285221 4.089171 8 530
|
||||
FAST f=22 a=8 0.087716 4.089171 8 530
|
||||
FAST f=22 a=9 8.282506 4.047470 8 722
|
||||
FAST f=22 a=9 0.089773 4.047470 8 722
|
||||
FAST f=22 a=10 8.241809 4.064151 8 818
|
||||
FAST f=22 a=10 0.090413 4.064151 8 818
|
||||
FAST f=23 a=1 12.389208 4.051635 6 530
|
||||
FAST f=23 a=1 0.147796 4.051635 6 530
|
||||
FAST f=23 a=2 11.300910 4.042835 6 914
|
||||
FAST f=23 a=2 0.133178 4.042835 6 914
|
||||
FAST f=23 a=3 10.879455 4.047415 8 626
|
||||
FAST f=23 a=3 0.129571 4.047415 8 626
|
||||
FAST f=23 a=4 10.522718 4.038269 6 914
|
||||
FAST f=23 a=4 0.118121 4.038269 6 914
|
||||
FAST f=23 a=5 10.348043 4.066884 8 434
|
||||
FAST f=23 a=5 0.112098 4.066884 8 434
|
||||
FAST f=23 a=6 10.238630 4.048635 8 1010
|
||||
FAST f=23 a=6 0.120281 4.048635 8 1010
|
||||
FAST f=23 a=7 10.213255 4.061809 8 530
|
||||
FAST f=23 a=7 0.1121 4.061809 8 530
|
||||
FAST f=23 a=8 10.107879 4.074104 8 818
|
||||
FAST f=23 a=8 0.116544 4.074104 8 818
|
||||
FAST f=23 a=9 10.063424 4.064811 8 674
|
||||
FAST f=23 a=9 0.109045 4.064811 8 674
|
||||
FAST f=23 a=10 10.035801 4.054918 8 530
|
||||
FAST f=23 a=10 0.108735 4.054918 8 530
|
||||
FAST f=24 a=1 14.963878 4.073490 8 722
|
||||
FAST f=24 a=1 0.206344 4.073490 8 722
|
||||
FAST f=24 a=2 13.833472 4.036100 8 962
|
||||
FAST f=24 a=2 0.17486 4.036100 8 962
|
||||
FAST f=24 a=3 13.404631 4.026281 6 1106
|
||||
FAST f=24 a=3 0.153961 4.026281 6 1106
|
||||
FAST f=24 a=4 13.041164 4.065448 8 674
|
||||
FAST f=24 a=4 0.155509 4.065448 8 674
|
||||
FAST f=24 a=5 12.879412 4.054636 8 674
|
||||
FAST f=24 a=5 0.148282 4.054636 8 674
|
||||
FAST f=24 a=6 12.773736 4.081376 8 530
|
||||
FAST f=24 a=6 0.142563 4.081376 8 530
|
||||
FAST f=24 a=7 12.711310 4.059834 8 770
|
||||
FAST f=24 a=7 0.149321 4.059834 8 770
|
||||
FAST f=24 a=8 12.635459 4.052050 8 1298
|
||||
FAST f=24 a=8 0.15095 4.052050 8 1298
|
||||
FAST f=24 a=9 12.558104 4.076516 8 722
|
||||
FAST f=24 a=9 0.144361 4.076516 8 722
|
||||
FAST f=24 a=10 10.661348 4.062137 8 818
|
||||
FAST f=24 a=10 0.108232 4.062137 8 818
|
||||
|
||||
|
||||
hg-changelog:
|
||||
NODICT 0.000017 1.377590
|
||||
RANDOM 0.186171 2.097487
|
||||
LEGACY 1.670867 2.058907
|
||||
COVER 173.561948 2.189685 8 98
|
||||
COVER 4.811180 2.189685 8 98
|
||||
FAST f=15 a=1 18.685906 2.129682 8 434
|
||||
FAST f=15 a=1 0.173376 2.129682 8 434
|
||||
FAST f=15 a=2 12.928259 2.131890 8 482
|
||||
FAST f=15 a=2 0.102582 2.131890 8 482
|
||||
FAST f=15 a=3 11.132343 2.128027 8 386
|
||||
FAST f=15 a=3 0.077122 2.128027 8 386
|
||||
FAST f=15 a=4 10.120683 2.125797 8 434
|
||||
FAST f=15 a=4 0.065175 2.125797 8 434
|
||||
FAST f=15 a=5 9.479092 2.127697 8 386
|
||||
FAST f=15 a=5 0.057905 2.127697 8 386
|
||||
FAST f=15 a=6 9.159523 2.127132 8 1682
|
||||
FAST f=15 a=6 0.058604 2.127132 8 1682
|
||||
FAST f=15 a=7 8.724003 2.129914 8 434
|
||||
FAST f=15 a=7 0.0493 2.129914 8 434
|
||||
FAST f=15 a=8 8.595001 2.127137 8 338
|
||||
FAST f=15 a=8 0.0474 2.127137 8 338
|
||||
FAST f=15 a=9 8.356405 2.125512 8 482
|
||||
FAST f=15 a=9 0.046126 2.125512 8 482
|
||||
FAST f=15 a=10 8.207111 2.126066 8 338
|
||||
FAST f=15 a=10 0.043292 2.126066 8 338
|
||||
FAST f=16 a=1 18.464436 2.144040 8 242
|
||||
FAST f=16 a=1 0.172156 2.144040 8 242
|
||||
FAST f=16 a=2 12.844825 2.148171 8 194
|
||||
FAST f=16 a=2 0.099619 2.148171 8 194
|
||||
FAST f=16 a=3 11.082568 2.140837 8 290
|
||||
FAST f=16 a=3 0.079165 2.140837 8 290
|
||||
FAST f=16 a=4 10.066749 2.144405 8 386
|
||||
FAST f=16 a=4 0.068411 2.144405 8 386
|
||||
FAST f=16 a=5 9.501121 2.140720 8 386
|
||||
FAST f=16 a=5 0.061316 2.140720 8 386
|
||||
FAST f=16 a=6 9.179332 2.139478 8 386
|
||||
FAST f=16 a=6 0.056322 2.139478 8 386
|
||||
FAST f=16 a=7 8.849438 2.142412 8 194
|
||||
FAST f=16 a=7 0.050493 2.142412 8 194
|
||||
FAST f=16 a=8 8.810919 2.143454 8 434
|
||||
FAST f=16 a=8 0.051304 2.143454 8 434
|
||||
FAST f=16 a=9 8.553900 2.140339 8 194
|
||||
FAST f=16 a=9 0.047285 2.140339 8 194
|
||||
FAST f=16 a=10 8.398027 2.143130 8 386
|
||||
FAST f=16 a=10 0.046386 2.143130 8 386
|
||||
FAST f=17 a=1 18.644657 2.157192 8 98
|
||||
FAST f=17 a=1 0.173884 2.157192 8 98
|
||||
FAST f=17 a=2 13.071242 2.159830 8 146
|
||||
FAST f=17 a=2 0.10388 2.159830 8 146
|
||||
FAST f=17 a=3 11.332366 2.153654 6 194
|
||||
FAST f=17 a=3 0.08983 2.153654 6 194
|
||||
FAST f=17 a=4 10.362413 2.156813 8 242
|
||||
FAST f=17 a=4 0.070389 2.156813 8 242
|
||||
FAST f=17 a=5 9.808159 2.155098 6 338
|
||||
FAST f=17 a=5 0.072661 2.155098 6 338
|
||||
FAST f=17 a=6 9.451165 2.153845 6 146
|
||||
FAST f=17 a=6 0.064959 2.153845 6 146
|
||||
FAST f=17 a=7 9.163097 2.155424 6 242
|
||||
FAST f=17 a=7 0.064323 2.155424 6 242
|
||||
FAST f=17 a=8 9.047276 2.156640 8 242
|
||||
FAST f=17 a=8 0.053382 2.156640 8 242
|
||||
FAST f=17 a=9 8.807671 2.152396 8 146
|
||||
FAST f=17 a=9 0.049617 2.152396 8 146
|
||||
FAST f=17 a=10 8.649827 2.152370 8 146
|
||||
FAST f=17 a=10 0.047849 2.152370 8 146
|
||||
FAST f=18 a=1 18.809502 2.168116 8 98
|
||||
FAST f=18 a=1 0.175226 2.168116 8 98
|
||||
FAST f=18 a=2 13.756502 2.170870 6 242
|
||||
FAST f=18 a=2 0.119507 2.170870 6 242
|
||||
FAST f=18 a=3 12.059748 2.163094 6 98
|
||||
FAST f=18 a=3 0.093912 2.163094 6 98
|
||||
FAST f=18 a=4 11.410294 2.172372 8 98
|
||||
FAST f=18 a=4 0.073048 2.172372 8 98
|
||||
FAST f=18 a=5 10.560297 2.166388 8 98
|
||||
FAST f=18 a=5 0.065136 2.166388 8 98
|
||||
FAST f=18 a=6 10.071390 2.162672 8 98
|
||||
FAST f=18 a=6 0.059402 2.162672 8 98
|
||||
FAST f=18 a=7 10.084214 2.166624 6 194
|
||||
FAST f=18 a=7 0.073276 2.166624 6 194
|
||||
FAST f=18 a=8 9.953226 2.167454 8 98
|
||||
FAST f=18 a=8 0.053659 2.167454 8 98
|
||||
FAST f=18 a=9 8.982461 2.161593 6 146
|
||||
FAST f=18 a=9 0.05955 2.161593 6 146
|
||||
FAST f=18 a=10 8.986092 2.164373 6 242
|
||||
FAST f=18 a=10 0.059135 2.164373 6 242
|
||||
FAST f=19 a=1 18.908277 2.176021 8 98
|
||||
FAST f=19 a=1 0.177316 2.176021 8 98
|
||||
FAST f=19 a=2 13.471313 2.176103 8 98
|
||||
FAST f=19 a=2 0.106344 2.176103 8 98
|
||||
FAST f=19 a=3 11.571406 2.172812 8 98
|
||||
FAST f=19 a=3 0.083293 2.172812 8 98
|
||||
FAST f=19 a=4 10.632775 2.177770 6 146
|
||||
FAST f=19 a=4 0.079864 2.177770 6 146
|
||||
FAST f=19 a=5 10.030190 2.175574 6 146
|
||||
FAST f=19 a=5 0.07223 2.175574 6 146
|
||||
FAST f=19 a=6 9.717818 2.169997 8 98
|
||||
FAST f=19 a=6 0.060049 2.169997 8 98
|
||||
FAST f=19 a=7 9.397531 2.172770 8 146
|
||||
FAST f=19 a=7 0.057188 2.172770 8 146
|
||||
FAST f=19 a=8 9.281061 2.175822 8 98
|
||||
FAST f=19 a=8 0.053711 2.175822 8 98
|
||||
FAST f=19 a=9 9.165242 2.169849 6 146
|
||||
FAST f=19 a=9 0.059898 2.169849 6 146
|
||||
FAST f=19 a=10 9.048763 2.173394 8 98
|
||||
FAST f=19 a=10 0.049757 2.173394 8 98
|
||||
FAST f=20 a=1 21.166917 2.183923 6 98
|
||||
FAST f=20 a=1 0.205425 2.183923 6 98
|
||||
FAST f=20 a=2 15.642753 2.182349 6 98
|
||||
FAST f=20 a=2 0.135957 2.182349 6 98
|
||||
FAST f=20 a=3 14.053730 2.173544 6 98
|
||||
FAST f=20 a=3 0.11266 2.173544 6 98
|
||||
FAST f=20 a=4 15.270019 2.183656 8 98
|
||||
FAST f=20 a=4 0.107892 2.183656 8 98
|
||||
FAST f=20 a=5 15.497927 2.174661 6 98
|
||||
FAST f=20 a=5 0.100305 2.174661 6 98
|
||||
FAST f=20 a=6 13.973505 2.172391 8 98
|
||||
FAST f=20 a=6 0.087565 2.172391 8 98
|
||||
FAST f=20 a=7 14.083296 2.172443 8 98
|
||||
FAST f=20 a=7 0.078062 2.172443 8 98
|
||||
FAST f=20 a=8 12.560048 2.175581 8 98
|
||||
FAST f=20 a=8 0.070282 2.175581 8 98
|
||||
FAST f=20 a=9 13.078645 2.173975 6 146
|
||||
FAST f=20 a=9 0.081041 2.173975 6 146
|
||||
FAST f=20 a=10 12.823328 2.177778 8 98
|
||||
FAST f=20 a=10 0.074522 2.177778 8 98
|
||||
FAST f=21 a=1 29.825370 2.183057 6 98
|
||||
FAST f=21 a=1 0.334453 2.183057 6 98
|
||||
FAST f=21 a=2 29.476474 2.182752 8 98
|
||||
FAST f=21 a=2 0.286602 2.182752 8 98
|
||||
FAST f=21 a=3 25.937186 2.175867 8 98
|
||||
FAST f=21 a=3 0.17626 2.175867 8 98
|
||||
FAST f=21 a=4 20.413865 2.179780 8 98
|
||||
FAST f=21 a=4 0.206085 2.179780 8 98
|
||||
FAST f=21 a=5 20.541889 2.178328 6 146
|
||||
FAST f=21 a=5 0.199157 2.178328 6 146
|
||||
FAST f=21 a=6 21.090670 2.174443 6 146
|
||||
FAST f=21 a=6 0.190645 2.174443 6 146
|
||||
FAST f=21 a=7 20.221569 2.177384 6 146
|
||||
FAST f=21 a=7 0.184278 2.177384 6 146
|
||||
FAST f=21 a=8 20.322357 2.179456 6 98
|
||||
FAST f=21 a=8 0.178458 2.179456 6 98
|
||||
FAST f=21 a=9 20.683912 2.174396 6 146
|
||||
FAST f=21 a=9 0.190829 2.174396 6 146
|
||||
FAST f=21 a=10 20.840865 2.174905 8 98
|
||||
FAST f=21 a=10 0.172515 2.174905 8 98
|
||||
FAST f=22 a=1 36.822827 2.181612 6 98
|
||||
FAST f=22 a=1 0.437389 2.181612 6 98
|
||||
FAST f=22 a=2 30.616902 2.183142 8 98
|
||||
FAST f=22 a=2 0.324284 2.183142 8 98
|
||||
FAST f=22 a=3 28.472482 2.178130 8 98
|
||||
FAST f=22 a=3 0.236538 2.178130 8 98
|
||||
FAST f=22 a=4 25.847028 2.181878 8 98
|
||||
FAST f=22 a=4 0.263744 2.181878 8 98
|
||||
FAST f=22 a=5 27.095881 2.180775 8 98
|
||||
FAST f=22 a=5 0.24988 2.180775 8 98
|
||||
FAST f=22 a=6 25.939172 2.170916 8 98
|
||||
FAST f=22 a=6 0.240033 2.170916 8 98
|
||||
FAST f=22 a=7 27.064194 2.177849 8 98
|
||||
FAST f=22 a=7 0.242383 2.177849 8 98
|
||||
FAST f=22 a=8 25.140221 2.178216 8 98
|
||||
FAST f=22 a=8 0.237601 2.178216 8 98
|
||||
FAST f=22 a=9 25.505283 2.177455 6 146
|
||||
FAST f=22 a=9 0.223217 2.177455 6 146
|
||||
FAST f=22 a=10 24.529362 2.176705 6 98
|
||||
FAST f=22 a=10 0.222876 2.176705 6 98
|
||||
FAST f=23 a=1 39.127310 2.183006 6 98
|
||||
FAST f=23 a=1 0.417338 2.183006 6 98
|
||||
FAST f=23 a=2 32.468161 2.183524 6 98
|
||||
FAST f=23 a=2 0.351645 2.183524 6 98
|
||||
FAST f=23 a=3 31.577620 2.172604 6 98
|
||||
FAST f=23 a=3 0.319659 2.172604 6 98
|
||||
FAST f=23 a=4 30.129247 2.183932 6 98
|
||||
FAST f=23 a=4 0.307239 2.183932 6 98
|
||||
FAST f=23 a=5 29.103376 2.183529 6 146
|
||||
FAST f=23 a=5 0.285533 2.183529 6 146
|
||||
FAST f=23 a=6 29.776045 2.174367 8 98
|
||||
FAST f=23 a=6 0.276846 2.174367 8 98
|
||||
FAST f=23 a=7 28.940407 2.178022 6 146
|
||||
FAST f=23 a=7 0.274082 2.178022 6 146
|
||||
FAST f=23 a=8 29.256009 2.179462 6 98
|
||||
FAST f=23 a=8 0.26949 2.179462 6 98
|
||||
FAST f=23 a=9 29.347312 2.170407 8 98
|
||||
FAST f=23 a=9 0.265034 2.170407 8 98
|
||||
FAST f=23 a=10 29.140081 2.171762 8 98
|
||||
FAST f=23 a=10 0.259183 2.171762 8 98
|
||||
FAST f=24 a=1 44.871179 2.182115 6 98
|
||||
FAST f=24 a=1 0.509433 2.182115 6 98
|
||||
FAST f=24 a=2 38.694867 2.180549 8 98
|
||||
FAST f=24 a=2 0.406695 2.180549 8 98
|
||||
FAST f=24 a=3 38.363769 2.172821 8 98
|
||||
FAST f=24 a=3 0.359581 2.172821 8 98
|
||||
FAST f=24 a=4 36.580797 2.184142 8 98
|
||||
FAST f=24 a=4 0.340614 2.184142 8 98
|
||||
FAST f=24 a=5 33.125701 2.183301 8 98
|
||||
FAST f=24 a=5 0.324874 2.183301 8 98
|
||||
FAST f=24 a=6 34.776068 2.173019 6 146
|
||||
FAST f=24 a=6 0.340397 2.173019 6 146
|
||||
FAST f=24 a=7 34.417625 2.176561 6 146
|
||||
FAST f=24 a=7 0.308223 2.176561 6 146
|
||||
FAST f=24 a=8 35.470291 2.182161 6 98
|
||||
FAST f=24 a=8 0.307724 2.182161 6 98
|
||||
FAST f=24 a=9 34.927252 2.172682 6 146
|
||||
FAST f=24 a=9 0.300598 2.172682 6 146
|
||||
FAST f=24 a=10 33.238355 2.173395 6 98
|
||||
FAST f=24 a=10 0.249916 2.173395 6 98
|
||||
|
||||
|
||||
hg-manifest:
|
||||
NODICT 0.000004 1.866377
|
||||
RANDOM 0.696346 2.309436
|
||||
LEGACY 7.064527 2.506977
|
||||
COVER 876.312865 2.582528 8 434
|
||||
COVER 35.684533 2.582528 8 434
|
||||
FAST f=15 a=1 76.618201 2.404013 8 1202
|
||||
FAST f=15 a=1 0.700722 2.404013 8 1202
|
||||
FAST f=15 a=2 49.213058 2.409248 6 1826
|
||||
FAST f=15 a=2 0.473393 2.409248 6 1826
|
||||
FAST f=15 a=3 41.753197 2.409677 8 1490
|
||||
FAST f=15 a=3 0.336848 2.409677 8 1490
|
||||
FAST f=15 a=4 38.648295 2.407996 8 1538
|
||||
FAST f=15 a=4 0.283952 2.407996 8 1538
|
||||
FAST f=15 a=5 36.144936 2.402895 8 1874
|
||||
FAST f=15 a=5 0.270128 2.402895 8 1874
|
||||
FAST f=15 a=6 35.484675 2.394873 8 1586
|
||||
FAST f=15 a=6 0.251637 2.394873 8 1586
|
||||
FAST f=15 a=7 34.280599 2.397311 8 1778
|
||||
FAST f=15 a=7 0.23984 2.397311 8 1778
|
||||
FAST f=15 a=8 32.122572 2.396089 6 1490
|
||||
FAST f=15 a=8 0.251508 2.396089 6 1490
|
||||
FAST f=15 a=9 29.909842 2.390092 6 1970
|
||||
FAST f=15 a=9 0.251233 2.390092 6 1970
|
||||
FAST f=15 a=10 30.102938 2.400086 6 1682
|
||||
FAST f=15 a=10 0.23688 2.400086 6 1682
|
||||
FAST f=16 a=1 67.750401 2.475460 6 1346
|
||||
FAST f=16 a=1 0.796035 2.475460 6 1346
|
||||
FAST f=16 a=2 52.812027 2.480860 6 1730
|
||||
FAST f=16 a=2 0.480384 2.480860 6 1730
|
||||
FAST f=16 a=3 44.179259 2.469304 8 1970
|
||||
FAST f=16 a=3 0.332657 2.469304 8 1970
|
||||
FAST f=16 a=4 37.612728 2.478208 6 1970
|
||||
FAST f=16 a=4 0.32498 2.478208 6 1970
|
||||
FAST f=16 a=5 35.056222 2.475568 6 1298
|
||||
FAST f=16 a=5 0.302824 2.475568 6 1298
|
||||
FAST f=16 a=6 34.713012 2.486079 8 1730
|
||||
FAST f=16 a=6 0.24755 2.486079 8 1730
|
||||
FAST f=16 a=7 33.713687 2.477180 6 1682
|
||||
FAST f=16 a=7 0.280358 2.477180 6 1682
|
||||
FAST f=16 a=8 31.571412 2.475418 8 1538
|
||||
FAST f=16 a=8 0.241241 2.475418 8 1538
|
||||
FAST f=16 a=9 31.608069 2.478263 8 1922
|
||||
FAST f=16 a=9 0.241764 2.478263 8 1922
|
||||
FAST f=16 a=10 31.358002 2.472263 8 1442
|
||||
FAST f=16 a=10 0.221661 2.472263 8 1442
|
||||
FAST f=17 a=1 66.185775 2.536085 6 1346
|
||||
FAST f=17 a=1 0.713549 2.536085 6 1346
|
||||
FAST f=17 a=2 50.365000 2.546105 8 1298
|
||||
FAST f=17 a=2 0.467846 2.546105 8 1298
|
||||
FAST f=17 a=3 42.712843 2.536250 8 1298
|
||||
FAST f=17 a=3 0.34047 2.536250 8 1298
|
||||
FAST f=17 a=4 39.514227 2.535555 8 1442
|
||||
FAST f=17 a=4 0.302989 2.535555 8 1442
|
||||
FAST f=17 a=5 35.189292 2.524925 8 1202
|
||||
FAST f=17 a=5 0.273451 2.524925 8 1202
|
||||
FAST f=17 a=6 35.791683 2.523466 8 1202
|
||||
FAST f=17 a=6 0.268261 2.523466 8 1202
|
||||
FAST f=17 a=7 37.416136 2.526625 6 1010
|
||||
FAST f=17 a=7 0.277558 2.526625 6 1010
|
||||
FAST f=17 a=8 37.084707 2.533274 6 1250
|
||||
FAST f=17 a=8 0.285104 2.533274 6 1250
|
||||
FAST f=17 a=9 34.183814 2.532765 8 1298
|
||||
FAST f=17 a=9 0.235133 2.532765 8 1298
|
||||
FAST f=17 a=10 31.149235 2.528722 8 1346
|
||||
FAST f=17 a=10 0.232679 2.528722 8 1346
|
||||
FAST f=18 a=1 72.942176 2.559857 6 386
|
||||
FAST f=18 a=1 0.718618 2.559857 6 386
|
||||
FAST f=18 a=2 51.690440 2.559572 8 290
|
||||
FAST f=18 a=2 0.403978 2.559572 8 290
|
||||
FAST f=18 a=3 45.344908 2.561040 8 962
|
||||
FAST f=18 a=3 0.357205 2.561040 8 962
|
||||
FAST f=18 a=4 39.804522 2.558446 8 1010
|
||||
FAST f=18 a=4 0.310526 2.558446 8 1010
|
||||
FAST f=18 a=5 38.134888 2.561811 8 626
|
||||
FAST f=18 a=5 0.273743 2.561811 8 626
|
||||
FAST f=18 a=6 35.091890 2.555518 8 722
|
||||
FAST f=18 a=6 0.260135 2.555518 8 722
|
||||
FAST f=18 a=7 34.639523 2.562938 8 290
|
||||
FAST f=18 a=7 0.234294 2.562938 8 290
|
||||
FAST f=18 a=8 36.076431 2.563567 8 1586
|
||||
FAST f=18 a=8 0.274075 2.563567 8 1586
|
||||
FAST f=18 a=9 36.376433 2.560950 8 722
|
||||
FAST f=18 a=9 0.240106 2.560950 8 722
|
||||
FAST f=18 a=10 32.624790 2.559340 8 578
|
||||
FAST f=18 a=10 0.234704 2.559340 8 578
|
||||
FAST f=19 a=1 70.513761 2.572441 8 194
|
||||
FAST f=19 a=1 0.726112 2.572441 8 194
|
||||
FAST f=19 a=2 59.263032 2.574560 8 482
|
||||
FAST f=19 a=2 0.451554 2.574560 8 482
|
||||
FAST f=19 a=3 51.509594 2.571546 6 194
|
||||
FAST f=19 a=3 0.393014 2.571546 6 194
|
||||
FAST f=19 a=4 55.393906 2.573386 8 482
|
||||
FAST f=19 a=4 0.38819 2.573386 8 482
|
||||
FAST f=19 a=5 43.201736 2.567589 8 674
|
||||
FAST f=19 a=5 0.292155 2.567589 8 674
|
||||
FAST f=19 a=6 42.911687 2.572666 6 434
|
||||
FAST f=19 a=6 0.303988 2.572666 6 434
|
||||
FAST f=19 a=7 44.687591 2.573613 6 290
|
||||
FAST f=19 a=7 0.308721 2.573613 6 290
|
||||
FAST f=19 a=8 37.372868 2.571039 6 194
|
||||
FAST f=19 a=8 0.287137 2.571039 6 194
|
||||
FAST f=19 a=9 36.074230 2.566473 6 482
|
||||
FAST f=19 a=9 0.280721 2.566473 6 482
|
||||
FAST f=19 a=10 33.731720 2.570306 8 194
|
||||
FAST f=19 a=10 0.224073 2.570306 8 194
|
||||
FAST f=20 a=1 79.670634 2.581146 6 290
|
||||
FAST f=20 a=1 0.899986 2.581146 6 290
|
||||
FAST f=20 a=2 58.827141 2.579782 8 386
|
||||
FAST f=20 a=2 0.602288 2.579782 8 386
|
||||
FAST f=20 a=3 51.289004 2.579627 8 722
|
||||
FAST f=20 a=3 0.446091 2.579627 8 722
|
||||
FAST f=20 a=4 47.711068 2.581508 8 722
|
||||
FAST f=20 a=4 0.473007 2.581508 8 722
|
||||
FAST f=20 a=5 47.402929 2.578062 6 434
|
||||
FAST f=20 a=5 0.497131 2.578062 6 434
|
||||
FAST f=20 a=6 54.797102 2.577365 8 482
|
||||
FAST f=20 a=6 0.515061 2.577365 8 482
|
||||
FAST f=20 a=7 51.370877 2.583050 8 386
|
||||
FAST f=20 a=7 0.402878 2.583050 8 386
|
||||
FAST f=20 a=8 51.437931 2.574875 6 242
|
||||
FAST f=20 a=8 0.453094 2.574875 6 242
|
||||
FAST f=20 a=9 44.105456 2.576700 6 242
|
||||
FAST f=20 a=9 0.456633 2.576700 6 242
|
||||
FAST f=20 a=10 44.447580 2.578305 8 338
|
||||
FAST f=20 a=10 0.409121 2.578305 8 338
|
||||
FAST f=21 a=1 113.031686 2.582449 6 242
|
||||
FAST f=21 a=1 1.456971 2.582449 6 242
|
||||
FAST f=21 a=2 97.700932 2.582124 8 194
|
||||
FAST f=21 a=2 1.072078 2.582124 8 194
|
||||
FAST f=21 a=3 96.563648 2.585479 8 434
|
||||
FAST f=21 a=3 0.949528 2.585479 8 434
|
||||
FAST f=21 a=4 90.597813 2.582366 6 386
|
||||
FAST f=21 a=4 0.76944 2.582366 6 386
|
||||
FAST f=21 a=5 86.815980 2.579043 8 434
|
||||
FAST f=21 a=5 0.858167 2.579043 8 434
|
||||
FAST f=21 a=6 91.235820 2.578378 8 530
|
||||
FAST f=21 a=6 0.684274 2.578378 8 530
|
||||
FAST f=21 a=7 84.392788 2.581243 8 386
|
||||
FAST f=21 a=7 0.814386 2.581243 8 386
|
||||
FAST f=21 a=8 82.052310 2.582547 8 338
|
||||
FAST f=21 a=8 0.822633 2.582547 8 338
|
||||
FAST f=21 a=9 74.696074 2.579319 8 194
|
||||
FAST f=21 a=9 0.811028 2.579319 8 194
|
||||
FAST f=21 a=10 76.211170 2.578766 8 290
|
||||
FAST f=21 a=10 0.809715 2.578766 8 290
|
||||
FAST f=22 a=1 138.976871 2.580478 8 194
|
||||
FAST f=22 a=1 1.748932 2.580478 8 194
|
||||
FAST f=22 a=2 120.164097 2.583633 8 386
|
||||
FAST f=22 a=2 1.333239 2.583633 8 386
|
||||
FAST f=22 a=3 111.986474 2.582566 6 194
|
||||
FAST f=22 a=3 1.305734 2.582566 6 194
|
||||
FAST f=22 a=4 108.548148 2.583068 6 194
|
||||
FAST f=22 a=4 1.314026 2.583068 6 194
|
||||
FAST f=22 a=5 103.173017 2.583495 6 290
|
||||
FAST f=22 a=5 1.228664 2.583495 6 290
|
||||
FAST f=22 a=6 108.421262 2.582349 8 530
|
||||
FAST f=22 a=6 1.076773 2.582349 8 530
|
||||
FAST f=22 a=7 103.284127 2.581022 8 386
|
||||
FAST f=22 a=7 1.112117 2.581022 8 386
|
||||
FAST f=22 a=8 96.330279 2.581073 8 290
|
||||
FAST f=22 a=8 1.109303 2.581073 8 290
|
||||
FAST f=22 a=9 97.651348 2.580075 6 194
|
||||
FAST f=22 a=9 0.933032 2.580075 6 194
|
||||
FAST f=22 a=10 101.660621 2.584886 8 194
|
||||
FAST f=22 a=10 0.796823 2.584886 8 194
|
||||
FAST f=23 a=1 159.322978 2.581474 6 242
|
||||
FAST f=23 a=1 2.015878 2.581474 6 242
|
||||
FAST f=23 a=2 134.331775 2.581619 8 194
|
||||
FAST f=23 a=2 1.545845 2.581619 8 194
|
||||
FAST f=23 a=3 127.724552 2.579888 6 338
|
||||
FAST f=23 a=3 1.444496 2.579888 6 338
|
||||
FAST f=23 a=4 126.077675 2.578137 6 242
|
||||
FAST f=23 a=4 1.364394 2.578137 6 242
|
||||
FAST f=23 a=5 124.914027 2.580843 8 338
|
||||
FAST f=23 a=5 1.116059 2.580843 8 338
|
||||
FAST f=23 a=6 122.874153 2.577637 6 338
|
||||
FAST f=23 a=6 1.164584 2.577637 6 338
|
||||
FAST f=23 a=7 123.099257 2.582715 6 386
|
||||
FAST f=23 a=7 1.354042 2.582715 6 386
|
||||
FAST f=23 a=8 122.026753 2.577681 8 194
|
||||
FAST f=23 a=8 1.210966 2.577681 8 194
|
||||
FAST f=23 a=9 121.164312 2.584599 6 290
|
||||
FAST f=23 a=9 1.174859 2.584599 6 290
|
||||
FAST f=23 a=10 117.462222 2.580358 8 194
|
||||
FAST f=23 a=10 1.075258 2.580358 8 194
|
||||
FAST f=24 a=1 169.539659 2.581642 6 194
|
||||
FAST f=24 a=1 1.916804 2.581642 6 194
|
||||
FAST f=24 a=2 160.539270 2.580421 6 290
|
||||
FAST f=24 a=2 1.71087 2.580421 6 290
|
||||
FAST f=24 a=3 155.455874 2.580449 6 242
|
||||
FAST f=24 a=3 1.60307 2.580449 6 242
|
||||
FAST f=24 a=4 147.630320 2.582953 6 338
|
||||
FAST f=24 a=4 1.396364 2.582953 6 338
|
||||
FAST f=24 a=5 133.767428 2.580589 6 290
|
||||
FAST f=24 a=5 1.19933 2.580589 6 290
|
||||
FAST f=24 a=6 146.437535 2.579453 8 194
|
||||
FAST f=24 a=6 1.385405 2.579453 8 194
|
||||
FAST f=24 a=7 147.227507 2.584155 8 386
|
||||
FAST f=24 a=7 1.48942 2.584155 8 386
|
||||
FAST f=24 a=8 138.005773 2.584115 8 194
|
||||
FAST f=24 a=8 1.352 2.584115 8 194
|
||||
FAST f=24 a=9 141.442625 2.582902 8 290
|
||||
FAST f=24 a=9 1.39647 2.582902 8 290
|
||||
FAST f=24 a=10 142.157446 2.582701 8 434
|
||||
FAST f=24 a=10 1.498889 2.582701 8 434
|
||||
@@ -0,0 +1,442 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include <ctype.h>
|
||||
#include <time.h>
|
||||
#include "random.h"
|
||||
#include "dictBuilder.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "io.h"
|
||||
#include "util.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAY("Error %i : ", error); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY("\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static const unsigned g_defaultMaxDictSize = 110 KB;
|
||||
#define DEFAULT_CLEVEL 3
|
||||
#define DEFAULT_DISPLAYLEVEL 2
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Struct
|
||||
***************************************/
|
||||
typedef struct {
|
||||
const void* dictBuffer;
|
||||
size_t dictSize;
|
||||
} dictInfo;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Dictionary related operations
|
||||
***************************************/
|
||||
/** createDictFromFiles() :
|
||||
* Based on type of param given, train dictionary using the corresponding algorithm
|
||||
* @return dictInfo containing dictionary buffer and dictionary size
|
||||
*/
|
||||
dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
|
||||
ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams,
|
||||
ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) {
|
||||
unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel :
|
||||
coverParams ? coverParams->zParams.notificationLevel :
|
||||
legacyParams ? legacyParams->zParams.notificationLevel :
|
||||
fastParams ? fastParams->zParams.notificationLevel :
|
||||
DEFAULT_DISPLAYLEVEL; /* no dict */
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
|
||||
dictInfo* dInfo = NULL;
|
||||
|
||||
/* Checks */
|
||||
if (!dictBuffer)
|
||||
EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */
|
||||
|
||||
{ size_t dictSize;
|
||||
if(randomParams) {
|
||||
dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *randomParams);
|
||||
}else if(coverParams) {
|
||||
/* Run the optimize version if either k or d is not provided */
|
||||
if (!coverParams->d || !coverParams->k){
|
||||
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, coverParams);
|
||||
} else {
|
||||
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *coverParams);
|
||||
}
|
||||
} else if(legacyParams) {
|
||||
dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *legacyParams);
|
||||
} else if(fastParams) {
|
||||
/* Run the optimize version if either k or d is not provided */
|
||||
if (!fastParams->d || !fastParams->k) {
|
||||
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, fastParams);
|
||||
} else {
|
||||
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *fastParams);
|
||||
}
|
||||
} else {
|
||||
dictSize = 0;
|
||||
}
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
free(dictBuffer);
|
||||
return dInfo;
|
||||
}
|
||||
dInfo = (dictInfo *)malloc(sizeof(dictInfo));
|
||||
dInfo->dictBuffer = dictBuffer;
|
||||
dInfo->dictSize = dictSize;
|
||||
}
|
||||
return dInfo;
|
||||
}
|
||||
|
||||
|
||||
/** compressWithDict() :
|
||||
* Compress samples from sample buffer given dicionary stored on dictionary buffer and compression level
|
||||
* @return compression ratio
|
||||
*/
|
||||
double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLevel, int displayLevel) {
|
||||
/* Local variables */
|
||||
size_t totalCompressedSize = 0;
|
||||
size_t totalOriginalSize = 0;
|
||||
const unsigned hasDict = dInfo->dictSize > 0 ? 1 : 0;
|
||||
double cRatio;
|
||||
size_t dstCapacity;
|
||||
int i;
|
||||
|
||||
/* Pointers */
|
||||
ZSTD_CDict *cdict = NULL;
|
||||
ZSTD_CCtx* cctx = NULL;
|
||||
size_t *offsets = NULL;
|
||||
void* dst = NULL;
|
||||
|
||||
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||
{
|
||||
size_t maxSampleSize = 0;
|
||||
for (i = 0; i < srcInfo->nbSamples; i++) {
|
||||
maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
|
||||
}
|
||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||
dst = malloc(dstCapacity);
|
||||
}
|
||||
|
||||
/* Calculate offset for each sample */
|
||||
offsets = (size_t *)malloc((srcInfo->nbSamples + 1) * sizeof(size_t));
|
||||
offsets[0] = 0;
|
||||
for (i = 1; i <= srcInfo->nbSamples; i++) {
|
||||
offsets[i] = offsets[i - 1] + srcInfo->samplesSizes[i - 1];
|
||||
}
|
||||
|
||||
/* Create the cctx */
|
||||
cctx = ZSTD_createCCtx();
|
||||
if(!cctx || !dst) {
|
||||
cRatio = -1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
/* Create CDict if there's a dictionary stored on buffer */
|
||||
if (hasDict) {
|
||||
cdict = ZSTD_createCDict(dInfo->dictBuffer, dInfo->dictSize, compressionLevel);
|
||||
if(!cdict) {
|
||||
cRatio = -1;
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
/* Compress each sample and sum their sizes*/
|
||||
const BYTE *const samples = (const BYTE *)srcInfo->srcBuffer;
|
||||
for (i = 0; i < srcInfo->nbSamples; i++) {
|
||||
size_t compressedSize;
|
||||
if(hasDict) {
|
||||
compressedSize = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, samples + offsets[i], srcInfo->samplesSizes[i], cdict);
|
||||
} else {
|
||||
compressedSize = ZSTD_compressCCtx(cctx, dst, dstCapacity,samples + offsets[i], srcInfo->samplesSizes[i], compressionLevel);
|
||||
}
|
||||
if (ZSTD_isError(compressedSize)) {
|
||||
cRatio = -1;
|
||||
goto _cleanup;
|
||||
}
|
||||
totalCompressedSize += compressedSize;
|
||||
}
|
||||
|
||||
/* Sum orignal sizes */
|
||||
for (i = 0; i<srcInfo->nbSamples; i++) {
|
||||
totalOriginalSize += srcInfo->samplesSizes[i];
|
||||
}
|
||||
|
||||
/* Calculate compression ratio */
|
||||
DISPLAYLEVEL(2, "original size is %lu\n", totalOriginalSize);
|
||||
DISPLAYLEVEL(2, "compressed size is %lu\n", totalCompressedSize);
|
||||
cRatio = (double)totalOriginalSize/(double)totalCompressedSize;
|
||||
|
||||
_cleanup:
|
||||
free(dst);
|
||||
free(offsets);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeCDict(cdict);
|
||||
return cRatio;
|
||||
}
|
||||
|
||||
|
||||
/** FreeDictInfo() :
|
||||
* Free memory allocated for dictInfo
|
||||
*/
|
||||
void freeDictInfo(dictInfo* info) {
|
||||
if (!info) return;
|
||||
if (info->dictBuffer) free((void*)(info->dictBuffer));
|
||||
free(info);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* Benchmarking functions
|
||||
**********************************************************/
|
||||
/** benchmarkDictBuilder() :
|
||||
* Measure how long a dictionary builder takes and compression ratio with the dictionary built
|
||||
* @return 0 if benchmark successfully, 1 otherwise
|
||||
*/
|
||||
int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam,
|
||||
ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam,
|
||||
ZDICT_fastCover_params_t *fastParam) {
|
||||
/* Local variables */
|
||||
const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel :
|
||||
coverParam ? coverParam->zParams.notificationLevel :
|
||||
legacyParam ? legacyParam->zParams.notificationLevel :
|
||||
fastParam ? fastParam->zParams.notificationLevel:
|
||||
DEFAULT_DISPLAYLEVEL; /* no dict */
|
||||
const char* name = randomParam ? "RANDOM" :
|
||||
coverParam ? "COVER" :
|
||||
legacyParam ? "LEGACY" :
|
||||
fastParam ? "FAST":
|
||||
"NODICT"; /* no dict */
|
||||
const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel :
|
||||
coverParam ? coverParam->zParams.compressionLevel :
|
||||
legacyParam ? legacyParam->zParams.compressionLevel :
|
||||
fastParam ? fastParam->zParams.compressionLevel:
|
||||
DEFAULT_CLEVEL; /* no dict */
|
||||
int result = 0;
|
||||
|
||||
/* Calculate speed */
|
||||
const UTIL_time_t begin = UTIL_getTime();
|
||||
dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam);
|
||||
const U64 timeMicro = UTIL_clockSpanMicro(begin);
|
||||
const double timeSec = timeMicro / (double)SEC_TO_MICRO;
|
||||
if (!dInfo) {
|
||||
DISPLAYLEVEL(1, "%s does not train successfully\n", name);
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
DISPLAYLEVEL(1, "%s took %f seconds to execute \n", name, timeSec);
|
||||
|
||||
/* Calculate compression ratio */
|
||||
const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel);
|
||||
if (cRatio < 0) {
|
||||
DISPLAYLEVEL(1, "Compressing with %s dictionary does not work\n", name);
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
|
||||
}
|
||||
DISPLAYLEVEL(1, "Compression ratio with %s dictionary is %f\n", name, cRatio);
|
||||
|
||||
_cleanup:
|
||||
freeDictInfo(dInfo);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int main(int argCount, const char* argv[])
|
||||
{
|
||||
const int displayLevel = DEFAULT_DISPLAYLEVEL;
|
||||
const char* programName = argv[0];
|
||||
int result = 0;
|
||||
|
||||
/* Initialize arguments to default values */
|
||||
unsigned k = 200;
|
||||
unsigned d = 8;
|
||||
unsigned f;
|
||||
unsigned accel;
|
||||
unsigned i;
|
||||
const unsigned cLevel = DEFAULT_CLEVEL;
|
||||
const unsigned dictID = 0;
|
||||
const unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
|
||||
/* Initialize table to store input files */
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
|
||||
unsigned filenameIdx = 0;
|
||||
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb = filenameIdx;
|
||||
const int followLinks = 0;
|
||||
const char** extendedFileList = NULL;
|
||||
|
||||
/* Parse arguments */
|
||||
for (i = 1; i < argCount; i++) {
|
||||
const char* argument = argv[i];
|
||||
if (longCommandWArg(&argument, "in=")) {
|
||||
filenameTable[filenameIdx] = argument;
|
||||
filenameIdx++;
|
||||
continue;
|
||||
}
|
||||
DISPLAYLEVEL(1, "benchmark: Incorrect parameters\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Get the list of all files recursively (because followLinks==0)*/
|
||||
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
|
||||
&fileNamesNb, followLinks);
|
||||
if (extendedFileList) {
|
||||
unsigned u;
|
||||
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
|
||||
free((void*)filenameTable);
|
||||
filenameTable = extendedFileList;
|
||||
filenameIdx = fileNamesNb;
|
||||
}
|
||||
|
||||
/* get sampleInfo */
|
||||
size_t blockSize = 0;
|
||||
sampleInfo* srcInfo= getSampleInfo(filenameTable,
|
||||
filenameIdx, blockSize, maxDictSize, displayLevel);
|
||||
|
||||
/* set up zParams */
|
||||
ZDICT_params_t zParams;
|
||||
zParams.compressionLevel = cLevel;
|
||||
zParams.notificationLevel = displayLevel;
|
||||
zParams.dictID = dictID;
|
||||
|
||||
/* with no dict */
|
||||
{
|
||||
const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL);
|
||||
if(noDictResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
/* for random */
|
||||
{
|
||||
ZDICT_random_params_t randomParam;
|
||||
randomParam.zParams = zParams;
|
||||
randomParam.k = k;
|
||||
const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL);
|
||||
DISPLAYLEVEL(2, "k=%u\n", randomParam.k);
|
||||
if(randomResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
/* for legacy */
|
||||
{
|
||||
ZDICT_legacy_params_t legacyParam;
|
||||
legacyParam.zParams = zParams;
|
||||
legacyParam.selectivityLevel = 9;
|
||||
const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL);
|
||||
DISPLAYLEVEL(2, "selectivityLevel=%u\n", legacyParam.selectivityLevel);
|
||||
if(legacyResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
/* for cover */
|
||||
{
|
||||
/* for cover (optimizing k and d) */
|
||||
ZDICT_cover_params_t coverParam;
|
||||
memset(&coverParam, 0, sizeof(coverParam));
|
||||
coverParam.zParams = zParams;
|
||||
coverParam.splitPoint = 1.0;
|
||||
coverParam.steps = 40;
|
||||
coverParam.nbThreads = 1;
|
||||
const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
|
||||
if(coverOptResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
/* for cover (with k and d provided) */
|
||||
const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
|
||||
if(coverResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/* for fastCover */
|
||||
for (f = 15; f < 25; f++){
|
||||
DISPLAYLEVEL(2, "current f is %u\n", f);
|
||||
for (accel = 1; accel < 11; accel++) {
|
||||
DISPLAYLEVEL(2, "current accel is %u\n", accel);
|
||||
/* for fastCover (optimizing k and d) */
|
||||
ZDICT_fastCover_params_t fastParam;
|
||||
memset(&fastParam, 0, sizeof(fastParam));
|
||||
fastParam.zParams = zParams;
|
||||
fastParam.f = f;
|
||||
fastParam.steps = 40;
|
||||
fastParam.nbThreads = 1;
|
||||
fastParam.accel = accel;
|
||||
const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
|
||||
if(fastOptResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
/* for fastCover (with k and d provided) */
|
||||
for (i = 0; i < 5; i++) {
|
||||
const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
|
||||
if(fastResult) {
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Free allocated memory */
|
||||
_cleanup:
|
||||
UTIL_freeFileList(extendedFileList, fileNamesBuf);
|
||||
freeSampleInfo(srcInfo);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
/* ZDICT_trainFromBuffer_legacy() :
|
||||
* issue : samplesBuffer need to be followed by a noisy guard band.
|
||||
* work around : duplicate the buffer, and add the noise */
|
||||
size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t params);
|
||||
@@ -0,0 +1,2 @@
|
||||
echo "Benchmark with in=../../lib/common"
|
||||
./benchmark in=../../../lib/common
|
||||
@@ -0,0 +1,54 @@
|
||||
ARG :=
|
||||
|
||||
CC ?= gcc
|
||||
CFLAGS ?= -O3 -g
|
||||
INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
|
||||
|
||||
IO_FILE := ../randomDictBuilder/io.c
|
||||
|
||||
TEST_INPUT := ../../../lib
|
||||
TEST_OUTPUT := fastCoverDict
|
||||
|
||||
all: main run clean
|
||||
|
||||
.PHONY: test
|
||||
test: main testrun testshell clean
|
||||
|
||||
.PHONY: run
|
||||
run:
|
||||
echo "Building a fastCover dictionary with given arguments"
|
||||
./main $(ARG)
|
||||
|
||||
main: main.o io.o fastCover.o libzstd.a
|
||||
$(CC) $(CFLAGS) main.o io.o fastCover.o libzstd.a -o main
|
||||
|
||||
main.o: main.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c main.c
|
||||
|
||||
fastCover.o: fastCover.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c fastCover.c
|
||||
|
||||
io.o: $(IO_FILE)
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
|
||||
|
||||
libzstd.a:
|
||||
$(MAKE) MOREFLAGS=-g -C ../../../lib libzstd.a
|
||||
mv ../../../lib/libzstd.a .
|
||||
|
||||
.PHONY: testrun
|
||||
testrun: main
|
||||
echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) "
|
||||
./main in=$(TEST_INPUT) out=$(TEST_OUTPUT)
|
||||
zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q
|
||||
rm -f $(TEST_OUTPUT)
|
||||
|
||||
.PHONY: testshell
|
||||
testshell: test.sh
|
||||
sh test.sh
|
||||
echo "Finish running test.sh"
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f *.o main libzstd.a
|
||||
$(MAKE) -C ../../../lib clean
|
||||
echo "Cleaning is completed"
|
||||
@@ -0,0 +1,24 @@
|
||||
FastCover Dictionary Builder
|
||||
|
||||
### Permitted Arguments:
|
||||
Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
|
||||
Output Dictionary (out=dictName): if not provided, default to fastCoverDict
|
||||
Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0
|
||||
Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB
|
||||
Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200
|
||||
Size of Dmer (d=#): either 6 or 8; if not provided, default to 8
|
||||
Number of steps (steps=#): positive number, if not provided, default to 32
|
||||
Percentage of samples used for training(split=#): positive number; if not provided, default to 100
|
||||
|
||||
|
||||
###Running Test:
|
||||
make test
|
||||
|
||||
|
||||
###Usage:
|
||||
To build a FASTCOVER dictionary with the provided arguments: make ARG= followed by arguments
|
||||
If k or d is not provided, the optimize version of FASTCOVER is run.
|
||||
|
||||
### Examples:
|
||||
make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520"
|
||||
make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
|
||||
@@ -0,0 +1,809 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "fastCover.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
#define FASTCOVER_MAX_F 32
|
||||
#define DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
static int g_displayLevel = 2;
|
||||
#define DISPLAY(...) \
|
||||
{ \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fflush(stderr); \
|
||||
}
|
||||
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
||||
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
}
|
||||
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Hash Functions
|
||||
***************************************/
|
||||
static const U64 prime6bytes = 227718039650203ULL;
|
||||
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
||||
|
||||
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
||||
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
||||
|
||||
|
||||
/**
|
||||
* Hash the d-byte value pointed to by p and mod 2^f
|
||||
*/
|
||||
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
|
||||
if (d == 6) {
|
||||
return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
|
||||
}
|
||||
return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Context
|
||||
***************************************/
|
||||
typedef struct {
|
||||
const BYTE *samples;
|
||||
size_t *offsets;
|
||||
const size_t *samplesSizes;
|
||||
size_t nbSamples;
|
||||
size_t nbTrainSamples;
|
||||
size_t nbTestSamples;
|
||||
size_t nbDmers;
|
||||
U32 *freqs;
|
||||
U16 *segmentFreqs;
|
||||
unsigned d;
|
||||
} FASTCOVER_ctx_t;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
***************************************/
|
||||
/**
|
||||
* Returns the sum of the sample sizes.
|
||||
*/
|
||||
static size_t FASTCOVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
||||
size_t sum = 0;
|
||||
unsigned i;
|
||||
for (i = 0; i < nbSamples; ++i) {
|
||||
sum += samplesSizes[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* fast functions
|
||||
***************************************/
|
||||
/**
|
||||
* A segment is a range in the source as well as the score of the segment.
|
||||
*/
|
||||
typedef struct {
|
||||
U32 begin;
|
||||
U32 end;
|
||||
U32 score;
|
||||
} FASTCOVER_segment_t;
|
||||
|
||||
|
||||
/**
|
||||
* Selects the best segment in an epoch.
|
||||
* Segments of are scored according to the function:
|
||||
*
|
||||
* Let F(d) be the frequency of all dmers with hash value d.
|
||||
* Let S_i be hash value of the dmer at position i of segment S which has length k.
|
||||
*
|
||||
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||
*
|
||||
* Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2.
|
||||
*/
|
||||
static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
||||
U32 *freqs, U32 begin,U32 end,
|
||||
ZDICT_fastCover_params_t parameters) {
|
||||
/* Constants */
|
||||
const U32 k = parameters.k;
|
||||
const U32 d = parameters.d;
|
||||
const U32 dmersInK = k - d + 1;
|
||||
/* Try each segment (activeSegment) and save the best (bestSegment) */
|
||||
FASTCOVER_segment_t bestSegment = {0, 0, 0};
|
||||
FASTCOVER_segment_t activeSegment;
|
||||
/* Reset the activeDmers in the segment */
|
||||
/* The activeSegment starts at the beginning of the epoch. */
|
||||
activeSegment.begin = begin;
|
||||
activeSegment.end = begin;
|
||||
activeSegment.score = 0;
|
||||
{
|
||||
/* Slide the activeSegment through the whole epoch.
|
||||
* Save the best segment in bestSegment.
|
||||
*/
|
||||
while (activeSegment.end < end) {
|
||||
/* Get hash value of current dmer */
|
||||
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
|
||||
/* Add frequency of this index to score if this is the first occurence of index in active segment */
|
||||
if (ctx->segmentFreqs[index] == 0) {
|
||||
activeSegment.score += freqs[index];
|
||||
}
|
||||
ctx->segmentFreqs[index] += 1;
|
||||
/* Increment end of segment */
|
||||
activeSegment.end += 1;
|
||||
/* If the window is now too large, drop the first position */
|
||||
if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
|
||||
/* Get hash value of the dmer to be eliminated from active segment */
|
||||
const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d);
|
||||
ctx->segmentFreqs[delIndex] -= 1;
|
||||
/* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
|
||||
if (ctx->segmentFreqs[delIndex] == 0) {
|
||||
activeSegment.score -= freqs[delIndex];
|
||||
}
|
||||
/* Increment start of segment */
|
||||
activeSegment.begin += 1;
|
||||
}
|
||||
/* If this segment is the best so far save it */
|
||||
if (activeSegment.score > bestSegment.score) {
|
||||
bestSegment = activeSegment;
|
||||
}
|
||||
}
|
||||
/* Zero out rest of segmentFreqs array */
|
||||
while (activeSegment.begin < end) {
|
||||
const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d);
|
||||
ctx->segmentFreqs[delIndex] -= 1;
|
||||
activeSegment.begin += 1;
|
||||
}
|
||||
}
|
||||
{
|
||||
/* Trim off the zero frequency head and tail from the segment. */
|
||||
U32 newBegin = bestSegment.end;
|
||||
U32 newEnd = bestSegment.begin;
|
||||
U32 pos;
|
||||
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
|
||||
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
|
||||
U32 freq = freqs[index];
|
||||
if (freq != 0) {
|
||||
newBegin = MIN(newBegin, pos);
|
||||
newEnd = pos + 1;
|
||||
}
|
||||
}
|
||||
bestSegment.begin = newBegin;
|
||||
bestSegment.end = newEnd;
|
||||
}
|
||||
{
|
||||
/* Zero the frequency of hash value of each dmer covered by the chosen segment. */
|
||||
U32 pos;
|
||||
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
|
||||
const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
|
||||
freqs[i] = 0;
|
||||
}
|
||||
}
|
||||
return bestSegment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the validity of the parameters.
|
||||
* Returns non-zero if the parameters are valid and 0 otherwise.
|
||||
*/
|
||||
static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters,
|
||||
size_t maxDictSize) {
|
||||
/* k, d, and f are required parameters */
|
||||
if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) {
|
||||
return 0;
|
||||
}
|
||||
/* d has to be 6 or 8 */
|
||||
if (parameters.d != 6 && parameters.d != 8) {
|
||||
return 0;
|
||||
}
|
||||
/* 0 < f <= FASTCOVER_MAX_F */
|
||||
if (parameters.f > FASTCOVER_MAX_F) {
|
||||
return 0;
|
||||
}
|
||||
/* k <= maxDictSize */
|
||||
if (parameters.k > maxDictSize) {
|
||||
return 0;
|
||||
}
|
||||
/* d <= k */
|
||||
if (parameters.d > parameters.k) {
|
||||
return 0;
|
||||
}
|
||||
/* 0 < splitPoint <= 1 */
|
||||
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) {
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Clean up a context initialized with `FASTCOVER_ctx_init()`.
|
||||
*/
|
||||
static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
|
||||
if (!ctx) {
|
||||
return;
|
||||
}
|
||||
if (ctx->segmentFreqs) {
|
||||
free(ctx->segmentFreqs);
|
||||
ctx->segmentFreqs = NULL;
|
||||
}
|
||||
if (ctx->freqs) {
|
||||
free(ctx->freqs);
|
||||
ctx->freqs = NULL;
|
||||
}
|
||||
if (ctx->offsets) {
|
||||
free(ctx->offsets);
|
||||
ctx->offsets = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate for frequency of hash value of each dmer in ctx->samples
|
||||
*/
|
||||
static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){
|
||||
size_t start; /* start of current dmer */
|
||||
for (unsigned i = 0; i < ctx->nbTrainSamples; i++) {
|
||||
size_t currSampleStart = ctx->offsets[i];
|
||||
size_t currSampleEnd = ctx->offsets[i+1];
|
||||
start = currSampleStart;
|
||||
while (start + ctx->d <= currSampleEnd) {
|
||||
const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d);
|
||||
freqs[dmerIndex]++;
|
||||
start++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare a context for dictionary building.
|
||||
* The context is only dependent on the parameter `d` and can used multiple
|
||||
* times.
|
||||
* Returns 1 on success or zero on error.
|
||||
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
|
||||
*/
|
||||
static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
unsigned d, double splitPoint, unsigned f) {
|
||||
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||
const size_t totalSamplesSize = FASTCOVER_sum(samplesSizes, nbSamples);
|
||||
/* Split samples into testing and training sets */
|
||||
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
|
||||
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
|
||||
const size_t trainingSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
|
||||
const size_t testSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
|
||||
/* Checks */
|
||||
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
|
||||
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
|
||||
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
|
||||
(U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
|
||||
return 0;
|
||||
}
|
||||
/* Check if there are at least 5 training samples */
|
||||
if (nbTrainSamples < 5) {
|
||||
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
|
||||
return 0;
|
||||
}
|
||||
/* Check if there's testing sample */
|
||||
if (nbTestSamples < 1) {
|
||||
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
|
||||
return 0;
|
||||
}
|
||||
/* Zero the context */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
|
||||
(U32)trainingSamplesSize);
|
||||
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
|
||||
(U32)testSamplesSize);
|
||||
|
||||
ctx->samples = samples;
|
||||
ctx->samplesSizes = samplesSizes;
|
||||
ctx->nbSamples = nbSamples;
|
||||
ctx->nbTrainSamples = nbTrainSamples;
|
||||
ctx->nbTestSamples = nbTestSamples;
|
||||
ctx->nbDmers = trainingSamplesSize - d + 1;
|
||||
ctx->d = d;
|
||||
|
||||
/* The offsets of each file */
|
||||
ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
|
||||
if (!ctx->offsets) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
|
||||
FASTCOVER_ctx_destroy(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Fill offsets from the samplesSizes */
|
||||
{
|
||||
U32 i;
|
||||
ctx->offsets[0] = 0;
|
||||
for (i = 1; i <= nbSamples; ++i) {
|
||||
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
|
||||
}
|
||||
}
|
||||
|
||||
/* Initialize frequency array of size 2^f */
|
||||
ctx->freqs = (U32 *)calloc((1 << f), sizeof(U32));
|
||||
ctx->segmentFreqs = (U16 *)calloc((1 << f), sizeof(U16));
|
||||
DISPLAYLEVEL(2, "Computing frequencies\n");
|
||||
FASTCOVER_computeFrequency(ctx->freqs, f, ctx);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Given the prepared context build the dictionary.
|
||||
*/
|
||||
static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
|
||||
void *dictBuffer,
|
||||
size_t dictBufferCapacity,
|
||||
ZDICT_fastCover_params_t parameters){
|
||||
BYTE *const dict = (BYTE *)dictBuffer;
|
||||
size_t tail = dictBufferCapacity;
|
||||
/* Divide the data up into epochs of equal size.
|
||||
* We will select at least one segment from each epoch.
|
||||
*/
|
||||
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
|
||||
const U32 epochSize = (U32)(ctx->nbDmers / epochs);
|
||||
size_t epoch;
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
|
||||
epochSize);
|
||||
/* Loop through the epochs until there are no more segments or the dictionary
|
||||
* is full.
|
||||
*/
|
||||
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
|
||||
const U32 epochBegin = (U32)(epoch * epochSize);
|
||||
const U32 epochEnd = epochBegin + epochSize;
|
||||
size_t segmentSize;
|
||||
/* Select a segment */
|
||||
FASTCOVER_segment_t segment = FASTCOVER_selectSegment(
|
||||
ctx, freqs, epochBegin, epochEnd, parameters);
|
||||
|
||||
/* If the segment covers no dmers, then we are out of content */
|
||||
if (segment.score == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* Trim the segment if necessary and if it is too small then we are done */
|
||||
segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
|
||||
if (segmentSize < parameters.d) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* We fill the dictionary from the back to allow the best segments to be
|
||||
* referenced with the smallest offsets.
|
||||
*/
|
||||
tail -= segmentSize;
|
||||
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
|
||||
DISPLAYUPDATE(
|
||||
2, "\r%u%% ",
|
||||
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
return tail;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* FASTCOVER_best_t is used for two purposes:
|
||||
* 1. Synchronizing threads.
|
||||
* 2. Saving the best parameters and dictionary.
|
||||
*
|
||||
* All of the methods except FASTCOVER_best_init() are thread safe if zstd is
|
||||
* compiled with multithreaded support.
|
||||
*/
|
||||
typedef struct fast_best_s {
|
||||
ZSTD_pthread_mutex_t mutex;
|
||||
ZSTD_pthread_cond_t cond;
|
||||
size_t liveJobs;
|
||||
void *dict;
|
||||
size_t dictSize;
|
||||
ZDICT_fastCover_params_t parameters;
|
||||
size_t compressedSize;
|
||||
} FASTCOVER_best_t;
|
||||
|
||||
/**
|
||||
* Initialize the `FASTCOVER_best_t`.
|
||||
*/
|
||||
static void FASTCOVER_best_init(FASTCOVER_best_t *best) {
|
||||
if (best==NULL) return; /* compatible with init on NULL */
|
||||
(void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&best->cond, NULL);
|
||||
best->liveJobs = 0;
|
||||
best->dict = NULL;
|
||||
best->dictSize = 0;
|
||||
best->compressedSize = (size_t)-1;
|
||||
memset(&best->parameters, 0, sizeof(best->parameters));
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until liveJobs == 0.
|
||||
*/
|
||||
static void FASTCOVER_best_wait(FASTCOVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
while (best->liveJobs != 0) {
|
||||
ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call FASTCOVER_best_wait() and then destroy the FASTCOVER_best_t.
|
||||
*/
|
||||
static void FASTCOVER_best_destroy(FASTCOVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
FASTCOVER_best_wait(best);
|
||||
if (best->dict) {
|
||||
free(best->dict);
|
||||
}
|
||||
ZSTD_pthread_mutex_destroy(&best->mutex);
|
||||
ZSTD_pthread_cond_destroy(&best->cond);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a thread is about to be launched.
|
||||
* Increments liveJobs.
|
||||
*/
|
||||
static void FASTCOVER_best_start(FASTCOVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
++best->liveJobs;
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a thread finishes executing, both on error or success.
|
||||
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
||||
* If this dictionary is the best so far save it and its parameters.
|
||||
*/
|
||||
static void FASTCOVER_best_finish(FASTCOVER_best_t *best, size_t compressedSize,
|
||||
ZDICT_fastCover_params_t parameters, void *dict,
|
||||
size_t dictSize) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
{
|
||||
size_t liveJobs;
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
--best->liveJobs;
|
||||
liveJobs = best->liveJobs;
|
||||
/* If the new dictionary is better */
|
||||
if (compressedSize < best->compressedSize) {
|
||||
/* Allocate space if necessary */
|
||||
if (!best->dict || best->dictSize < dictSize) {
|
||||
if (best->dict) {
|
||||
free(best->dict);
|
||||
}
|
||||
best->dict = malloc(dictSize);
|
||||
if (!best->dict) {
|
||||
best->compressedSize = ERROR(GENERIC);
|
||||
best->dictSize = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
/* Save the dictionary, parameters, and size */
|
||||
memcpy(best->dict, dict, dictSize);
|
||||
best->dictSize = dictSize;
|
||||
best->parameters = parameters;
|
||||
best->compressedSize = compressedSize;
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
if (liveJobs == 0) {
|
||||
ZSTD_pthread_cond_broadcast(&best->cond);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parameters for FASTCOVER_tryParameters().
|
||||
*/
|
||||
typedef struct FASTCOVER_tryParameters_data_s {
|
||||
const FASTCOVER_ctx_t *ctx;
|
||||
FASTCOVER_best_t *best;
|
||||
size_t dictBufferCapacity;
|
||||
ZDICT_fastCover_params_t parameters;
|
||||
} FASTCOVER_tryParameters_data_t;
|
||||
|
||||
/**
|
||||
* Tries a set of parameters and updates the FASTCOVER_best_t with the results.
|
||||
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||
*/
|
||||
static void FASTCOVER_tryParameters(void *opaque) {
|
||||
/* Save parameters as local variables */
|
||||
FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
|
||||
const FASTCOVER_ctx_t *const ctx = data->ctx;
|
||||
const ZDICT_fastCover_params_t parameters = data->parameters;
|
||||
size_t dictBufferCapacity = data->dictBufferCapacity;
|
||||
size_t totalCompressedSize = ERROR(GENERIC);
|
||||
/* Allocate space for hash table, dict, and freqs */
|
||||
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
|
||||
U32 *freqs = (U32*) malloc((1 << parameters.f) * sizeof(U32));
|
||||
if (!dict || !freqs) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
|
||||
goto _cleanup;
|
||||
}
|
||||
/* Copy the frequencies because we need to modify them */
|
||||
memcpy(freqs, ctx->freqs, (1 << parameters.f) * sizeof(U32));
|
||||
/* Build the dictionary */
|
||||
{
|
||||
const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict,
|
||||
dictBufferCapacity, parameters);
|
||||
|
||||
dictBufferCapacity = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
|
||||
parameters.zParams);
|
||||
if (ZDICT_isError(dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
/* Check total compressed size */
|
||||
{
|
||||
/* Pointers */
|
||||
ZSTD_CCtx *cctx;
|
||||
ZSTD_CDict *cdict;
|
||||
void *dst;
|
||||
/* Local variables */
|
||||
size_t dstCapacity;
|
||||
size_t i;
|
||||
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||
{
|
||||
size_t maxSampleSize = 0;
|
||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
||||
for (; i < ctx->nbSamples; ++i) {
|
||||
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
|
||||
}
|
||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||
dst = malloc(dstCapacity);
|
||||
}
|
||||
/* Create the cctx and cdict */
|
||||
cctx = ZSTD_createCCtx();
|
||||
cdict = ZSTD_createCDict(dict, dictBufferCapacity,
|
||||
parameters.zParams.compressionLevel);
|
||||
if (!dst || !cctx || !cdict) {
|
||||
goto _compressCleanup;
|
||||
}
|
||||
/* Compress each sample and sum their sizes (or error) */
|
||||
totalCompressedSize = dictBufferCapacity;
|
||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
||||
for (; i < ctx->nbSamples; ++i) {
|
||||
const size_t size = ZSTD_compress_usingCDict(
|
||||
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
|
||||
ctx->samplesSizes[i], cdict);
|
||||
if (ZSTD_isError(size)) {
|
||||
totalCompressedSize = ERROR(GENERIC);
|
||||
goto _compressCleanup;
|
||||
}
|
||||
totalCompressedSize += size;
|
||||
}
|
||||
_compressCleanup:
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (dst) {
|
||||
free(dst);
|
||||
}
|
||||
}
|
||||
|
||||
_cleanup:
|
||||
FASTCOVER_best_finish(data->best, totalCompressedSize, parameters, dict,
|
||||
dictBufferCapacity);
|
||||
free(data);
|
||||
if (dict) {
|
||||
free(dict);
|
||||
}
|
||||
if (freqs) {
|
||||
free(freqs);
|
||||
}
|
||||
}
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) {
|
||||
BYTE* const dict = (BYTE*)dictBuffer;
|
||||
FASTCOVER_ctx_t ctx;
|
||||
parameters.splitPoint = 1.0;
|
||||
/* Initialize global data */
|
||||
g_displayLevel = parameters.zParams.notificationLevel;
|
||||
/* Checks */
|
||||
if (!FASTCOVER_checkParameters(parameters, dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (nbSamples == 0) {
|
||||
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||
ZDICT_DICTSIZE_MIN);
|
||||
return ERROR(dstSize_tooSmall);
|
||||
}
|
||||
/* Initialize context */
|
||||
if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
|
||||
parameters.d, parameters.splitPoint, parameters.f)) {
|
||||
DISPLAYLEVEL(1, "Failed to initialize context\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
/* Build the dictionary */
|
||||
DISPLAYLEVEL(2, "Building dictionary\n");
|
||||
{
|
||||
const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer,
|
||||
dictBufferCapacity, parameters);
|
||||
|
||||
const size_t dictionarySize = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
samplesBuffer, samplesSizes, (unsigned)ctx.nbTrainSamples,
|
||||
parameters.zParams);
|
||||
if (!ZSTD_isError(dictionarySize)) {
|
||||
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||
(U32)dictionarySize);
|
||||
}
|
||||
FASTCOVER_ctx_destroy(&ctx);
|
||||
return dictionarySize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_fastCover_params_t *parameters) {
|
||||
/* constants */
|
||||
const unsigned nbThreads = parameters->nbThreads;
|
||||
const double splitPoint =
|
||||
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
||||
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
||||
const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
|
||||
const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
|
||||
const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
|
||||
const unsigned kIterations =
|
||||
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
|
||||
const unsigned f = parameters->f == 0 ? 23 : parameters->f;
|
||||
|
||||
/* Local variables */
|
||||
const int displayLevel = parameters->zParams.notificationLevel;
|
||||
unsigned iteration = 1;
|
||||
unsigned d;
|
||||
unsigned k;
|
||||
FASTCOVER_best_t best;
|
||||
POOL_ctx *pool = NULL;
|
||||
|
||||
/* Checks */
|
||||
if (splitPoint <= 0 || splitPoint > 1) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (kMinK < kMaxD || kMaxK < kMinK) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (nbSamples == 0) {
|
||||
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||
ZDICT_DICTSIZE_MIN);
|
||||
return ERROR(dstSize_tooSmall);
|
||||
}
|
||||
if (nbThreads > 1) {
|
||||
pool = POOL_create(nbThreads, 1);
|
||||
if (!pool) {
|
||||
return ERROR(memory_allocation);
|
||||
}
|
||||
}
|
||||
/* Initialization */
|
||||
FASTCOVER_best_init(&best);
|
||||
/* Turn down global display level to clean up display at level 2 and below */
|
||||
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
|
||||
/* Loop through d first because each new value needs a new context */
|
||||
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
|
||||
kIterations);
|
||||
for (d = kMinD; d <= kMaxD; d += 2) {
|
||||
/* Initialize the context for this value of d */
|
||||
FASTCOVER_ctx_t ctx;
|
||||
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
|
||||
if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f)) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
|
||||
FASTCOVER_best_destroy(&best);
|
||||
POOL_free(pool);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
/* Loop through k reusing the same context */
|
||||
for (k = kMinK; k <= kMaxK; k += kStepSize) {
|
||||
/* Prepare the arguments */
|
||||
FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
|
||||
sizeof(FASTCOVER_tryParameters_data_t));
|
||||
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
|
||||
if (!data) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
|
||||
FASTCOVER_best_destroy(&best);
|
||||
FASTCOVER_ctx_destroy(&ctx);
|
||||
POOL_free(pool);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
data->ctx = &ctx;
|
||||
data->best = &best;
|
||||
data->dictBufferCapacity = dictBufferCapacity;
|
||||
data->parameters = *parameters;
|
||||
data->parameters.k = k;
|
||||
data->parameters.d = d;
|
||||
data->parameters.f = f;
|
||||
data->parameters.splitPoint = splitPoint;
|
||||
data->parameters.steps = kSteps;
|
||||
data->parameters.zParams.notificationLevel = g_displayLevel;
|
||||
/* Check the parameters */
|
||||
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "fastCover parameters incorrect\n");
|
||||
free(data);
|
||||
continue;
|
||||
}
|
||||
/* Call the function and pass ownership of data to it */
|
||||
FASTCOVER_best_start(&best);
|
||||
if (pool) {
|
||||
POOL_add(pool, &FASTCOVER_tryParameters, data);
|
||||
} else {
|
||||
FASTCOVER_tryParameters(data);
|
||||
}
|
||||
/* Print status */
|
||||
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
|
||||
(U32)((iteration * 100) / kIterations));
|
||||
++iteration;
|
||||
}
|
||||
FASTCOVER_best_wait(&best);
|
||||
FASTCOVER_ctx_destroy(&ctx);
|
||||
}
|
||||
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
|
||||
/* Fill the output buffer and parameters with output of the best parameters */
|
||||
{
|
||||
const size_t dictSize = best.dictSize;
|
||||
if (ZSTD_isError(best.compressedSize)) {
|
||||
const size_t compressedSize = best.compressedSize;
|
||||
FASTCOVER_best_destroy(&best);
|
||||
POOL_free(pool);
|
||||
return compressedSize;
|
||||
}
|
||||
*parameters = best.parameters;
|
||||
memcpy(dictBuffer, best.dict, dictSize);
|
||||
FASTCOVER_best_destroy(&best);
|
||||
POOL_free(pool);
|
||||
return dictSize;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#include "mem.h" /* read */
|
||||
#include "pool.h"
|
||||
#include "threading.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
typedef struct {
|
||||
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
||||
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
||||
unsigned f; /* log of size of frequency array */
|
||||
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
|
||||
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
||||
double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
||||
ZDICT_params_t zParams;
|
||||
} ZDICT_fastCover_params_t;
|
||||
|
||||
|
||||
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
|
||||
* Train a dictionary from an array of samples using a modified version of the COVER algorithm.
|
||||
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||
* The resulting dictionary will be saved into `dictBuffer`.
|
||||
* All of the parameters except for f are optional.
|
||||
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
|
||||
* if steps is zero it defaults to its default value.
|
||||
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
|
||||
*
|
||||
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
* or an error code, which can be tested with ZDICT_isError().
|
||||
* On success `*parameters` contains the parameters selected.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_fastCover_params_t *parameters);
|
||||
|
||||
|
||||
/*! ZDICT_trainFromBuffer_fastCover():
|
||||
* Train a dictionary from an array of samples using a modified version of the COVER algorithm.
|
||||
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||
* The resulting dictionary will be saved into `dictBuffer`.
|
||||
* d, k, and f are required.
|
||||
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
* or an error code, which can be tested with ZDICT_isError().
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters);
|
||||
@@ -0,0 +1,183 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include <ctype.h>
|
||||
#include "fastCover.h"
|
||||
#include "io.h"
|
||||
#include "util.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAY("Error %i : ", error); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY("\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static const unsigned g_defaultMaxDictSize = 110 KB;
|
||||
#define DEFAULT_CLEVEL 3
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* FASTCOVER
|
||||
***************************************/
|
||||
int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info,
|
||||
unsigned maxDictSize,
|
||||
ZDICT_fastCover_params_t *params) {
|
||||
unsigned const displayLevel = params->zParams.notificationLevel;
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
|
||||
int result = 0;
|
||||
|
||||
/* Checks */
|
||||
if (!dictBuffer)
|
||||
EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */
|
||||
|
||||
{ size_t dictSize;
|
||||
/* Run the optimize version if either k or d is not provided */
|
||||
if (!params->d || !params->k) {
|
||||
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, params);
|
||||
} else {
|
||||
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *params);
|
||||
}
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100));
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
result = 1;
|
||||
goto _done;
|
||||
}
|
||||
/* save dict */
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||
saveDict(dictFileName, dictBuffer, dictSize);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
_done:
|
||||
free(dictBuffer);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int main(int argCount, const char* argv[])
|
||||
{
|
||||
int displayLevel = 2;
|
||||
const char* programName = argv[0];
|
||||
int operationResult = 0;
|
||||
|
||||
/* Initialize arguments to default values */
|
||||
unsigned k = 0;
|
||||
unsigned d = 0;
|
||||
unsigned f = 23;
|
||||
unsigned steps = 32;
|
||||
unsigned nbThreads = 1;
|
||||
unsigned split = 100;
|
||||
const char* outputFile = "fastCoverDict";
|
||||
unsigned dictID = 0;
|
||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
|
||||
/* Initialize table to store input files */
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
|
||||
unsigned filenameIdx = 0;
|
||||
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb = filenameIdx;
|
||||
int followLinks = 0; /* follow directory recursively */
|
||||
const char** extendedFileList = NULL;
|
||||
|
||||
/* Parse arguments */
|
||||
for (int i = 1; i < argCount; i++) {
|
||||
const char* argument = argv[i];
|
||||
if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "d=")) { d = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "f=")) { f = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "steps=")) { steps = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "split=")) { split = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "in=")) {
|
||||
filenameTable[filenameIdx] = argument;
|
||||
filenameIdx++;
|
||||
continue;
|
||||
}
|
||||
if (longCommandWArg(&argument, "out=")) {
|
||||
outputFile = argument;
|
||||
continue;
|
||||
}
|
||||
DISPLAYLEVEL(1, "Incorrect parameters\n");
|
||||
operationResult = 1;
|
||||
return operationResult;
|
||||
}
|
||||
|
||||
/* Get the list of all files recursively (because followLinks==0)*/
|
||||
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
|
||||
&fileNamesNb, followLinks);
|
||||
if (extendedFileList) {
|
||||
unsigned u;
|
||||
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
|
||||
free((void*)filenameTable);
|
||||
filenameTable = extendedFileList;
|
||||
filenameIdx = fileNamesNb;
|
||||
}
|
||||
|
||||
size_t blockSize = 0;
|
||||
|
||||
/* Set up zParams */
|
||||
ZDICT_params_t zParams;
|
||||
zParams.compressionLevel = DEFAULT_CLEVEL;
|
||||
zParams.notificationLevel = displayLevel;
|
||||
zParams.dictID = dictID;
|
||||
|
||||
/* Set up fastCover params */
|
||||
ZDICT_fastCover_params_t params;
|
||||
params.zParams = zParams;
|
||||
params.k = k;
|
||||
params.d = d;
|
||||
params.f = f;
|
||||
params.steps = steps;
|
||||
params.nbThreads = nbThreads;
|
||||
params.splitPoint = (double)split/100;
|
||||
|
||||
/* Build dictionary */
|
||||
sampleInfo* info = getSampleInfo(filenameTable,
|
||||
filenameIdx, blockSize, maxDictSize, zParams.notificationLevel);
|
||||
operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, ¶ms);
|
||||
|
||||
/* Free allocated memory */
|
||||
UTIL_freeFileList(extendedFileList, fileNamesBuf);
|
||||
freeSampleInfo(info);
|
||||
|
||||
return operationResult;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
echo "Building fastCover dictionary with in=../../lib/common f=20 out=dict1"
|
||||
./main in=../../../lib/common f=20 out=dict1
|
||||
zstd -be3 -D dict1 -r ../../../lib/common -q
|
||||
echo "Building fastCover dictionary with in=../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000"
|
||||
./main in=../../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000
|
||||
zstd -be3 -D dict2 -r ../../../lib/common -q
|
||||
echo "Building fastCover dictionary with 2 sample sources"
|
||||
./main in=../../../lib/common in=../../../lib/compress out=dict3
|
||||
zstd -be3 -D dict3 -r ../../../lib/common -q
|
||||
echo "Removing dict1 dict2 dict3"
|
||||
rm -f dict1 dict2 dict3
|
||||
|
||||
echo "Testing with invalid parameters, should fail"
|
||||
! ./main in=../../../lib/common r=10
|
||||
! ./main in=../../../lib/common d=10
|
||||
@@ -0,0 +1,52 @@
|
||||
ARG :=
|
||||
|
||||
CC ?= gcc
|
||||
CFLAGS ?= -O3
|
||||
INCLUDES := -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
|
||||
|
||||
TEST_INPUT := ../../../lib
|
||||
TEST_OUTPUT := randomDict
|
||||
|
||||
all: main run clean
|
||||
|
||||
.PHONY: test
|
||||
test: main testrun testshell clean
|
||||
|
||||
.PHONY: run
|
||||
run:
|
||||
echo "Building a random dictionary with given arguments"
|
||||
./main $(ARG)
|
||||
|
||||
main: main.o io.o random.o libzstd.a
|
||||
$(CC) $(CFLAGS) main.o io.o random.o libzstd.a -o main
|
||||
|
||||
main.o: main.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c main.c
|
||||
|
||||
random.o: random.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c random.c
|
||||
|
||||
io.o: io.c
|
||||
$(CC) $(CFLAGS) $(INCLUDES) -c io.c
|
||||
|
||||
libzstd.a:
|
||||
$(MAKE) -C ../../../lib libzstd.a
|
||||
mv ../../../lib/libzstd.a .
|
||||
|
||||
.PHONY: testrun
|
||||
testrun: main
|
||||
echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) "
|
||||
./main in=$(TEST_INPUT) out=$(TEST_OUTPUT)
|
||||
zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q
|
||||
rm -f $(TEST_OUTPUT)
|
||||
|
||||
.PHONY: testshell
|
||||
testshell: test.sh
|
||||
sh test.sh
|
||||
echo "Finish running test.sh"
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f *.o main libzstd.a
|
||||
$(MAKE) -C ../../../lib clean
|
||||
echo "Cleaning is completed"
|
||||
@@ -0,0 +1,20 @@
|
||||
Random Dictionary Builder
|
||||
|
||||
### Permitted Arguments:
|
||||
Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
|
||||
Output Dictionary (out=dictName): if not provided, default to defaultDict
|
||||
Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0
|
||||
Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB
|
||||
Size of Randomly Selected Segment (k=#): positive number; in bytes; if not provided, default to 200
|
||||
|
||||
###Running Test:
|
||||
make test
|
||||
|
||||
|
||||
###Usage:
|
||||
To build a random dictionary with the provided arguments: make ARG= followed by arguments
|
||||
|
||||
|
||||
### Examples:
|
||||
make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520"
|
||||
make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
|
||||
@@ -0,0 +1,284 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include <ctype.h>
|
||||
#include "io.h"
|
||||
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
|
||||
#include "platform.h" /* Large Files support */
|
||||
#include "util.h"
|
||||
#include "zdict.h"
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAY("Error %i : ", error); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY("\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
|
||||
#define SAMPLESIZE_MAX (128 KB)
|
||||
#define RANDOM_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
#define RANDOM_MEMMULT 9
|
||||
static const size_t g_maxMemory = (sizeof(size_t) == 4) ?
|
||||
(2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
|
||||
|
||||
#define NOISELENGTH 32
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Commandline related functions
|
||||
***************************************/
|
||||
unsigned readU32FromChar(const char** stringPtr){
|
||||
const char errorMsg[] = "error: numeric value too large";
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
if (result > max) exit(1);
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
if (result > maxK) exit(1);
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') {
|
||||
if (result > maxK) exit(1);
|
||||
result <<= 10;
|
||||
}
|
||||
(*stringPtr)++; /* skip `K` or `M` */
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
unsigned longCommandWArg(const char** stringPtr, const char* longCommand){
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
if (result) *stringPtr += comSize;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* File related operations
|
||||
**********************************************************/
|
||||
/** loadFiles() :
|
||||
* load samples from files listed in fileNamesTable into buffer.
|
||||
* works even if buffer is too small to load all samples.
|
||||
* Also provides the size of each sample into sampleSizes table
|
||||
* which must be sized correctly, using DiB_fileStats().
|
||||
* @return : nb of samples effectively loaded into `buffer`
|
||||
* *bufferSizePtr is modified, it provides the amount data loaded within buffer.
|
||||
* sampleSizes is filled with the size of each sample.
|
||||
*/
|
||||
static unsigned loadFiles(void* buffer, size_t* bufferSizePtr, size_t* sampleSizes,
|
||||
unsigned sstSize, const char** fileNamesTable, unsigned nbFiles,
|
||||
size_t targetChunkSize, unsigned displayLevel) {
|
||||
char* const buff = (char*)buffer;
|
||||
size_t pos = 0;
|
||||
unsigned nbLoadedChunks = 0, fileIndex;
|
||||
|
||||
for (fileIndex=0; fileIndex<nbFiles; fileIndex++) {
|
||||
const char* const fileName = fileNamesTable[fileIndex];
|
||||
unsigned long long const fs64 = UTIL_getFileSize(fileName);
|
||||
unsigned long long remainingToLoad = (fs64 == UTIL_FILESIZE_UNKNOWN) ? 0 : fs64;
|
||||
U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1;
|
||||
U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64;
|
||||
size_t const maxChunkSize = (size_t)MIN(chunkSize, SAMPLESIZE_MAX);
|
||||
U32 cnb;
|
||||
FILE* const f = fopen(fileName, "rb");
|
||||
if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileName);
|
||||
for (cnb=0; cnb<nbChunks; cnb++) {
|
||||
size_t const toLoad = (size_t)MIN(maxChunkSize, remainingToLoad);
|
||||
if (toLoad > *bufferSizePtr-pos) break;
|
||||
{ size_t const readSize = fread(buff+pos, 1, toLoad, f);
|
||||
if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName);
|
||||
pos += readSize;
|
||||
sampleSizes[nbLoadedChunks++] = toLoad;
|
||||
remainingToLoad -= targetChunkSize;
|
||||
if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */
|
||||
fileIndex = nbFiles; /* stop there */
|
||||
break;
|
||||
}
|
||||
if (toLoad < targetChunkSize) {
|
||||
fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR);
|
||||
} } }
|
||||
fclose(f);
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
*bufferSizePtr = pos;
|
||||
DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10))
|
||||
return nbLoadedChunks;
|
||||
}
|
||||
|
||||
#define rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static U32 getRand(U32* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 ^= prime2;
|
||||
rand32 = rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
/* shuffle() :
|
||||
* shuffle a table of file names in a semi-random way
|
||||
* It improves dictionary quality by reducing "locality" impact, so if sample set is very large,
|
||||
* it will load random elements from it, instead of just the first ones. */
|
||||
static void shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
||||
U32 seed = 0xFD2FB528;
|
||||
unsigned i;
|
||||
for (i = nbFiles - 1; i > 0; --i) {
|
||||
unsigned const j = getRand(&seed) % (i + 1);
|
||||
const char* const tmp = fileNamesTable[j];
|
||||
fileNamesTable[j] = fileNamesTable[i];
|
||||
fileNamesTable[i] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* Dictionary training functions
|
||||
**********************************************************/
|
||||
size_t findMaxMem(unsigned long long requiredMem) {
|
||||
size_t const step = 8 MB;
|
||||
void* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 23) + 1) << 23);
|
||||
requiredMem += step;
|
||||
if (requiredMem > g_maxMemory) requiredMem = g_maxMemory;
|
||||
|
||||
while (!testmem) {
|
||||
testmem = malloc((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
}
|
||||
|
||||
free(testmem);
|
||||
return (size_t)requiredMem;
|
||||
}
|
||||
|
||||
void saveDict(const char* dictFileName,
|
||||
const void* buff, size_t buffSize) {
|
||||
FILE* const f = fopen(dictFileName, "wb");
|
||||
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
|
||||
|
||||
{ size_t const n = fwrite(buff, 1, buffSize, f);
|
||||
if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) }
|
||||
|
||||
{ size_t const n = (size_t)fclose(f);
|
||||
if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) }
|
||||
}
|
||||
|
||||
/*! getFileStats() :
|
||||
* Given a list of files, and a chunkSize (0 == no chunk, whole files)
|
||||
* provides the amount of data to be loaded and the resulting nb of samples.
|
||||
* This is useful primarily for allocation purpose => sample buffer, and sample sizes table.
|
||||
*/
|
||||
static fileStats getFileStats(const char** fileNamesTable, unsigned nbFiles,
|
||||
size_t chunkSize, unsigned displayLevel) {
|
||||
fileStats fs;
|
||||
unsigned n;
|
||||
memset(&fs, 0, sizeof(fs));
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? 0 : fileSize;
|
||||
U32 const nbSamples = (U32)(chunkSize ? (srcSize + (chunkSize-1)) / chunkSize : 1);
|
||||
U64 const chunkToLoad = chunkSize ? MIN(chunkSize, srcSize) : srcSize;
|
||||
size_t const cappedChunkSize = (size_t)MIN(chunkToLoad, SAMPLESIZE_MAX);
|
||||
fs.totalSizeToLoad += cappedChunkSize * nbSamples;
|
||||
fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
|
||||
fs.nbSamples += nbSamples;
|
||||
}
|
||||
DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10));
|
||||
return fs;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||
unsigned maxDictSize, const unsigned displayLevel) {
|
||||
fileStats const fs = getFileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
|
||||
size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
|
||||
size_t const memMult = RANDOM_MEMMULT;
|
||||
size_t const maxMem = findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
|
||||
size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
|
||||
void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
|
||||
|
||||
/* Checks */
|
||||
if ((!sampleSizes) || (!srcBuffer))
|
||||
EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */
|
||||
if (fs.oneSampleTooLarge) {
|
||||
DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n");
|
||||
DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n");
|
||||
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX);
|
||||
}
|
||||
if (fs.nbSamples < 5) {
|
||||
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
|
||||
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
|
||||
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n");
|
||||
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
|
||||
}
|
||||
if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) {
|
||||
DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n");
|
||||
DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n");
|
||||
}
|
||||
|
||||
/* init */
|
||||
if (loadedSize < fs.totalSizeToLoad)
|
||||
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20));
|
||||
|
||||
/* Load input buffer */
|
||||
DISPLAYLEVEL(3, "Shuffling input files\n");
|
||||
shuffle(fileNamesTable, nbFiles);
|
||||
nbFiles = loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples,
|
||||
fileNamesTable, nbFiles, chunkSize, displayLevel);
|
||||
|
||||
sampleInfo *info = (sampleInfo *)malloc(sizeof(sampleInfo));
|
||||
|
||||
info->nbSamples = fs.nbSamples;
|
||||
info->samplesSizes = sampleSizes;
|
||||
info->srcBuffer = srcBuffer;
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
|
||||
void freeSampleInfo(sampleInfo *info) {
|
||||
if (!info) return;
|
||||
if (info->samplesSizes) free((void*)(info->samplesSizes));
|
||||
if (info->srcBuffer) free((void*)(info->srcBuffer));
|
||||
free(info);
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include <ctype.h>
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
|
||||
#include "platform.h" /* Large Files support */
|
||||
#include "util.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Structs
|
||||
***************************************/
|
||||
typedef struct {
|
||||
U64 totalSizeToLoad;
|
||||
unsigned oneSampleTooLarge;
|
||||
unsigned nbSamples;
|
||||
} fileStats;
|
||||
|
||||
typedef struct {
|
||||
const void* srcBuffer;
|
||||
const size_t *samplesSizes;
|
||||
size_t nbSamples;
|
||||
}sampleInfo;
|
||||
|
||||
|
||||
|
||||
/*! getSampleInfo():
|
||||
* Load from input files and add samples to buffer
|
||||
* @return: a sampleInfo struct containing infomation about buffer where samples are stored,
|
||||
* size of each sample, and total number of samples
|
||||
*/
|
||||
sampleInfo* getSampleInfo(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||
unsigned maxDictSize, const unsigned displayLevel);
|
||||
|
||||
|
||||
|
||||
/*! freeSampleInfo():
|
||||
* Free memory allocated for info
|
||||
*/
|
||||
void freeSampleInfo(sampleInfo *info);
|
||||
|
||||
|
||||
|
||||
/*! saveDict():
|
||||
* Save data stored on buff to dictFileName
|
||||
*/
|
||||
void saveDict(const char* dictFileName, const void* buff, size_t buffSize);
|
||||
|
||||
|
||||
unsigned readU32FromChar(const char** stringPtr);
|
||||
|
||||
/** longCommandWArg() :
|
||||
* check if *stringPtr is the same as longCommand.
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
unsigned longCommandWArg(const char** stringPtr, const char* longCommand);
|
||||
@@ -0,0 +1,161 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include <ctype.h>
|
||||
#include "random.h"
|
||||
#include "io.h"
|
||||
#include "util.h"
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAY("Error %i : ", error); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY("\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static const unsigned g_defaultMaxDictSize = 110 KB;
|
||||
#define DEFAULT_CLEVEL 3
|
||||
#define DEFAULT_k 200
|
||||
#define DEFAULT_OUTPUTFILE "defaultDict"
|
||||
#define DEFAULT_DICTID 0
|
||||
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* RANDOM
|
||||
***************************************/
|
||||
int RANDOM_trainFromFiles(const char* dictFileName, sampleInfo *info,
|
||||
unsigned maxDictSize,
|
||||
ZDICT_random_params_t *params) {
|
||||
unsigned const displayLevel = params->zParams.notificationLevel;
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
|
||||
int result = 0;
|
||||
|
||||
/* Checks */
|
||||
if (!dictBuffer)
|
||||
EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */
|
||||
|
||||
{ size_t dictSize;
|
||||
dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer,
|
||||
info->samplesSizes, info->nbSamples, *params);
|
||||
DISPLAYLEVEL(2, "k=%u\n", params->k);
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
result = 1;
|
||||
goto _done;
|
||||
}
|
||||
/* save dict */
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||
saveDict(dictFileName, dictBuffer, dictSize);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
_done:
|
||||
free(dictBuffer);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int main(int argCount, const char* argv[])
|
||||
{
|
||||
int displayLevel = 2;
|
||||
const char* programName = argv[0];
|
||||
int operationResult = 0;
|
||||
|
||||
/* Initialize arguments to default values */
|
||||
unsigned k = DEFAULT_k;
|
||||
const char* outputFile = DEFAULT_OUTPUTFILE;
|
||||
unsigned dictID = DEFAULT_DICTID;
|
||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
|
||||
/* Initialize table to store input files */
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));
|
||||
unsigned filenameIdx = 0;
|
||||
|
||||
/* Parse arguments */
|
||||
for (int i = 1; i < argCount; i++) {
|
||||
const char* argument = argv[i];
|
||||
if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "in=")) {
|
||||
filenameTable[filenameIdx] = argument;
|
||||
filenameIdx++;
|
||||
continue;
|
||||
}
|
||||
if (longCommandWArg(&argument, "out=")) {
|
||||
outputFile = argument;
|
||||
continue;
|
||||
}
|
||||
DISPLAYLEVEL(1, "Incorrect parameters\n");
|
||||
operationResult = 1;
|
||||
return operationResult;
|
||||
}
|
||||
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb = filenameIdx;
|
||||
int followLinks = 0; /* follow directory recursively */
|
||||
const char** extendedFileList = NULL;
|
||||
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf,
|
||||
&fileNamesNb, followLinks);
|
||||
if (extendedFileList) {
|
||||
unsigned u;
|
||||
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
|
||||
free((void*)filenameTable);
|
||||
filenameTable = extendedFileList;
|
||||
filenameIdx = fileNamesNb;
|
||||
}
|
||||
|
||||
size_t blockSize = 0;
|
||||
|
||||
ZDICT_random_params_t params;
|
||||
ZDICT_params_t zParams;
|
||||
zParams.compressionLevel = DEFAULT_CLEVEL;
|
||||
zParams.notificationLevel = displayLevel;
|
||||
zParams.dictID = dictID;
|
||||
params.zParams = zParams;
|
||||
params.k = k;
|
||||
|
||||
sampleInfo* info = getSampleInfo(filenameTable,
|
||||
filenameIdx, blockSize, maxDictSize, zParams.notificationLevel);
|
||||
operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, ¶ms);
|
||||
|
||||
/* Free allocated memory */
|
||||
UTIL_freeFileList(extendedFileList, fileNamesBuf);
|
||||
freeSampleInfo(info);
|
||||
|
||||
return operationResult;
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#include "random.h"
|
||||
#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} \
|
||||
}
|
||||
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(displayLevel, l, __VA_ARGS__)
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Random Dictionary Builder
|
||||
**********************************************************/
|
||||
/**
|
||||
* Returns the sum of the sample sizes.
|
||||
*/
|
||||
static size_t RANDOM_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
||||
size_t sum = 0;
|
||||
unsigned i;
|
||||
for (i = 0; i < nbSamples; ++i) {
|
||||
sum += samplesSizes[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A segment is an inclusive range in the source.
|
||||
*/
|
||||
typedef struct {
|
||||
U32 begin;
|
||||
U32 end;
|
||||
} RANDOM_segment_t;
|
||||
|
||||
|
||||
/**
|
||||
* Selects a random segment from totalSamplesSize - k + 1 possible segments
|
||||
*/
|
||||
static RANDOM_segment_t RANDOM_selectSegment(const size_t totalSamplesSize,
|
||||
ZDICT_random_params_t parameters) {
|
||||
const U32 k = parameters.k;
|
||||
RANDOM_segment_t segment;
|
||||
unsigned index;
|
||||
|
||||
/* Randomly generate a number from 0 to sampleSizes - k */
|
||||
index = rand()%(totalSamplesSize - k + 1);
|
||||
|
||||
/* inclusive */
|
||||
segment.begin = index;
|
||||
segment.end = index + k - 1;
|
||||
|
||||
return segment;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Check the validity of the parameters.
|
||||
* Returns non-zero if the parameters are valid and 0 otherwise.
|
||||
*/
|
||||
static int RANDOM_checkParameters(ZDICT_random_params_t parameters,
|
||||
size_t maxDictSize) {
|
||||
/* k is a required parameter */
|
||||
if (parameters.k == 0) {
|
||||
return 0;
|
||||
}
|
||||
/* k <= maxDictSize */
|
||||
if (parameters.k > maxDictSize) {
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Given the prepared context build the dictionary.
|
||||
*/
|
||||
static size_t RANDOM_buildDictionary(const size_t totalSamplesSize, const BYTE *samples,
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
ZDICT_random_params_t parameters) {
|
||||
BYTE *const dict = (BYTE *)dictBuffer;
|
||||
size_t tail = dictBufferCapacity;
|
||||
const int displayLevel = parameters.zParams.notificationLevel;
|
||||
while (tail > 0) {
|
||||
|
||||
/* Select a segment */
|
||||
RANDOM_segment_t segment = RANDOM_selectSegment(totalSamplesSize, parameters);
|
||||
|
||||
size_t segmentSize;
|
||||
segmentSize = MIN(segment.end - segment.begin + 1, tail);
|
||||
|
||||
tail -= segmentSize;
|
||||
memcpy(dict + tail, samples + segment.begin, segmentSize);
|
||||
DISPLAYUPDATE(
|
||||
2, "\r%u%% ",
|
||||
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
}
|
||||
|
||||
return tail;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_random(
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_random_params_t parameters) {
|
||||
const int displayLevel = parameters.zParams.notificationLevel;
|
||||
BYTE* const dict = (BYTE*)dictBuffer;
|
||||
/* Checks */
|
||||
if (!RANDOM_checkParameters(parameters, dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "k is incorrect\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (nbSamples == 0) {
|
||||
DISPLAYLEVEL(1, "Random must have at least one input file\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||
ZDICT_DICTSIZE_MIN);
|
||||
return ERROR(dstSize_tooSmall);
|
||||
}
|
||||
const size_t totalSamplesSize = RANDOM_sum(samplesSizes, nbSamples);
|
||||
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||
|
||||
DISPLAYLEVEL(2, "Building dictionary\n");
|
||||
{
|
||||
const size_t tail = RANDOM_buildDictionary(totalSamplesSize, samples,
|
||||
dictBuffer, dictBufferCapacity, parameters);
|
||||
const size_t dictSize = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
samplesBuffer, samplesSizes, nbSamples, parameters.zParams);
|
||||
if (!ZSTD_isError(dictSize)) {
|
||||
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||
(U32)dictSize);
|
||||
}
|
||||
return dictSize;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
|
||||
typedef struct {
|
||||
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+]; Default to 200 */
|
||||
ZDICT_params_t zParams;
|
||||
} ZDICT_random_params_t;
|
||||
|
||||
|
||||
/*! ZDICT_trainFromBuffer_random():
|
||||
* Train a dictionary from an array of samples.
|
||||
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||
* The resulting dictionary will be saved into `dictBuffer`.
|
||||
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
* or an error code, which can be tested with ZDICT_isError().
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_random( void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
ZDICT_random_params_t parameters);
|
||||
@@ -0,0 +1,14 @@
|
||||
echo "Building random dictionary with in=../../lib/common k=200 out=dict1"
|
||||
./main in=../../../lib/common k=200 out=dict1
|
||||
zstd -be3 -D dict1 -r ../../../lib/common -q
|
||||
echo "Building random dictionary with in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000"
|
||||
./main in=../../../lib/common k=500 out=dict2 dictID=100 maxdict=140000
|
||||
zstd -be3 -D dict2 -r ../../../lib/common -q
|
||||
echo "Building random dictionary with 2 sample sources"
|
||||
./main in=../../../lib/common in=../../../lib/compress out=dict3
|
||||
zstd -be3 -D dict3 -r ../../../lib/common -q
|
||||
echo "Removing dict1 dict2 dict3"
|
||||
rm -f dict1 dict2 dict3
|
||||
|
||||
echo "Testing with invalid parameters, should fail"
|
||||
! ./main r=10
|
||||
@@ -1,748 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# ################################################################
|
||||
# Copyright (c) 2021-2021, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ##########################################################################
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
|
||||
INCLUDED_SUBDIRS = ["common", "compress", "decompress"]
|
||||
|
||||
SKIPPED_FILES = [
|
||||
"common/mem.h",
|
||||
"common/zstd_deps.h",
|
||||
"common/pool.c",
|
||||
"common/pool.h",
|
||||
"common/threading.c",
|
||||
"common/threading.h",
|
||||
"common/zstd_trace.c",
|
||||
"common/zstd_trace.h",
|
||||
"compress/zstdmt_compress.h",
|
||||
"compress/zstdmt_compress.c",
|
||||
]
|
||||
|
||||
XXHASH_FILES = [
|
||||
"common/xxhash.c",
|
||||
"common/xxhash.h",
|
||||
]
|
||||
|
||||
|
||||
class FileLines(object):
|
||||
def __init__(self, filename):
|
||||
self.filename = filename
|
||||
with open(self.filename, "r") as f:
|
||||
self.lines = f.readlines()
|
||||
|
||||
def write(self):
|
||||
with open(self.filename, "w") as f:
|
||||
f.write("".join(self.lines))
|
||||
|
||||
|
||||
class PartialPreprocessor(object):
|
||||
"""
|
||||
Looks for simple ifdefs and ifndefs and replaces them.
|
||||
Handles && and ||.
|
||||
Has fancy logic to handle translating elifs to ifs.
|
||||
Only looks for macros in the first part of the expression with no
|
||||
parens.
|
||||
Does not handle multi-line macros (only looks in first line).
|
||||
"""
|
||||
def __init__(self, defs: [(str, Optional[str])], replaces: [(str, str)], undefs: [str]):
|
||||
MACRO_GROUP = r"(?P<macro>[a-zA-Z_][a-zA-Z_0-9]*)"
|
||||
ELIF_GROUP = r"(?P<elif>el)?"
|
||||
OP_GROUP = r"(?P<op>&&|\|\|)?"
|
||||
|
||||
self._defs = {macro:value for macro, value in defs}
|
||||
self._replaces = {macro:value for macro, value in replaces}
|
||||
self._defs.update(self._replaces)
|
||||
self._undefs = set(undefs)
|
||||
|
||||
self._define = re.compile(r"\s*#\s*define")
|
||||
self._if = re.compile(r"\s*#\s*if")
|
||||
self._elif = re.compile(r"\s*#\s*(?P<elif>el)if")
|
||||
self._else = re.compile(r"\s*#\s*(?P<else>else)")
|
||||
self._endif = re.compile(r"\s*#\s*endif")
|
||||
|
||||
self._ifdef = re.compile(fr"\s*#\s*if(?P<not>n)?def {MACRO_GROUP}\s*")
|
||||
self._if_defined = re.compile(
|
||||
fr"\s*#\s*{ELIF_GROUP}if\s+(?P<not>!)?\s*defined\s*\(\s*{MACRO_GROUP}\s*\)\s*{OP_GROUP}"
|
||||
)
|
||||
self._if_defined_value = re.compile(
|
||||
fr"\s*#\s*{ELIF_GROUP}if\s+defined\s*\(\s*{MACRO_GROUP}\s*\)\s*"
|
||||
fr"(?P<op>&&)\s*"
|
||||
fr"(?P<openp>\()?\s*"
|
||||
fr"(?P<macro2>[a-zA-Z_][a-zA-Z_0-9]*)\s*"
|
||||
fr"(?P<cmp>[=><!]+)\s*"
|
||||
fr"(?P<value>[0-9]*)\s*"
|
||||
fr"(?P<closep>\))?\s*"
|
||||
)
|
||||
self._if_true = re.compile(
|
||||
fr"\s*#\s*{ELIF_GROUP}if\s+{MACRO_GROUP}\s*{OP_GROUP}"
|
||||
)
|
||||
|
||||
self._c_comment = re.compile(r"/\*.*?\*/")
|
||||
self._cpp_comment = re.compile(r"//")
|
||||
|
||||
def _log(self, *args, **kwargs):
|
||||
print(*args, **kwargs)
|
||||
|
||||
def _strip_comments(self, line):
|
||||
# First strip c-style comments (may include //)
|
||||
while True:
|
||||
m = self._c_comment.search(line)
|
||||
if m is None:
|
||||
break
|
||||
line = line[:m.start()] + line[m.end():]
|
||||
|
||||
# Then strip cpp-style comments
|
||||
m = self._cpp_comment.search(line)
|
||||
if m is not None:
|
||||
line = line[:m.start()]
|
||||
|
||||
return line
|
||||
|
||||
def _fixup_indentation(self, macro, replace: [str]):
|
||||
if len(replace) == 0:
|
||||
return replace
|
||||
if len(replace) == 1 and self._define.match(replace[0]) is None:
|
||||
# If there is only one line, only replace defines
|
||||
return replace
|
||||
|
||||
|
||||
all_pound = True
|
||||
for line in replace:
|
||||
if not line.startswith('#'):
|
||||
all_pound = False
|
||||
if all_pound:
|
||||
replace = [line[1:] for line in replace]
|
||||
|
||||
min_spaces = len(replace[0])
|
||||
for line in replace:
|
||||
spaces = 0
|
||||
for i, c in enumerate(line):
|
||||
if c != ' ':
|
||||
# Non-preprocessor line ==> skip the fixup
|
||||
if not all_pound and c != '#':
|
||||
return replace
|
||||
spaces = i
|
||||
break
|
||||
min_spaces = min(min_spaces, spaces)
|
||||
|
||||
replace = [line[min_spaces:] for line in replace]
|
||||
|
||||
if all_pound:
|
||||
replace = ["#" + line for line in replace]
|
||||
|
||||
return replace
|
||||
|
||||
def _handle_if_block(self, macro, idx, is_true, prepend):
|
||||
"""
|
||||
Remove the #if or #elif block starting on this line.
|
||||
"""
|
||||
REMOVE_ONE = 0
|
||||
KEEP_ONE = 1
|
||||
REMOVE_REST = 2
|
||||
|
||||
if is_true:
|
||||
state = KEEP_ONE
|
||||
else:
|
||||
state = REMOVE_ONE
|
||||
|
||||
line = self._inlines[idx]
|
||||
is_if = self._if.match(line) is not None
|
||||
assert is_if or self._elif.match(line) is not None
|
||||
depth = 0
|
||||
|
||||
start_idx = idx
|
||||
|
||||
idx += 1
|
||||
replace = prepend
|
||||
finished = False
|
||||
while idx < len(self._inlines):
|
||||
line = self._inlines[idx]
|
||||
# Nested if statement
|
||||
if self._if.match(line):
|
||||
depth += 1
|
||||
idx += 1
|
||||
continue
|
||||
# We're inside a nested statement
|
||||
if depth > 0:
|
||||
if self._endif.match(line):
|
||||
depth -= 1
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
# We're at the original depth
|
||||
|
||||
# Looking only for an endif.
|
||||
# We've found a true statement, but haven't
|
||||
# completely elided the if block, so we just
|
||||
# remove the remainder.
|
||||
if state == REMOVE_REST:
|
||||
if self._endif.match(line):
|
||||
if is_if:
|
||||
# Remove the endif because we took the first if
|
||||
idx += 1
|
||||
finished = True
|
||||
break
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
if state == KEEP_ONE:
|
||||
m = self._elif.match(line)
|
||||
if self._endif.match(line):
|
||||
replace += self._inlines[start_idx + 1:idx]
|
||||
idx += 1
|
||||
finished = True
|
||||
break
|
||||
if self._elif.match(line) or self._else.match(line):
|
||||
replace += self._inlines[start_idx + 1:idx]
|
||||
state = REMOVE_REST
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
if state == REMOVE_ONE:
|
||||
m = self._elif.match(line)
|
||||
if m is not None:
|
||||
if is_if:
|
||||
idx += 1
|
||||
b = m.start('elif')
|
||||
e = m.end('elif')
|
||||
assert e - b == 2
|
||||
replace.append(line[:b] + line[e:])
|
||||
finished = True
|
||||
break
|
||||
m = self._else.match(line)
|
||||
if m is not None:
|
||||
if is_if:
|
||||
idx += 1
|
||||
while self._endif.match(self._inlines[idx]) is None:
|
||||
replace.append(self._inlines[idx])
|
||||
idx += 1
|
||||
idx += 1
|
||||
finished = True
|
||||
break
|
||||
if self._endif.match(line):
|
||||
if is_if:
|
||||
# Remove the endif because no other elifs
|
||||
idx += 1
|
||||
finished = True
|
||||
break
|
||||
idx += 1
|
||||
continue
|
||||
if not finished:
|
||||
raise RuntimeError("Unterminated if block!")
|
||||
|
||||
replace = self._fixup_indentation(macro, replace)
|
||||
|
||||
self._log(f"\tHardwiring {macro}")
|
||||
if start_idx > 0:
|
||||
self._log(f"\t\t {self._inlines[start_idx - 1][:-1]}")
|
||||
for x in range(start_idx, idx):
|
||||
self._log(f"\t\t- {self._inlines[x][:-1]}")
|
||||
for line in replace:
|
||||
self._log(f"\t\t+ {line[:-1]}")
|
||||
if idx < len(self._inlines):
|
||||
self._log(f"\t\t {self._inlines[idx][:-1]}")
|
||||
|
||||
return idx, replace
|
||||
|
||||
def _preprocess_once(self):
|
||||
outlines = []
|
||||
idx = 0
|
||||
changed = False
|
||||
while idx < len(self._inlines):
|
||||
line = self._inlines[idx]
|
||||
sline = self._strip_comments(line)
|
||||
m = self._ifdef.fullmatch(sline)
|
||||
if_true = False
|
||||
if m is None:
|
||||
m = self._if_defined_value.fullmatch(sline)
|
||||
if m is None:
|
||||
m = self._if_defined.match(sline)
|
||||
if m is None:
|
||||
m = self._if_true.match(sline)
|
||||
if_true = (m is not None)
|
||||
if m is None:
|
||||
outlines.append(line)
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
groups = m.groupdict()
|
||||
macro = groups['macro']
|
||||
op = groups.get('op')
|
||||
|
||||
if not (macro in self._defs or macro in self._undefs):
|
||||
outlines.append(line)
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
defined = macro in self._defs
|
||||
|
||||
# Needed variables set:
|
||||
# resolved: Is the statement fully resolved?
|
||||
# is_true: If resolved, is the statement true?
|
||||
ifdef = False
|
||||
if if_true:
|
||||
if not defined:
|
||||
outlines.append(line)
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
defined_value = self._defs[macro]
|
||||
is_int = True
|
||||
try:
|
||||
defined_value = int(defined_value)
|
||||
except TypeError:
|
||||
is_int = False
|
||||
except ValueError:
|
||||
is_int = False
|
||||
|
||||
resolved = is_int
|
||||
is_true = (defined_value != 0)
|
||||
|
||||
if resolved and op is not None:
|
||||
if op == '&&':
|
||||
resolved = not is_true
|
||||
else:
|
||||
assert op == '||'
|
||||
resolved = is_true
|
||||
|
||||
else:
|
||||
ifdef = groups.get('not') is None
|
||||
elseif = groups.get('elif') is not None
|
||||
|
||||
macro2 = groups.get('macro2')
|
||||
cmp = groups.get('cmp')
|
||||
value = groups.get('value')
|
||||
openp = groups.get('openp')
|
||||
closep = groups.get('closep')
|
||||
|
||||
is_true = (ifdef == defined)
|
||||
resolved = True
|
||||
if op is not None:
|
||||
if op == '&&':
|
||||
resolved = not is_true
|
||||
else:
|
||||
assert op == '||'
|
||||
resolved = is_true
|
||||
|
||||
if macro2 is not None and not resolved:
|
||||
assert ifdef and defined and op == '&&' and cmp is not None
|
||||
# If the statment is true, but we have a single value check, then
|
||||
# check the value.
|
||||
defined_value = self._defs[macro]
|
||||
are_ints = True
|
||||
try:
|
||||
defined_value = int(defined_value)
|
||||
value = int(value)
|
||||
except TypeError:
|
||||
are_ints = False
|
||||
except ValueError:
|
||||
are_ints = False
|
||||
if (
|
||||
macro == macro2 and
|
||||
((openp is None) == (closep is None)) and
|
||||
are_ints
|
||||
):
|
||||
resolved = True
|
||||
if cmp == '<':
|
||||
is_true = defined_value < value
|
||||
elif cmp == '<=':
|
||||
is_true = defined_value <= value
|
||||
elif cmp == '==':
|
||||
is_true = defined_value == value
|
||||
elif cmp == '!=':
|
||||
is_true = defined_value != value
|
||||
elif cmp == '>=':
|
||||
is_true = defined_value >= value
|
||||
elif cmp == '>':
|
||||
is_true = defined_value > value
|
||||
else:
|
||||
resolved = False
|
||||
|
||||
if op is not None and not resolved:
|
||||
# Remove the first op in the line + spaces
|
||||
if op == '&&':
|
||||
opre = op
|
||||
else:
|
||||
assert op == '||'
|
||||
opre = r'\|\|'
|
||||
needle = re.compile(fr"(?P<if>\s*#\s*(el)?if\s+).*?(?P<op>{opre}\s*)")
|
||||
match = needle.match(line)
|
||||
assert match is not None
|
||||
newline = line[:match.end('if')] + line[match.end('op'):]
|
||||
|
||||
self._log(f"\tHardwiring partially resolved {macro}")
|
||||
self._log(f"\t\t- {line[:-1]}")
|
||||
self._log(f"\t\t+ {newline[:-1]}")
|
||||
|
||||
outlines.append(newline)
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
# Skip any statements we cannot fully compute
|
||||
if not resolved:
|
||||
outlines.append(line)
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
prepend = []
|
||||
if macro in self._replaces:
|
||||
assert not ifdef
|
||||
assert op is None
|
||||
value = self._replaces.pop(macro)
|
||||
prepend = [f"#define {macro} {value}\n"]
|
||||
|
||||
idx, replace = self._handle_if_block(macro, idx, is_true, prepend)
|
||||
outlines += replace
|
||||
changed = True
|
||||
|
||||
return changed, outlines
|
||||
|
||||
def preprocess(self, filename):
|
||||
with open(filename, 'r') as f:
|
||||
self._inlines = f.readlines()
|
||||
changed = True
|
||||
iters = 0
|
||||
while changed:
|
||||
iters += 1
|
||||
changed, outlines = self._preprocess_once()
|
||||
self._inlines = outlines
|
||||
|
||||
with open(filename, 'w') as f:
|
||||
f.write(''.join(self._inlines))
|
||||
|
||||
|
||||
class Freestanding(object):
|
||||
def __init__(
|
||||
self, zstd_deps: str, mem: str, source_lib: str, output_lib: str,
|
||||
external_xxhash: bool, xxh64_state: Optional[str],
|
||||
xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
|
||||
defs: [(str, Optional[str])], replaces: [(str, str)],
|
||||
undefs: [str], excludes: [str], seds: [str],
|
||||
):
|
||||
self._zstd_deps = zstd_deps
|
||||
self._mem = mem
|
||||
self._src_lib = source_lib
|
||||
self._dst_lib = output_lib
|
||||
self._external_xxhash = external_xxhash
|
||||
self._xxh64_state = xxh64_state
|
||||
self._xxh64_prefix = xxh64_prefix
|
||||
self._rewritten_includes = rewritten_includes
|
||||
self._defs = defs
|
||||
self._replaces = replaces
|
||||
self._undefs = undefs
|
||||
self._excludes = excludes
|
||||
self._seds = seds
|
||||
|
||||
def _dst_lib_file_paths(self):
|
||||
"""
|
||||
Yields all the file paths in the dst_lib.
|
||||
"""
|
||||
for root, dirname, filenames in os.walk(self._dst_lib):
|
||||
for filename in filenames:
|
||||
filepath = os.path.join(root, filename)
|
||||
yield filepath
|
||||
|
||||
def _log(self, *args, **kwargs):
|
||||
print(*args, **kwargs)
|
||||
|
||||
def _copy_file(self, lib_path):
|
||||
if not (lib_path.endswith(".c") or lib_path.endswith(".h")):
|
||||
return
|
||||
if lib_path in SKIPPED_FILES:
|
||||
self._log(f"\tSkipping file: {lib_path}")
|
||||
return
|
||||
if self._external_xxhash and lib_path in XXHASH_FILES:
|
||||
self._log(f"\tSkipping xxhash file: {lib_path}")
|
||||
return
|
||||
|
||||
src_path = os.path.join(self._src_lib, lib_path)
|
||||
dst_path = os.path.join(self._dst_lib, lib_path)
|
||||
self._log(f"\tCopying: {src_path} -> {dst_path}")
|
||||
shutil.copyfile(src_path, dst_path)
|
||||
|
||||
def _copy_source_lib(self):
|
||||
self._log("Copying source library into output library")
|
||||
|
||||
assert os.path.exists(self._src_lib)
|
||||
os.makedirs(self._dst_lib, exist_ok=True)
|
||||
self._copy_file("zstd.h")
|
||||
for subdir in INCLUDED_SUBDIRS:
|
||||
src_dir = os.path.join(self._src_lib, subdir)
|
||||
dst_dir = os.path.join(self._dst_lib, subdir)
|
||||
|
||||
assert os.path.exists(src_dir)
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
|
||||
for filename in os.listdir(src_dir):
|
||||
lib_path = os.path.join(subdir, filename)
|
||||
self._copy_file(lib_path)
|
||||
|
||||
def _copy_zstd_deps(self):
|
||||
dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
|
||||
self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
|
||||
shutil.copyfile(self._zstd_deps, dst_zstd_deps)
|
||||
|
||||
def _copy_mem(self):
|
||||
dst_mem = os.path.join(self._dst_lib, "common", "mem.h")
|
||||
self._log(f"Copying mem: {self._mem} -> {dst_mem}")
|
||||
shutil.copyfile(self._mem, dst_mem)
|
||||
|
||||
def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False):
|
||||
"""
|
||||
If value=None then hardwire that it is defined, but not what the value is.
|
||||
If undef=True then value must be None.
|
||||
If value='' then the macro is defined to '' exactly.
|
||||
"""
|
||||
assert not (undef and value is not None)
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
file = FileLines(filepath)
|
||||
|
||||
def _hardwire_defines(self):
|
||||
self._log("Hardwiring macros")
|
||||
partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs)
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
partial_preprocessor.preprocess(filepath)
|
||||
|
||||
def _remove_excludes(self):
|
||||
self._log("Removing excluded sections")
|
||||
for exclude in self._excludes:
|
||||
self._log(f"\tRemoving excluded sections for: {exclude}")
|
||||
begin_re = re.compile(f"BEGIN {exclude}")
|
||||
end_re = re.compile(f"END {exclude}")
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
file = FileLines(filepath)
|
||||
outlines = []
|
||||
skipped = []
|
||||
emit = True
|
||||
for line in file.lines:
|
||||
if emit and begin_re.search(line) is not None:
|
||||
assert end_re.search(line) is None
|
||||
emit = False
|
||||
if emit:
|
||||
outlines.append(line)
|
||||
else:
|
||||
skipped.append(line)
|
||||
if end_re.search(line) is not None:
|
||||
assert begin_re.search(line) is None
|
||||
self._log(f"\t\tRemoving excluded section: {exclude}")
|
||||
for s in skipped:
|
||||
self._log(f"\t\t\t- {s}")
|
||||
emit = True
|
||||
skipped = []
|
||||
if not emit:
|
||||
raise RuntimeError("Excluded section unfinished!")
|
||||
file.lines = outlines
|
||||
file.write()
|
||||
|
||||
def _rewrite_include(self, original, rewritten):
|
||||
self._log(f"\tRewriting include: {original} -> {rewritten}")
|
||||
regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})")
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
file = FileLines(filepath)
|
||||
for i, line in enumerate(file.lines):
|
||||
match = regex.match(line)
|
||||
if match is None:
|
||||
continue
|
||||
s = match.start('include')
|
||||
e = match.end('include')
|
||||
file.lines[i] = line[:s] + rewritten + line[e:]
|
||||
file.write()
|
||||
|
||||
def _rewrite_includes(self):
|
||||
self._log("Rewriting includes")
|
||||
for original, rewritten in self._rewritten_includes:
|
||||
self._rewrite_include(original, rewritten)
|
||||
|
||||
def _replace_xxh64_prefix(self):
|
||||
if self._xxh64_prefix is None:
|
||||
return
|
||||
self._log(f"Replacing XXH64 prefix with {self._xxh64_prefix}")
|
||||
replacements = []
|
||||
if self._xxh64_state is not None:
|
||||
replacements.append(
|
||||
(re.compile(r"([^\w]|^)(?P<orig>XXH64_state_t)([^\w]|$)"), self._xxh64_state)
|
||||
)
|
||||
if self._xxh64_prefix is not None:
|
||||
replacements.append(
|
||||
(re.compile(r"([^\w]|^)(?P<orig>XXH64)[\(_]"), self._xxh64_prefix)
|
||||
)
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
file = FileLines(filepath)
|
||||
for i, line in enumerate(file.lines):
|
||||
modified = False
|
||||
for regex, replacement in replacements:
|
||||
match = regex.search(line)
|
||||
while match is not None:
|
||||
modified = True
|
||||
b = match.start('orig')
|
||||
e = match.end('orig')
|
||||
line = line[:b] + replacement + line[e:]
|
||||
match = regex.search(line)
|
||||
if modified:
|
||||
self._log(f"\t- {file.lines[i][:-1]}")
|
||||
self._log(f"\t+ {line[:-1]}")
|
||||
file.lines[i] = line
|
||||
file.write()
|
||||
|
||||
def _parse_sed(self, sed):
|
||||
assert sed[0] == 's'
|
||||
delim = sed[1]
|
||||
match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed)
|
||||
assert match is not None
|
||||
regex = re.compile(match.group(1))
|
||||
format_str = match.group(2)
|
||||
is_global = match.group(3) == 'g'
|
||||
return regex, format_str, is_global
|
||||
|
||||
def _process_sed(self, sed):
|
||||
self._log(f"Processing sed: {sed}")
|
||||
regex, format_str, is_global = self._parse_sed(sed)
|
||||
|
||||
for filepath in self._dst_lib_file_paths():
|
||||
file = FileLines(filepath)
|
||||
for i, line in enumerate(file.lines):
|
||||
modified = False
|
||||
while True:
|
||||
match = regex.search(line)
|
||||
if match is None:
|
||||
break
|
||||
replacement = format_str.format(match.groups(''), match.groupdict(''))
|
||||
b = match.start()
|
||||
e = match.end()
|
||||
line = line[:b] + replacement + line[e:]
|
||||
modified = True
|
||||
if not is_global:
|
||||
break
|
||||
if modified:
|
||||
self._log(f"\t- {file.lines[i][:-1]}")
|
||||
self._log(f"\t+ {line[:-1]}")
|
||||
file.lines[i] = line
|
||||
file.write()
|
||||
|
||||
def _process_seds(self):
|
||||
self._log("Processing seds")
|
||||
for sed in self._seds:
|
||||
self._process_sed(sed)
|
||||
|
||||
|
||||
|
||||
def go(self):
|
||||
self._copy_source_lib()
|
||||
self._copy_zstd_deps()
|
||||
self._copy_mem()
|
||||
self._hardwire_defines()
|
||||
self._remove_excludes()
|
||||
self._rewrite_includes()
|
||||
self._replace_xxh64_prefix()
|
||||
self._process_seds()
|
||||
|
||||
|
||||
def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
|
||||
output = []
|
||||
for define in defines:
|
||||
parsed = define.split('=')
|
||||
if len(parsed) == 1:
|
||||
output.append((parsed[0], None))
|
||||
elif len(parsed) == 2:
|
||||
output.append((parsed[0], parsed[1]))
|
||||
else:
|
||||
raise RuntimeError(f"Bad define: {define}")
|
||||
return output
|
||||
|
||||
|
||||
def parse_pair(rewritten_includes: [str]) -> [(str, str)]:
|
||||
output = []
|
||||
for rewritten_include in rewritten_includes:
|
||||
parsed = rewritten_include.split('=')
|
||||
if len(parsed) == 2:
|
||||
output.append((parsed[0], parsed[1]))
|
||||
else:
|
||||
raise RuntimeError(f"Bad rewritten include: {rewritten_include}")
|
||||
return output
|
||||
|
||||
|
||||
|
||||
def main(name, args):
|
||||
parser = argparse.ArgumentParser(prog=name)
|
||||
parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file")
|
||||
parser.add_argument("--mem", default="mem.h", help="Memory module")
|
||||
parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library")
|
||||
parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library")
|
||||
parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.")
|
||||
parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'")
|
||||
parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
|
||||
parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
|
||||
parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
|
||||
parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
|
||||
parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed mutliple times)")
|
||||
parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
|
||||
parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'")
|
||||
args = parser.parse_args(args)
|
||||
|
||||
# Always remove threading
|
||||
if "ZSTD_MULTITHREAD" not in args.undefs:
|
||||
args.undefs.append("ZSTD_MULTITHREAD")
|
||||
|
||||
args.defs = parse_optional_pair(args.defs)
|
||||
for name, _ in args.defs:
|
||||
if name in args.undefs:
|
||||
raise RuntimeError(f"{name} is both defined and undefined!")
|
||||
|
||||
# Always set tracing to 0
|
||||
if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs):
|
||||
args.defs.append(("ZSTD_NO_TRACE", None))
|
||||
args.defs.append(("ZSTD_TRACE", "0"))
|
||||
|
||||
args.replaces = parse_pair(args.replaces)
|
||||
for name, _ in args.replaces:
|
||||
if name in args.undefs or name in args.defs:
|
||||
raise RuntimeError(f"{name} is both replaced and (un)defined!")
|
||||
|
||||
args.rewritten_includes = parse_pair(args.rewritten_includes)
|
||||
|
||||
external_xxhash = False
|
||||
if args.xxhash is not None:
|
||||
external_xxhash = True
|
||||
args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash))
|
||||
|
||||
if args.xxh64_prefix is not None:
|
||||
if not external_xxhash:
|
||||
raise RuntimeError("--xxh64-prefix may only be used with --xxhash provided")
|
||||
|
||||
if args.xxh64_state is not None:
|
||||
if not external_xxhash:
|
||||
raise RuntimeError("--xxh64-state may only be used with --xxhash provided")
|
||||
|
||||
Freestanding(
|
||||
args.zstd_deps,
|
||||
args.mem,
|
||||
args.source_lib,
|
||||
args.output_lib,
|
||||
external_xxhash,
|
||||
args.xxh64_state,
|
||||
args.xxh64_prefix,
|
||||
args.rewritten_includes,
|
||||
args.defs,
|
||||
args.replaces,
|
||||
args.undefs,
|
||||
args.excludes,
|
||||
args.seds,
|
||||
).go()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[0], sys.argv[1:])
|
||||
@@ -18,7 +18,7 @@ CFLAGS ?= -O3
|
||||
CFLAGS += -std=gnu99
|
||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
@@ -28,31 +28,22 @@ default: largeNbDicts
|
||||
|
||||
all : largeNbDicts
|
||||
|
||||
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
|
||||
largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
.PHONY: $(LIBZSTD)
|
||||
$(LIBZSTD):
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a
|
||||
|
||||
benchfn.o: $(PROGDIR)/benchfn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
timefn.o: $(PROGDIR)/timefn.c
|
||||
bench.o : $(PROGDIR)/bench.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
datagen.o: $(PROGDIR)/datagen.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
util.o: $(PROGDIR)/util.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
xxhash.o : $(LIBDIR)/common/xxhash.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
clean:
|
||||
$(RM) *.o
|
||||
$(MAKE) -C $(LIBDIR) clean > /dev/null
|
||||
$(RM) largeNbDicts
|
||||
|
||||
@@ -21,11 +21,10 @@
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, abort */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <limits.h> /* UINT_MAX */
|
||||
#include <assert.h> /* assert */
|
||||
|
||||
#include "util.h"
|
||||
#include "benchfn.h"
|
||||
#include "bench.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zdict.h"
|
||||
@@ -50,7 +49,6 @@
|
||||
|
||||
|
||||
/*--- Macros ---*/
|
||||
|
||||
#define CONTROL(c) { if (!(c)) abort(); }
|
||||
#undef MIN
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
@@ -128,7 +126,7 @@ static buffer_t createBuffer_fromFile(const char* fileName)
|
||||
static buffer_t
|
||||
createDictionaryBuffer(const char* dictionaryName,
|
||||
const void* srcBuffer,
|
||||
const size_t* srcBlockSizes, size_t nbBlocks,
|
||||
const size_t* srcBlockSizes, unsigned nbBlocks,
|
||||
size_t requestedDictSize)
|
||||
{
|
||||
if (dictionaryName) {
|
||||
@@ -142,10 +140,9 @@ createDictionaryBuffer(const char* dictionaryName,
|
||||
void* const dictBuffer = malloc(requestedDictSize);
|
||||
CONTROL(dictBuffer != NULL);
|
||||
|
||||
assert(nbBlocks <= UINT_MAX);
|
||||
size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
|
||||
srcBuffer,
|
||||
srcBlockSizes, (unsigned)nbBlocks);
|
||||
srcBlockSizes, nbBlocks);
|
||||
CONTROL(!ZSTD_isError(dictSize));
|
||||
|
||||
buffer_t result;
|
||||
@@ -156,18 +153,6 @@ createDictionaryBuffer(const char* dictionaryName,
|
||||
}
|
||||
}
|
||||
|
||||
static ZSTD_CDict* createCDictForDedicatedDictSearch(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
|
||||
ZSTD_CCtxParams_init(params, compressionLevel);
|
||||
ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableDedicatedDictSearch, 1);
|
||||
ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, compressionLevel);
|
||||
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params, ZSTD_defaultCMem);
|
||||
|
||||
ZSTD_freeCCtxParams(params);
|
||||
return cdict;
|
||||
}
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
* Loads `buffer`, with content from files listed within `fileNamesTable`.
|
||||
@@ -346,42 +331,6 @@ createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc)
|
||||
return result;
|
||||
}
|
||||
|
||||
static buffer_collection_t
|
||||
createBufferCollection_fromSliceCollection(slice_collection_t sc)
|
||||
{
|
||||
size_t const bufferSize = sliceCollection_totalCapacity(sc);
|
||||
|
||||
buffer_t buffer = createBuffer(bufferSize);
|
||||
CONTROL(buffer.ptr != NULL);
|
||||
|
||||
size_t const nbSlices = sc.nbSlices;
|
||||
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
|
||||
CONTROL(slices != NULL);
|
||||
|
||||
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
|
||||
CONTROL(capacities != NULL);
|
||||
|
||||
char* const ptr = (char*)buffer.ptr;
|
||||
size_t pos = 0;
|
||||
for (size_t n=0; n < nbSlices; n++) {
|
||||
capacities[n] = sc.capacities[n];
|
||||
slices[n] = ptr + pos;
|
||||
pos += capacities[n];
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < nbSlices; i++) {
|
||||
memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]);
|
||||
capacities[i] = sc.capacities[i];
|
||||
}
|
||||
|
||||
buffer_collection_t result;
|
||||
result.buffer = buffer;
|
||||
result.slices.nbSlices = nbSlices;
|
||||
result.slices.capacities = capacities;
|
||||
result.slices.slicePtrs = slices;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* @return : kBuffNull if any error */
|
||||
static buffer_collection_t
|
||||
@@ -445,38 +394,6 @@ typedef struct {
|
||||
size_t nbDDict;
|
||||
} ddict_collection_t;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CDict** cdicts;
|
||||
size_t nbCDict;
|
||||
} cdict_collection_t;
|
||||
|
||||
static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeCDictCollection(cdict_collection_t cdictc)
|
||||
{
|
||||
for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
|
||||
ZSTD_freeCDict(cdictc.cdicts[dictNb]);
|
||||
}
|
||||
free(cdictc.cdicts);
|
||||
}
|
||||
|
||||
/* returns .buffers=NULL if operation fails */
|
||||
static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel, int dedicatedDictSearch)
|
||||
{
|
||||
ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
|
||||
if (cdicts==NULL) return kNullCDictCollection;
|
||||
for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
|
||||
cdicts[dictNb] = dedicatedDictSearch ?
|
||||
createCDictForDedicatedDictSearch(dictBuffer, dictSize, cLevel) :
|
||||
ZSTD_createCDict(dictBuffer, dictSize, cLevel);
|
||||
CONTROL(cdicts[dictNb] != NULL);
|
||||
}
|
||||
cdict_collection_t cdictc;
|
||||
cdictc.cdicts = cdicts;
|
||||
cdictc.nbCDict = nbCDict;
|
||||
return cdictc;
|
||||
}
|
||||
|
||||
static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
|
||||
|
||||
static void freeDDictCollection(ddict_collection_t ddictc)
|
||||
@@ -504,38 +421,19 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
|
||||
}
|
||||
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleCDictionaries(cdict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbCDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d];
|
||||
dicts.cdicts[d] = dicts.cdicts[r];
|
||||
dicts.cdicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
ZSTD_CDict* tmpd = dicts.cdicts[d1];
|
||||
dicts.cdicts[d1] = dicts.cdicts[d2];
|
||||
dicts.cdicts[d2] = tmpd;
|
||||
}
|
||||
}
|
||||
|
||||
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleDDictionaries(ddict_collection_t dicts)
|
||||
/* mess with adresses, so that linear scanning dictionaries != linear address scanning */
|
||||
void shuffleDictionaries(ddict_collection_t dicts)
|
||||
{
|
||||
size_t const nbDicts = dicts.nbDDict;
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d = (size_t)rand() % nbDicts;
|
||||
size_t const d = rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d];
|
||||
dicts.ddicts[d] = dicts.ddicts[r];
|
||||
dicts.ddicts[r] = tmpd;
|
||||
}
|
||||
for (size_t r=0; r<nbDicts; r++) {
|
||||
size_t const d1 = (size_t)rand() % nbDicts;
|
||||
size_t const d2 = (size_t)rand() % nbDicts;
|
||||
size_t const d1 = rand() % nbDicts;
|
||||
size_t const d2 = rand() % nbDicts;
|
||||
ZSTD_DDict* tmpd = dicts.ddicts[d1];
|
||||
dicts.ddicts[d1] = dicts.ddicts[d2];
|
||||
dicts.ddicts[d2] = tmpd;
|
||||
@@ -584,29 +482,6 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre
|
||||
|
||||
/* --- Benchmark --- */
|
||||
|
||||
typedef struct {
|
||||
ZSTD_CCtx* cctx;
|
||||
size_t nbDicts;
|
||||
size_t dictNb;
|
||||
cdict_collection_t dictionaries;
|
||||
} compressInstructions;
|
||||
|
||||
compressInstructions createCompressInstructions(cdict_collection_t dictionaries)
|
||||
{
|
||||
compressInstructions ci;
|
||||
ci.cctx = ZSTD_createCCtx();
|
||||
CONTROL(ci.cctx != NULL);
|
||||
ci.nbDicts = dictionaries.nbCDict;
|
||||
ci.dictNb = 0;
|
||||
ci.dictionaries = dictionaries;
|
||||
return ci;
|
||||
}
|
||||
|
||||
void freeCompressInstructions(compressInstructions ci)
|
||||
{
|
||||
ZSTD_freeCCtx(ci.cctx);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
ZSTD_DCtx* dctx;
|
||||
size_t nbDicts;
|
||||
@@ -630,23 +505,6 @@ void freeDecompressInstructions(decompressInstructions di)
|
||||
ZSTD_freeDCtx(di.dctx);
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
compressInstructions* const ci = (compressInstructions*) payload;
|
||||
(void)dstCapacity;
|
||||
|
||||
ZSTD_compress_usingCDict(ci->cctx,
|
||||
dst, srcSize,
|
||||
src, srcSize,
|
||||
ci->dictionaries.cdicts[ci->dictNb]);
|
||||
|
||||
ci->dictNb = ci->dictNb + 1;
|
||||
if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/* benched function */
|
||||
size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
|
||||
{
|
||||
@@ -666,9 +524,8 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity
|
||||
|
||||
static int benchMem(slice_collection_t dstBlocks,
|
||||
slice_collection_t srcBlocks,
|
||||
ddict_collection_t ddictionaries,
|
||||
cdict_collection_t cdictionaries,
|
||||
unsigned nbRounds, int benchCompression)
|
||||
ddict_collection_t dictionaries,
|
||||
int nbRounds)
|
||||
{
|
||||
assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
|
||||
|
||||
@@ -679,46 +536,31 @@ static int benchMem(slice_collection_t dstBlocks,
|
||||
|
||||
BMK_timedFnState_t* const benchState =
|
||||
BMK_createTimedFnState(total_time_ms, ms_per_round);
|
||||
|
||||
decompressInstructions di = createDecompressInstructions(ddictionaries);
|
||||
compressInstructions ci = createCompressInstructions(cdictionaries);
|
||||
void* payload = benchCompression ? (void*)&ci : (void*)&di;
|
||||
BMK_benchParams_t const bp = {
|
||||
.benchFn = benchCompression ? compress : decompress,
|
||||
.benchPayload = payload,
|
||||
.initFn = NULL,
|
||||
.initPayload = NULL,
|
||||
.errorFn = ZSTD_isError,
|
||||
.blockCount = dstBlocks.nbSlices,
|
||||
.srcBuffers = (const void* const*) srcBlocks.slicePtrs,
|
||||
.srcSizes = srcBlocks.capacities,
|
||||
.dstBuffers = dstBlocks.slicePtrs,
|
||||
.dstCapacities = dstBlocks.capacities,
|
||||
.blockResults = NULL
|
||||
};
|
||||
decompressInstructions di = createDecompressInstructions(dictionaries);
|
||||
|
||||
for (;;) {
|
||||
BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, bp);
|
||||
BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState,
|
||||
decompress, &di,
|
||||
NULL, NULL,
|
||||
dstBlocks.nbSlices,
|
||||
(const void* const *)srcBlocks.slicePtrs, srcBlocks.capacities,
|
||||
dstBlocks.slicePtrs, dstBlocks.capacities,
|
||||
NULL);
|
||||
CONTROL(BMK_isSuccessful_runOutcome(outcome));
|
||||
|
||||
BMK_runTime_t const result = BMK_extract_runTime(outcome);
|
||||
double const dTime_ns = result.nanoSecPerRun;
|
||||
U64 const dTime_ns = result.nanoSecPerRun;
|
||||
double const dTime_sec = (double)dTime_ns / 1000000000;
|
||||
size_t const srcSize = result.sumOfReturn;
|
||||
double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
|
||||
if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
|
||||
if (benchCompression)
|
||||
DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
|
||||
else
|
||||
DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
|
||||
|
||||
double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB);
|
||||
if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps;
|
||||
DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
|
||||
fflush(stdout);
|
||||
if (BMK_isCompleted_TimedFn(benchState)) break;
|
||||
}
|
||||
DISPLAY("\n");
|
||||
|
||||
freeDecompressInstructions(di);
|
||||
freeCompressInstructions(ci);
|
||||
BMK_freeTimedFnState(benchState);
|
||||
|
||||
return 0; /* success */
|
||||
@@ -734,8 +576,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
const char* dictionary,
|
||||
size_t blockSize, int clevel,
|
||||
unsigned nbDictMax, unsigned nbBlocks,
|
||||
unsigned nbRounds, int benchCompression,
|
||||
int dedicatedDictSearch)
|
||||
int nbRounds)
|
||||
{
|
||||
int result = 0;
|
||||
|
||||
@@ -753,7 +594,6 @@ int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
if (blockSize)
|
||||
DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
|
||||
DISPLAYLEVEL(3, "\n");
|
||||
size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
|
||||
|
||||
|
||||
size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
|
||||
@@ -785,21 +625,19 @@ int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
|
||||
/* dictionary determination */
|
||||
buffer_t const dictBuffer = createDictionaryBuffer(dictionary,
|
||||
srcs.buffer.ptr,
|
||||
srcs.slices.capacities, srcs.slices.nbSlices,
|
||||
srcBuffer.ptr,
|
||||
srcSlices.capacities, nbBlocks,
|
||||
DICTSIZE);
|
||||
CONTROL(dictBuffer.ptr != NULL);
|
||||
|
||||
ZSTD_CDict* const cdict = dedicatedDictSearch ?
|
||||
createCDictForDedicatedDictSearch(dictBuffer.ptr, dictBuffer.size, clevel) :
|
||||
ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel);
|
||||
CONTROL(cdict != NULL);
|
||||
|
||||
size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel);
|
||||
CONTROL(cTotalSizeNoDict != 0);
|
||||
DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
clevel,
|
||||
(double)totalSrcSlicesSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
|
||||
(double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict);
|
||||
|
||||
size_t* const cSizes = malloc(nbBlocks * sizeof(size_t));
|
||||
CONTROL(cSizes != NULL);
|
||||
@@ -808,52 +646,30 @@ int bench(const char** fileNameTable, unsigned nbFiles,
|
||||
CONTROL(cTotalSize != 0);
|
||||
DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n",
|
||||
(unsigned)dictBuffer.size,
|
||||
(double)totalSrcSlicesSize / cTotalSize, (unsigned)cTotalSize);
|
||||
(double)srcSize / cTotalSize, (unsigned)cTotalSize);
|
||||
|
||||
/* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
|
||||
shrinkSizes(dstSlices, cSizes);
|
||||
|
||||
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel, dedicatedDictSearch);
|
||||
CONTROL(cdictionaries.cdicts != NULL);
|
||||
ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
|
||||
CONTROL(dictionaries.ddicts != NULL);
|
||||
|
||||
ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
|
||||
CONTROL(ddictionaries.ddicts != NULL);
|
||||
shuffleDictionaries(dictionaries);
|
||||
|
||||
if (benchCompression) {
|
||||
size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
shuffleCDictionaries(cdictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
} else {
|
||||
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
|
||||
size_t const allDictMem = dictMem * nbDicts;
|
||||
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
|
||||
nbDicts, (double)allDictMem / (1 MB));
|
||||
|
||||
shuffleDDictionaries(ddictionaries);
|
||||
|
||||
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
|
||||
CONTROL(resultCollection.buffer.ptr != NULL);
|
||||
|
||||
result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression);
|
||||
|
||||
freeBufferCollection(resultCollection);
|
||||
}
|
||||
result = benchMem(resultCollection.slices, dstSlices, dictionaries, nbRounds);
|
||||
|
||||
/* free all heap objects in reverse order */
|
||||
freeCDictCollection(cdictionaries);
|
||||
freeDDictCollection(ddictionaries);
|
||||
freeBufferCollection(resultCollection);
|
||||
freeDDictCollection(dictionaries);
|
||||
free(cSizes);
|
||||
ZSTD_freeCDict(cdict);
|
||||
freeBuffer(dictBuffer);
|
||||
@@ -880,7 +696,7 @@ static unsigned readU32FromChar(const char** stringPtr)
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
assert(result <= max); /* check overflow */
|
||||
result *= 10, result += (unsigned)**stringPtr - '0', (*stringPtr)++ ;
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
@@ -902,7 +718,7 @@ static unsigned readU32FromChar(const char** stringPtr)
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static int longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
@@ -917,8 +733,6 @@ int usage(const char* exeName)
|
||||
DISPLAY (" %s [Options] filename(s) \n", exeName);
|
||||
DISPLAY (" \n");
|
||||
DISPLAY ("Options : \n");
|
||||
DISPLAY ("-z : benchmark compression (default) \n");
|
||||
DISPLAY ("-d : benchmark decompression \n");
|
||||
DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n");
|
||||
DISPLAY ("-B# : split input into blocks of size # (default: no split) \n");
|
||||
DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
|
||||
@@ -940,14 +754,12 @@ int bad_usage(const char* exeName)
|
||||
int main (int argc, const char** argv)
|
||||
{
|
||||
int recursiveMode = 0;
|
||||
int benchCompression = 1;
|
||||
int dedicatedDictSearch = 0;
|
||||
unsigned nbRounds = BENCH_TIME_DEFAULT_S;
|
||||
int nbRounds = BENCH_TIME_DEFAULT_S;
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc < 2) return bad_usage(exeName);
|
||||
|
||||
const char** nameTable = (const char**)malloc((size_t)argc * sizeof(const char*));
|
||||
const char** nameTable = (const char**)malloc(argc * sizeof(const char*));
|
||||
assert(nameTable != NULL);
|
||||
unsigned nameIdx = 0;
|
||||
|
||||
@@ -960,8 +772,6 @@ int main (int argc, const char** argv)
|
||||
for (int argNb = 1; argNb < argc ; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
|
||||
if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
|
||||
if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
|
||||
if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
|
||||
if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
|
||||
if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
|
||||
@@ -970,28 +780,26 @@ int main (int argc, const char** argv)
|
||||
if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--dedicated-dict-search")) { dedicatedDictSearch = 1; continue; }
|
||||
if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "-")) { cLevel = readU32FromChar(&argument); continue; }
|
||||
/* anything that's not a command is a filename */
|
||||
nameTable[nameIdx++] = argument;
|
||||
}
|
||||
|
||||
FileNamesTable* filenameTable;
|
||||
const char** filenameTable = nameTable;
|
||||
unsigned nbFiles = nameIdx;
|
||||
char* buffer_containing_filenames = NULL;
|
||||
|
||||
if (recursiveMode) {
|
||||
#ifndef UTIL_HAS_CREATEFILELIST
|
||||
assert(0); /* missing capability, do not run */
|
||||
#endif
|
||||
filenameTable = UTIL_createExpandedFNT(nameTable, nameIdx, 1 /* follow_links */);
|
||||
} else {
|
||||
filenameTable = UTIL_assembleFileNamesTable(nameTable, nameIdx, NULL);
|
||||
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
|
||||
filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */);
|
||||
}
|
||||
|
||||
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression, dedicatedDictSearch);
|
||||
int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
|
||||
|
||||
UTIL_freeFileNamesTable(filenameTable);
|
||||
free(buffer_containing_filenames);
|
||||
free(nameTable);
|
||||
|
||||
return result;
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Tue, 8 Aug 2017 19:20:25 -0700
|
||||
Subject: [PATCH v5 0/5] Add xxhash and zstd modules
|
||||
|
||||
Hi all,
|
||||
|
||||
This patch set adds xxhash, zstd compression, and zstd decompression
|
||||
modules. It also adds zstd support to BtrFS and SquashFS.
|
||||
|
||||
Each patch has relevant summaries, benchmarks, and tests.
|
||||
|
||||
Best,
|
||||
Nick Terrell
|
||||
|
||||
Changelog:
|
||||
|
||||
v1 -> v2:
|
||||
- Make pointer in lib/xxhash.c:394 non-const (1/5)
|
||||
- Use div_u64() for division of u64s (2/5)
|
||||
- Reduce stack usage of ZSTD_compressSequences(), ZSTD_buildSeqTable(),
|
||||
ZSTD_decompressSequencesLong(), FSE_buildDTable(), FSE_decompress_wksp(),
|
||||
HUF_writeCTable(), HUF_readStats(), HUF_readCTable(),
|
||||
HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/5)
|
||||
- No zstd function uses more than 400 B of stack space (2/5)
|
||||
|
||||
v2 -> v3:
|
||||
- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
|
||||
(2/5)
|
||||
- Fix bug in dictionary compression from upstream commit cc1522351f (2/5)
|
||||
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/5)
|
||||
- Change default compression level for BtrFS to 3 (3/5)
|
||||
|
||||
v3 -> v4:
|
||||
- Fix compiler warnings (2/5)
|
||||
- Add missing includes (3/5)
|
||||
- Fix minor linter warnings (3/5, 4/5)
|
||||
- Add crypto patch (5/5)
|
||||
|
||||
v4 -> v5:
|
||||
- Fix rare compression bug from upstream commit 308047eb5d (2/5)
|
||||
- Fix bug introduced in v3 when working around the gcc-7 bug (2/5)
|
||||
- Fix ZSTD_DStream initialization code in squashfs (4/5)
|
||||
- Fix patch documentation for patches written by Sean Purcell (4/5)
|
||||
|
||||
Nick Terrell (5):
|
||||
lib: Add xxhash module
|
||||
lib: Add zstd modules
|
||||
btrfs: Add zstd support
|
||||
squashfs: Add zstd support
|
||||
crypto: Add zstd support
|
||||
|
||||
crypto/Kconfig | 9 +
|
||||
crypto/Makefile | 1 +
|
||||
crypto/testmgr.c | 10 +
|
||||
crypto/testmgr.h | 71 +
|
||||
crypto/zstd.c | 265 ++++
|
||||
fs/btrfs/Kconfig | 2 +
|
||||
fs/btrfs/Makefile | 2 +-
|
||||
fs/btrfs/compression.c | 1 +
|
||||
fs/btrfs/compression.h | 6 +-
|
||||
fs/btrfs/ctree.h | 1 +
|
||||
fs/btrfs/disk-io.c | 2 +
|
||||
fs/btrfs/ioctl.c | 6 +-
|
||||
fs/btrfs/props.c | 6 +
|
||||
fs/btrfs/super.c | 12 +-
|
||||
fs/btrfs/sysfs.c | 2 +
|
||||
fs/btrfs/zstd.c | 432 ++++++
|
||||
fs/squashfs/Kconfig | 14 +
|
||||
fs/squashfs/Makefile | 1 +
|
||||
fs/squashfs/decompressor.c | 7 +
|
||||
fs/squashfs/decompressor.h | 4 +
|
||||
fs/squashfs/squashfs_fs.h | 1 +
|
||||
fs/squashfs/zstd_wrapper.c | 151 ++
|
||||
include/linux/xxhash.h | 236 +++
|
||||
include/linux/zstd.h | 1157 +++++++++++++++
|
||||
include/uapi/linux/btrfs.h | 8 +-
|
||||
lib/Kconfig | 11 +
|
||||
lib/Makefile | 3 +
|
||||
lib/xxhash.c | 500 +++++++
|
||||
lib/zstd/Makefile | 18 +
|
||||
lib/zstd/bitstream.h | 374 +++++
|
||||
lib/zstd/compress.c | 3484 ++++++++++++++++++++++++++++++++++++++++++++
|
||||
lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++
|
||||
lib/zstd/entropy_common.c | 243 +++
|
||||
lib/zstd/error_private.h | 53 +
|
||||
lib/zstd/fse.h | 575 ++++++++
|
||||
lib/zstd/fse_compress.c | 795 ++++++++++
|
||||
lib/zstd/fse_decompress.c | 332 +++++
|
||||
lib/zstd/huf.h | 212 +++
|
||||
lib/zstd/huf_compress.c | 770 ++++++++++
|
||||
lib/zstd/huf_decompress.c | 960 ++++++++++++
|
||||
lib/zstd/mem.h | 151 ++
|
||||
lib/zstd/zstd_common.c | 75 +
|
||||
lib/zstd/zstd_internal.h | 263 ++++
|
||||
lib/zstd/zstd_opt.h | 1014 +++++++++++++
|
||||
44 files changed, 14756 insertions(+), 12 deletions(-)
|
||||
create mode 100644 crypto/zstd.c
|
||||
create mode 100644 fs/btrfs/zstd.c
|
||||
create mode 100644 fs/squashfs/zstd_wrapper.c
|
||||
create mode 100644 include/linux/xxhash.h
|
||||
create mode 100644 include/linux/zstd.h
|
||||
create mode 100644 lib/xxhash.c
|
||||
create mode 100644 lib/zstd/Makefile
|
||||
create mode 100644 lib/zstd/bitstream.h
|
||||
create mode 100644 lib/zstd/compress.c
|
||||
create mode 100644 lib/zstd/decompress.c
|
||||
create mode 100644 lib/zstd/entropy_common.c
|
||||
create mode 100644 lib/zstd/error_private.h
|
||||
create mode 100644 lib/zstd/fse.h
|
||||
create mode 100644 lib/zstd/fse_compress.c
|
||||
create mode 100644 lib/zstd/fse_decompress.c
|
||||
create mode 100644 lib/zstd/huf.h
|
||||
create mode 100644 lib/zstd/huf_compress.c
|
||||
create mode 100644 lib/zstd/huf_decompress.c
|
||||
create mode 100644 lib/zstd/mem.h
|
||||
create mode 100644 lib/zstd/zstd_common.c
|
||||
create mode 100644 lib/zstd/zstd_internal.h
|
||||
create mode 100644 lib/zstd/zstd_opt.h
|
||||
|
||||
--
|
||||
2.9.3
|
||||
@@ -0,0 +1,862 @@
|
||||
From a4b1ffb6e89bbccd519f9afa0910635668436105 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Mon, 17 Jul 2017 17:07:18 -0700
|
||||
Subject: [PATCH v5 1/5] lib: Add xxhash module
|
||||
|
||||
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
|
||||
extremely fast non-cryptographic hash algorithm for checksumming.
|
||||
The zstd compression and decompression modules added in the next patch
|
||||
require xxhash. I extracted it out from zstd since it is useful on its
|
||||
own. I copied the code from the upstream XXHash source repository and
|
||||
translated it into kernel style. I ran benchmarks and tests in the kernel
|
||||
and tests in userland.
|
||||
|
||||
I benchmarked xxhash as a special character device. I ran in four modes,
|
||||
no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to
|
||||
kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute
|
||||
hashes on the copied data. I also ran it with four different buffer sizes.
|
||||
The benchmark file is located in the upstream zstd source repository under
|
||||
`contrib/linux-kernel/xxhash_test.c` [1].
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs`
|
||||
from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large.
|
||||
Run the following commands for the benchmark:
|
||||
|
||||
modprobe xxhash_test
|
||||
mknod xxhash_test c 245 0
|
||||
time cp filesystem.squashfs xxhash_test
|
||||
|
||||
The time is reported by the time of the userland `cp`.
|
||||
The GB/s is computed with
|
||||
|
||||
1,536,217,008 B / time(buffer size, hash)
|
||||
|
||||
which includes the time to copy from userland.
|
||||
The Normalized GB/s is computed with
|
||||
|
||||
1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)).
|
||||
|
||||
|
||||
| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s |
|
||||
|-----------------|-------|----------|------|---------------|
|
||||
| 1024 | none | 0.408 | 3.77 | - |
|
||||
| 1024 | xxh32 | 0.649 | 2.37 | 6.37 |
|
||||
| 1024 | xxh64 | 0.542 | 2.83 | 11.46 |
|
||||
| 1024 | crc32 | 1.290 | 1.19 | 1.74 |
|
||||
| 4096 | none | 0.380 | 4.04 | - |
|
||||
| 4096 | xxh32 | 0.645 | 2.38 | 5.79 |
|
||||
| 4096 | xxh64 | 0.500 | 3.07 | 12.80 |
|
||||
| 4096 | crc32 | 1.168 | 1.32 | 1.95 |
|
||||
| 8192 | none | 0.351 | 4.38 | - |
|
||||
| 8192 | xxh32 | 0.614 | 2.50 | 5.84 |
|
||||
| 8192 | xxh64 | 0.464 | 3.31 | 13.60 |
|
||||
| 8192 | crc32 | 1.163 | 1.32 | 1.89 |
|
||||
| 16384 | none | 0.346 | 4.43 | - |
|
||||
| 16384 | xxh32 | 0.590 | 2.60 | 6.30 |
|
||||
| 16384 | xxh64 | 0.466 | 3.30 | 12.80 |
|
||||
| 16384 | crc32 | 1.183 | 1.30 | 1.84 |
|
||||
|
||||
Tested in userland using the test-suite in the zstd repo under
|
||||
`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the
|
||||
kernel functions. A line in each branch of every function in `xxhash.c`
|
||||
was commented out to ensure that the test-suite fails. Additionally
|
||||
tested while testing zstd and with SMHasher [3].
|
||||
|
||||
[1] https://phabricator.intern.facebook.com/P57526246
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp
|
||||
[3] https://github.com/aappleby/smhasher
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
XXHash source repository: https://github.com/cyan4973/xxhash
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v1 -> v2:
|
||||
- Make pointer in lib/xxhash.c:394 non-const
|
||||
|
||||
include/linux/xxhash.h | 236 +++++++++++++++++++++++
|
||||
lib/Kconfig | 3 +
|
||||
lib/Makefile | 1 +
|
||||
lib/xxhash.c | 500 +++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
4 files changed, 740 insertions(+)
|
||||
create mode 100644 include/linux/xxhash.h
|
||||
create mode 100644 lib/xxhash.c
|
||||
|
||||
diff --git a/include/linux/xxhash.h b/include/linux/xxhash.h
|
||||
new file mode 100644
|
||||
index 0000000..9e1f42c
|
||||
--- /dev/null
|
||||
+++ b/include/linux/xxhash.h
|
||||
@@ -0,0 +1,236 @@
|
||||
+/*
|
||||
+ * xxHash - Extremely Fast Hash algorithm
|
||||
+ * Copyright (C) 2012-2016, Yann Collet.
|
||||
+ *
|
||||
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
+ *
|
||||
+ * Redistribution and use in source and binary forms, with or without
|
||||
+ * modification, are permitted provided that the following conditions are
|
||||
+ * met:
|
||||
+ *
|
||||
+ * * Redistributions of source code must retain the above copyright
|
||||
+ * notice, this list of conditions and the following disclaimer.
|
||||
+ * * Redistributions in binary form must reproduce the above
|
||||
+ * copyright notice, this list of conditions and the following disclaimer
|
||||
+ * in the documentation and/or other materials provided with the
|
||||
+ * distribution.
|
||||
+ *
|
||||
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it under
|
||||
+ * the terms of the GNU General Public License version 2 as published by the
|
||||
+ * Free Software Foundation. This program is dual-licensed; you may select
|
||||
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
+ * ("BSD").
|
||||
+ *
|
||||
+ * You can contact the author at:
|
||||
+ * - xxHash homepage: http://cyan4973.github.io/xxHash/
|
||||
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
+ */
|
||||
+
|
||||
+/*
|
||||
+ * Notice extracted from xxHash homepage:
|
||||
+ *
|
||||
+ * xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
|
||||
+ * It also successfully passes all tests from the SMHasher suite.
|
||||
+ *
|
||||
+ * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2
|
||||
+ * Duo @3GHz)
|
||||
+ *
|
||||
+ * Name Speed Q.Score Author
|
||||
+ * xxHash 5.4 GB/s 10
|
||||
+ * CrapWow 3.2 GB/s 2 Andrew
|
||||
+ * MumurHash 3a 2.7 GB/s 10 Austin Appleby
|
||||
+ * SpookyHash 2.0 GB/s 10 Bob Jenkins
|
||||
+ * SBox 1.4 GB/s 9 Bret Mulvey
|
||||
+ * Lookup3 1.2 GB/s 9 Bob Jenkins
|
||||
+ * SuperFastHash 1.2 GB/s 1 Paul Hsieh
|
||||
+ * CityHash64 1.05 GB/s 10 Pike & Alakuijala
|
||||
+ * FNV 0.55 GB/s 5 Fowler, Noll, Vo
|
||||
+ * CRC32 0.43 GB/s 9
|
||||
+ * MD5-32 0.33 GB/s 10 Ronald L. Rivest
|
||||
+ * SHA1-32 0.28 GB/s 10
|
||||
+ *
|
||||
+ * Q.Score is a measure of quality of the hash function.
|
||||
+ * It depends on successfully passing SMHasher test set.
|
||||
+ * 10 is a perfect score.
|
||||
+ *
|
||||
+ * A 64-bits version, named xxh64 offers much better speed,
|
||||
+ * but for 64-bits applications only.
|
||||
+ * Name Speed on 64 bits Speed on 32 bits
|
||||
+ * xxh64 13.8 GB/s 1.9 GB/s
|
||||
+ * xxh32 6.8 GB/s 6.0 GB/s
|
||||
+ */
|
||||
+
|
||||
+#ifndef XXHASH_H
|
||||
+#define XXHASH_H
|
||||
+
|
||||
+#include <linux/types.h>
|
||||
+
|
||||
+/*-****************************
|
||||
+ * Simple Hash Functions
|
||||
+ *****************************/
|
||||
+
|
||||
+/**
|
||||
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
|
||||
+ *
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ * @seed: The seed can be used to alter the result predictably.
|
||||
+ *
|
||||
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
|
||||
+ *
|
||||
+ * Return: The 32-bit hash of the data.
|
||||
+ */
|
||||
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64() - calculate the 64-bit hash of the input with a given seed.
|
||||
+ *
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ * @seed: The seed can be used to alter the result predictably.
|
||||
+ *
|
||||
+ * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems.
|
||||
+ *
|
||||
+ * Return: The 64-bit hash of the data.
|
||||
+ */
|
||||
+uint64_t xxh64(const void *input, size_t length, uint64_t seed);
|
||||
+
|
||||
+/*-****************************
|
||||
+ * Streaming Hash Functions
|
||||
+ *****************************/
|
||||
+
|
||||
+/*
|
||||
+ * These definitions are only meant to allow allocation of XXH state
|
||||
+ * statically, on stack, or in a struct for example.
|
||||
+ * Do not use members directly.
|
||||
+ */
|
||||
+
|
||||
+/**
|
||||
+ * struct xxh32_state - private xxh32 state, do not use members directly
|
||||
+ */
|
||||
+struct xxh32_state {
|
||||
+ uint32_t total_len_32;
|
||||
+ uint32_t large_len;
|
||||
+ uint32_t v1;
|
||||
+ uint32_t v2;
|
||||
+ uint32_t v3;
|
||||
+ uint32_t v4;
|
||||
+ uint32_t mem32[4];
|
||||
+ uint32_t memsize;
|
||||
+};
|
||||
+
|
||||
+/**
|
||||
+ * struct xxh32_state - private xxh64 state, do not use members directly
|
||||
+ */
|
||||
+struct xxh64_state {
|
||||
+ uint64_t total_len;
|
||||
+ uint64_t v1;
|
||||
+ uint64_t v2;
|
||||
+ uint64_t v3;
|
||||
+ uint64_t v4;
|
||||
+ uint64_t mem64[4];
|
||||
+ uint32_t memsize;
|
||||
+};
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_reset() - reset the xxh32 state to start a new hashing operation
|
||||
+ *
|
||||
+ * @state: The xxh32 state to reset.
|
||||
+ * @seed: Initialize the hash state with this seed.
|
||||
+ *
|
||||
+ * Call this function on any xxh32_state to prepare for a new hashing operation.
|
||||
+ */
|
||||
+void xxh32_reset(struct xxh32_state *state, uint32_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_update() - hash the data given and update the xxh32 state
|
||||
+ *
|
||||
+ * @state: The xxh32 state to update.
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ *
|
||||
+ * After calling xxh32_reset() call xxh32_update() as many times as necessary.
|
||||
+ *
|
||||
+ * Return: Zero on success, otherwise an error code.
|
||||
+ */
|
||||
+int xxh32_update(struct xxh32_state *state, const void *input, size_t length);
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_digest() - produce the current xxh32 hash
|
||||
+ *
|
||||
+ * @state: Produce the current xxh32 hash of this state.
|
||||
+ *
|
||||
+ * A hash value can be produced at any time. It is still possible to continue
|
||||
+ * inserting input into the hash state after a call to xxh32_digest(), and
|
||||
+ * generate new hashes later on, by calling xxh32_digest() again.
|
||||
+ *
|
||||
+ * Return: The xxh32 hash stored in the state.
|
||||
+ */
|
||||
+uint32_t xxh32_digest(const struct xxh32_state *state);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_reset() - reset the xxh64 state to start a new hashing operation
|
||||
+ *
|
||||
+ * @state: The xxh64 state to reset.
|
||||
+ * @seed: Initialize the hash state with this seed.
|
||||
+ */
|
||||
+void xxh64_reset(struct xxh64_state *state, uint64_t seed);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_update() - hash the data given and update the xxh64 state
|
||||
+ * @state: The xxh64 state to update.
|
||||
+ * @input: The data to hash.
|
||||
+ * @length: The length of the data to hash.
|
||||
+ *
|
||||
+ * After calling xxh64_reset() call xxh64_update() as many times as necessary.
|
||||
+ *
|
||||
+ * Return: Zero on success, otherwise an error code.
|
||||
+ */
|
||||
+int xxh64_update(struct xxh64_state *state, const void *input, size_t length);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_digest() - produce the current xxh64 hash
|
||||
+ *
|
||||
+ * @state: Produce the current xxh64 hash of this state.
|
||||
+ *
|
||||
+ * A hash value can be produced at any time. It is still possible to continue
|
||||
+ * inserting input into the hash state after a call to xxh64_digest(), and
|
||||
+ * generate new hashes later on, by calling xxh64_digest() again.
|
||||
+ *
|
||||
+ * Return: The xxh64 hash stored in the state.
|
||||
+ */
|
||||
+uint64_t xxh64_digest(const struct xxh64_state *state);
|
||||
+
|
||||
+/*-**************************
|
||||
+ * Utils
|
||||
+ ***************************/
|
||||
+
|
||||
+/**
|
||||
+ * xxh32_copy_state() - copy the source state into the destination state
|
||||
+ *
|
||||
+ * @src: The source xxh32 state.
|
||||
+ * @dst: The destination xxh32 state.
|
||||
+ */
|
||||
+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src);
|
||||
+
|
||||
+/**
|
||||
+ * xxh64_copy_state() - copy the source state into the destination state
|
||||
+ *
|
||||
+ * @src: The source xxh64 state.
|
||||
+ * @dst: The destination xxh64 state.
|
||||
+ */
|
||||
+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src);
|
||||
+
|
||||
+#endif /* XXHASH_H */
|
||||
diff --git a/lib/Kconfig b/lib/Kconfig
|
||||
index 6762529..5e7541f 100644
|
||||
--- a/lib/Kconfig
|
||||
+++ b/lib/Kconfig
|
||||
@@ -192,6 +192,9 @@ config CRC8
|
||||
when they need to do cyclic redundancy check according CRC8
|
||||
algorithm. Module will be called crc8.
|
||||
|
||||
+config XXHASH
|
||||
+ tristate
|
||||
+
|
||||
config AUDIT_GENERIC
|
||||
bool
|
||||
depends on AUDIT && !AUDIT_ARCH
|
||||
diff --git a/lib/Makefile b/lib/Makefile
|
||||
index 40c1837..d06b68a 100644
|
||||
--- a/lib/Makefile
|
||||
+++ b/lib/Makefile
|
||||
@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4) += crc4.o
|
||||
obj-$(CONFIG_CRC7) += crc7.o
|
||||
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
|
||||
obj-$(CONFIG_CRC8) += crc8.o
|
||||
+obj-$(CONFIG_XXHASH) += xxhash.o
|
||||
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
|
||||
|
||||
obj-$(CONFIG_842_COMPRESS) += 842/
|
||||
diff --git a/lib/xxhash.c b/lib/xxhash.c
|
||||
new file mode 100644
|
||||
index 0000000..aa61e2a
|
||||
--- /dev/null
|
||||
+++ b/lib/xxhash.c
|
||||
@@ -0,0 +1,500 @@
|
||||
+/*
|
||||
+ * xxHash - Extremely Fast Hash algorithm
|
||||
+ * Copyright (C) 2012-2016, Yann Collet.
|
||||
+ *
|
||||
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
+ *
|
||||
+ * Redistribution and use in source and binary forms, with or without
|
||||
+ * modification, are permitted provided that the following conditions are
|
||||
+ * met:
|
||||
+ *
|
||||
+ * * Redistributions of source code must retain the above copyright
|
||||
+ * notice, this list of conditions and the following disclaimer.
|
||||
+ * * Redistributions in binary form must reproduce the above
|
||||
+ * copyright notice, this list of conditions and the following disclaimer
|
||||
+ * in the documentation and/or other materials provided with the
|
||||
+ * distribution.
|
||||
+ *
|
||||
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it under
|
||||
+ * the terms of the GNU General Public License version 2 as published by the
|
||||
+ * Free Software Foundation. This program is dual-licensed; you may select
|
||||
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
|
||||
+ * ("BSD").
|
||||
+ *
|
||||
+ * You can contact the author at:
|
||||
+ * - xxHash homepage: http://cyan4973.github.io/xxHash/
|
||||
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
+ */
|
||||
+
|
||||
+#include <asm/unaligned.h>
|
||||
+#include <linux/errno.h>
|
||||
+#include <linux/compiler.h>
|
||||
+#include <linux/kernel.h>
|
||||
+#include <linux/module.h>
|
||||
+#include <linux/string.h>
|
||||
+#include <linux/xxhash.h>
|
||||
+
|
||||
+/*-*************************************
|
||||
+ * Macros
|
||||
+ **************************************/
|
||||
+#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r)))
|
||||
+#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r)))
|
||||
+
|
||||
+#ifdef __LITTLE_ENDIAN
|
||||
+# define XXH_CPU_LITTLE_ENDIAN 1
|
||||
+#else
|
||||
+# define XXH_CPU_LITTLE_ENDIAN 0
|
||||
+#endif
|
||||
+
|
||||
+/*-*************************************
|
||||
+ * Constants
|
||||
+ **************************************/
|
||||
+static const uint32_t PRIME32_1 = 2654435761U;
|
||||
+static const uint32_t PRIME32_2 = 2246822519U;
|
||||
+static const uint32_t PRIME32_3 = 3266489917U;
|
||||
+static const uint32_t PRIME32_4 = 668265263U;
|
||||
+static const uint32_t PRIME32_5 = 374761393U;
|
||||
+
|
||||
+static const uint64_t PRIME64_1 = 11400714785074694791ULL;
|
||||
+static const uint64_t PRIME64_2 = 14029467366897019727ULL;
|
||||
+static const uint64_t PRIME64_3 = 1609587929392839161ULL;
|
||||
+static const uint64_t PRIME64_4 = 9650029242287828579ULL;
|
||||
+static const uint64_t PRIME64_5 = 2870177450012600261ULL;
|
||||
+
|
||||
+/*-**************************
|
||||
+ * Utils
|
||||
+ ***************************/
|
||||
+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
|
||||
+{
|
||||
+ memcpy(dst, src, sizeof(*dst));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_copy_state);
|
||||
+
|
||||
+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
|
||||
+{
|
||||
+ memcpy(dst, src, sizeof(*dst));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_copy_state);
|
||||
+
|
||||
+/*-***************************
|
||||
+ * Simple Hash Functions
|
||||
+ ****************************/
|
||||
+static uint32_t xxh32_round(uint32_t seed, const uint32_t input)
|
||||
+{
|
||||
+ seed += input * PRIME32_2;
|
||||
+ seed = xxh_rotl32(seed, 13);
|
||||
+ seed *= PRIME32_1;
|
||||
+ return seed;
|
||||
+}
|
||||
+
|
||||
+uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *b_end = p + len;
|
||||
+ uint32_t h32;
|
||||
+
|
||||
+ if (len >= 16) {
|
||||
+ const uint8_t *const limit = b_end - 16;
|
||||
+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
+ uint32_t v2 = seed + PRIME32_2;
|
||||
+ uint32_t v3 = seed + 0;
|
||||
+ uint32_t v4 = seed - PRIME32_1;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) +
|
||||
+ xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18);
|
||||
+ } else {
|
||||
+ h32 = seed + PRIME32_5;
|
||||
+ }
|
||||
+
|
||||
+ h32 += (uint32_t)len;
|
||||
+
|
||||
+ while (p + 4 <= b_end) {
|
||||
+ h32 += get_unaligned_le32(p) * PRIME32_3;
|
||||
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h32 += (*p) * PRIME32_5;
|
||||
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h32 ^= h32 >> 15;
|
||||
+ h32 *= PRIME32_2;
|
||||
+ h32 ^= h32 >> 13;
|
||||
+ h32 *= PRIME32_3;
|
||||
+ h32 ^= h32 >> 16;
|
||||
+
|
||||
+ return h32;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32);
|
||||
+
|
||||
+static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
|
||||
+{
|
||||
+ acc += input * PRIME64_2;
|
||||
+ acc = xxh_rotl64(acc, 31);
|
||||
+ acc *= PRIME64_1;
|
||||
+ return acc;
|
||||
+}
|
||||
+
|
||||
+static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val)
|
||||
+{
|
||||
+ val = xxh64_round(0, val);
|
||||
+ acc ^= val;
|
||||
+ acc = acc * PRIME64_1 + PRIME64_4;
|
||||
+ return acc;
|
||||
+}
|
||||
+
|
||||
+uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+ uint64_t h64;
|
||||
+
|
||||
+ if (len >= 32) {
|
||||
+ const uint8_t *const limit = b_end - 32;
|
||||
+ uint64_t v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
+ uint64_t v2 = seed + PRIME64_2;
|
||||
+ uint64_t v3 = seed + 0;
|
||||
+ uint64_t v4 = seed - PRIME64_1;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
|
||||
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
|
||||
+ h64 = xxh64_merge_round(h64, v1);
|
||||
+ h64 = xxh64_merge_round(h64, v2);
|
||||
+ h64 = xxh64_merge_round(h64, v3);
|
||||
+ h64 = xxh64_merge_round(h64, v4);
|
||||
+
|
||||
+ } else {
|
||||
+ h64 = seed + PRIME64_5;
|
||||
+ }
|
||||
+
|
||||
+ h64 += (uint64_t)len;
|
||||
+
|
||||
+ while (p + 8 <= b_end) {
|
||||
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
|
||||
+
|
||||
+ h64 ^= k1;
|
||||
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
|
||||
+ p += 8;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 4 <= b_end) {
|
||||
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
|
||||
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h64 ^= (*p) * PRIME64_5;
|
||||
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h64 ^= h64 >> 33;
|
||||
+ h64 *= PRIME64_2;
|
||||
+ h64 ^= h64 >> 29;
|
||||
+ h64 *= PRIME64_3;
|
||||
+ h64 ^= h64 >> 32;
|
||||
+
|
||||
+ return h64;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64);
|
||||
+
|
||||
+/*-**************************************************
|
||||
+ * Advanced Hash Functions
|
||||
+ ***************************************************/
|
||||
+void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
|
||||
+{
|
||||
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
|
||||
+ struct xxh32_state state;
|
||||
+
|
||||
+ memset(&state, 0, sizeof(state));
|
||||
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
|
||||
+ state.v2 = seed + PRIME32_2;
|
||||
+ state.v3 = seed + 0;
|
||||
+ state.v4 = seed - PRIME32_1;
|
||||
+ memcpy(statePtr, &state, sizeof(state));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_reset);
|
||||
+
|
||||
+void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
|
||||
+{
|
||||
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
|
||||
+ struct xxh64_state state;
|
||||
+
|
||||
+ memset(&state, 0, sizeof(state));
|
||||
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
|
||||
+ state.v2 = seed + PRIME64_2;
|
||||
+ state.v3 = seed + 0;
|
||||
+ state.v4 = seed - PRIME64_1;
|
||||
+ memcpy(statePtr, &state, sizeof(state));
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_reset);
|
||||
+
|
||||
+int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+
|
||||
+ if (input == NULL)
|
||||
+ return -EINVAL;
|
||||
+
|
||||
+ state->total_len_32 += (uint32_t)len;
|
||||
+ state->large_len |= (len >= 16) | (state->total_len_32 >= 16);
|
||||
+
|
||||
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
|
||||
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input, len);
|
||||
+ state->memsize += (uint32_t)len;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ if (state->memsize) { /* some data left from previous update */
|
||||
+ const uint32_t *p32 = state->mem32;
|
||||
+
|
||||
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input,
|
||||
+ 16 - state->memsize);
|
||||
+
|
||||
+ state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+ state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32));
|
||||
+ p32++;
|
||||
+
|
||||
+ p += 16-state->memsize;
|
||||
+ state->memsize = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (p <= b_end - 16) {
|
||||
+ const uint8_t *const limit = b_end - 16;
|
||||
+ uint32_t v1 = state->v1;
|
||||
+ uint32_t v2 = state->v2;
|
||||
+ uint32_t v3 = state->v3;
|
||||
+ uint32_t v4 = state->v4;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
|
||||
+ p += 4;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ state->v1 = v1;
|
||||
+ state->v2 = v2;
|
||||
+ state->v3 = v3;
|
||||
+ state->v4 = v4;
|
||||
+ }
|
||||
+
|
||||
+ if (p < b_end) {
|
||||
+ memcpy(state->mem32, p, (size_t)(b_end-p));
|
||||
+ state->memsize = (uint32_t)(b_end-p);
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_update);
|
||||
+
|
||||
+uint32_t xxh32_digest(const struct xxh32_state *state)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)state->mem32;
|
||||
+ const uint8_t *const b_end = (const uint8_t *)(state->mem32) +
|
||||
+ state->memsize;
|
||||
+ uint32_t h32;
|
||||
+
|
||||
+ if (state->large_len) {
|
||||
+ h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) +
|
||||
+ xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18);
|
||||
+ } else {
|
||||
+ h32 = state->v3 /* == seed */ + PRIME32_5;
|
||||
+ }
|
||||
+
|
||||
+ h32 += state->total_len_32;
|
||||
+
|
||||
+ while (p + 4 <= b_end) {
|
||||
+ h32 += get_unaligned_le32(p) * PRIME32_3;
|
||||
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h32 += (*p) * PRIME32_5;
|
||||
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h32 ^= h32 >> 15;
|
||||
+ h32 *= PRIME32_2;
|
||||
+ h32 ^= h32 >> 13;
|
||||
+ h32 *= PRIME32_3;
|
||||
+ h32 ^= h32 >> 16;
|
||||
+
|
||||
+ return h32;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh32_digest);
|
||||
+
|
||||
+int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)input;
|
||||
+ const uint8_t *const b_end = p + len;
|
||||
+
|
||||
+ if (input == NULL)
|
||||
+ return -EINVAL;
|
||||
+
|
||||
+ state->total_len += len;
|
||||
+
|
||||
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
|
||||
+ memcpy(((uint8_t *)state->mem64) + state->memsize, input, len);
|
||||
+ state->memsize += (uint32_t)len;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ if (state->memsize) { /* tmp buffer is full */
|
||||
+ uint64_t *p64 = state->mem64;
|
||||
+
|
||||
+ memcpy(((uint8_t *)p64) + state->memsize, input,
|
||||
+ 32 - state->memsize);
|
||||
+
|
||||
+ state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64));
|
||||
+ p64++;
|
||||
+ state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64));
|
||||
+
|
||||
+ p += 32 - state->memsize;
|
||||
+ state->memsize = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 32 <= b_end) {
|
||||
+ const uint8_t *const limit = b_end - 32;
|
||||
+ uint64_t v1 = state->v1;
|
||||
+ uint64_t v2 = state->v2;
|
||||
+ uint64_t v3 = state->v3;
|
||||
+ uint64_t v4 = state->v4;
|
||||
+
|
||||
+ do {
|
||||
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
|
||||
+ p += 8;
|
||||
+ } while (p <= limit);
|
||||
+
|
||||
+ state->v1 = v1;
|
||||
+ state->v2 = v2;
|
||||
+ state->v3 = v3;
|
||||
+ state->v4 = v4;
|
||||
+ }
|
||||
+
|
||||
+ if (p < b_end) {
|
||||
+ memcpy(state->mem64, p, (size_t)(b_end-p));
|
||||
+ state->memsize = (uint32_t)(b_end - p);
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_update);
|
||||
+
|
||||
+uint64_t xxh64_digest(const struct xxh64_state *state)
|
||||
+{
|
||||
+ const uint8_t *p = (const uint8_t *)state->mem64;
|
||||
+ const uint8_t *const b_end = (const uint8_t *)state->mem64 +
|
||||
+ state->memsize;
|
||||
+ uint64_t h64;
|
||||
+
|
||||
+ if (state->total_len >= 32) {
|
||||
+ const uint64_t v1 = state->v1;
|
||||
+ const uint64_t v2 = state->v2;
|
||||
+ const uint64_t v3 = state->v3;
|
||||
+ const uint64_t v4 = state->v4;
|
||||
+
|
||||
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
|
||||
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
|
||||
+ h64 = xxh64_merge_round(h64, v1);
|
||||
+ h64 = xxh64_merge_round(h64, v2);
|
||||
+ h64 = xxh64_merge_round(h64, v3);
|
||||
+ h64 = xxh64_merge_round(h64, v4);
|
||||
+ } else {
|
||||
+ h64 = state->v3 + PRIME64_5;
|
||||
+ }
|
||||
+
|
||||
+ h64 += (uint64_t)state->total_len;
|
||||
+
|
||||
+ while (p + 8 <= b_end) {
|
||||
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
|
||||
+
|
||||
+ h64 ^= k1;
|
||||
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
|
||||
+ p += 8;
|
||||
+ }
|
||||
+
|
||||
+ if (p + 4 <= b_end) {
|
||||
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
|
||||
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
|
||||
+ p += 4;
|
||||
+ }
|
||||
+
|
||||
+ while (p < b_end) {
|
||||
+ h64 ^= (*p) * PRIME64_5;
|
||||
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
|
||||
+ p++;
|
||||
+ }
|
||||
+
|
||||
+ h64 ^= h64 >> 33;
|
||||
+ h64 *= PRIME64_2;
|
||||
+ h64 ^= h64 >> 29;
|
||||
+ h64 *= PRIME64_3;
|
||||
+ h64 ^= h64 >> 32;
|
||||
+
|
||||
+ return h64;
|
||||
+}
|
||||
+EXPORT_SYMBOL(xxh64_digest);
|
||||
+
|
||||
+MODULE_LICENSE("Dual BSD/GPL");
|
||||
+MODULE_DESCRIPTION("xxHash");
|
||||
--
|
||||
2.9.3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,740 @@
|
||||
From 8a9dddfbf6551afea73911e367dd4be64d62b9fd Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Mon, 17 Jul 2017 17:08:39 -0700
|
||||
Subject: [PATCH v5 3/5] btrfs: Add zstd support
|
||||
|
||||
Add zstd compression and decompression support to BtrFS. zstd at its
|
||||
fastest level compresses almost as well as zlib, while offering much
|
||||
faster compression and decompression, approaching lzo speeds.
|
||||
|
||||
I benchmarked btrfs with zstd compression against no compression, lzo
|
||||
compression, and zlib compression. I benchmarked two scenarios. Copying
|
||||
a set of files to btrfs, and then reading the files. Copying a tarball
|
||||
to btrfs, extracting it to btrfs, and then reading the extracted files.
|
||||
After every operation, I call `sync` and include the sync time.
|
||||
Between every pair of operations I unmount and remount the filesystem
|
||||
to avoid caching. The benchmark files can be found in the upstream
|
||||
zstd source repository under
|
||||
`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
|
||||
[1] [2].
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD.
|
||||
|
||||
The first compression benchmark is copying 10 copies of the unzipped
|
||||
Silesia corpus [3] into a BtrFS filesystem mounted with
|
||||
`-o compress-force=Method`. The decompression benchmark times how long
|
||||
it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
|
||||
measured by comparing the output of `df` and `du`. See the benchmark file
|
||||
[1] for details. I benchmarked multiple zstd compression levels, although
|
||||
the patch uses zstd level 1.
|
||||
|
||||
| Method | Ratio | Compression MB/s | Decompression speed |
|
||||
|---------|-------|------------------|---------------------|
|
||||
| None | 0.99 | 504 | 686 |
|
||||
| lzo | 1.66 | 398 | 442 |
|
||||
| zlib | 2.58 | 65 | 241 |
|
||||
| zstd 1 | 2.57 | 260 | 383 |
|
||||
| zstd 3 | 2.71 | 174 | 408 |
|
||||
| zstd 6 | 2.87 | 70 | 398 |
|
||||
| zstd 9 | 2.92 | 43 | 406 |
|
||||
| zstd 12 | 2.93 | 21 | 408 |
|
||||
| zstd 15 | 3.01 | 11 | 354 |
|
||||
|
||||
The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
|
||||
measures the compression ratio, extracts the tar, and deletes the tar.
|
||||
Then it measures the compression ratio again, and `tar`s the extracted
|
||||
files into `/dev/null`. See the benchmark file [2] for details.
|
||||
|
||||
| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
|
||||
|--------|-----------|---------------|----------|------------|----------|
|
||||
| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 |
|
||||
| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 |
|
||||
| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 |
|
||||
| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 |
|
||||
|
||||
[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
|
||||
[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||
[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v2 -> v3:
|
||||
- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff
|
||||
- Change default compression level for BtrFS to 3
|
||||
|
||||
v3 -> v4:
|
||||
- Add missing includes, which fixes the aarch64 build
|
||||
- Fix minor linter warnings
|
||||
|
||||
fs/btrfs/Kconfig | 2 +
|
||||
fs/btrfs/Makefile | 2 +-
|
||||
fs/btrfs/compression.c | 1 +
|
||||
fs/btrfs/compression.h | 6 +-
|
||||
fs/btrfs/ctree.h | 1 +
|
||||
fs/btrfs/disk-io.c | 2 +
|
||||
fs/btrfs/ioctl.c | 6 +-
|
||||
fs/btrfs/props.c | 6 +
|
||||
fs/btrfs/super.c | 12 +-
|
||||
fs/btrfs/sysfs.c | 2 +
|
||||
fs/btrfs/zstd.c | 432 +++++++++++++++++++++++++++++++++++++++++++++
|
||||
include/uapi/linux/btrfs.h | 8 +-
|
||||
12 files changed, 468 insertions(+), 12 deletions(-)
|
||||
create mode 100644 fs/btrfs/zstd.c
|
||||
|
||||
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
|
||||
index 80e9c18..a26c63b 100644
|
||||
--- a/fs/btrfs/Kconfig
|
||||
+++ b/fs/btrfs/Kconfig
|
||||
@@ -6,6 +6,8 @@ config BTRFS_FS
|
||||
select ZLIB_DEFLATE
|
||||
select LZO_COMPRESS
|
||||
select LZO_DECOMPRESS
|
||||
+ select ZSTD_COMPRESS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
select RAID6_PQ
|
||||
select XOR_BLOCKS
|
||||
select SRCU
|
||||
diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
|
||||
index 128ce17..962a95a 100644
|
||||
--- a/fs/btrfs/Makefile
|
||||
+++ b/fs/btrfs/Makefile
|
||||
@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
|
||||
transaction.o inode.o file.o tree-defrag.o \
|
||||
extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \
|
||||
extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
|
||||
- export.o tree-log.o free-space-cache.o zlib.o lzo.o \
|
||||
+ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \
|
||||
compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
|
||||
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
|
||||
uuid-tree.o props.o hash.o free-space-tree.o
|
||||
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
|
||||
index d2ef9ac..4ff42d1 100644
|
||||
--- a/fs/btrfs/compression.c
|
||||
+++ b/fs/btrfs/compression.c
|
||||
@@ -704,6 +704,7 @@ static struct {
|
||||
static const struct btrfs_compress_op * const btrfs_compress_op[] = {
|
||||
&btrfs_zlib_compress,
|
||||
&btrfs_lzo_compress,
|
||||
+ &btrfs_zstd_compress,
|
||||
};
|
||||
|
||||
void __init btrfs_init_compress(void)
|
||||
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
|
||||
index 87f6d33..2269e00 100644
|
||||
--- a/fs/btrfs/compression.h
|
||||
+++ b/fs/btrfs/compression.h
|
||||
@@ -99,8 +99,9 @@ enum btrfs_compression_type {
|
||||
BTRFS_COMPRESS_NONE = 0,
|
||||
BTRFS_COMPRESS_ZLIB = 1,
|
||||
BTRFS_COMPRESS_LZO = 2,
|
||||
- BTRFS_COMPRESS_TYPES = 2,
|
||||
- BTRFS_COMPRESS_LAST = 3,
|
||||
+ BTRFS_COMPRESS_ZSTD = 3,
|
||||
+ BTRFS_COMPRESS_TYPES = 3,
|
||||
+ BTRFS_COMPRESS_LAST = 4,
|
||||
};
|
||||
|
||||
struct btrfs_compress_op {
|
||||
@@ -128,5 +129,6 @@ struct btrfs_compress_op {
|
||||
|
||||
extern const struct btrfs_compress_op btrfs_zlib_compress;
|
||||
extern const struct btrfs_compress_op btrfs_lzo_compress;
|
||||
+extern const struct btrfs_compress_op btrfs_zstd_compress;
|
||||
|
||||
#endif
|
||||
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
|
||||
index 3f3eb7b..845d77c 100644
|
||||
--- a/fs/btrfs/ctree.h
|
||||
+++ b/fs/btrfs/ctree.h
|
||||
@@ -270,6 +270,7 @@ struct btrfs_super_block {
|
||||
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
|
||||
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
|
||||
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
|
||||
+ BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \
|
||||
BTRFS_FEATURE_INCOMPAT_RAID56 | \
|
||||
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
|
||||
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
|
||||
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
|
||||
index 080e2eb..04632f4 100644
|
||||
--- a/fs/btrfs/disk-io.c
|
||||
+++ b/fs/btrfs/disk-io.c
|
||||
@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
|
||||
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
|
||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
|
||||
+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
|
||||
+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
|
||||
|
||||
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
|
||||
btrfs_info(fs_info, "has skinny extents");
|
||||
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
|
||||
index fa1b78c..b9963d9 100644
|
||||
--- a/fs/btrfs/ioctl.c
|
||||
+++ b/fs/btrfs/ioctl.c
|
||||
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
|
||||
|
||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
comp = "lzo";
|
||||
- else
|
||||
+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZLIB)
|
||||
comp = "zlib";
|
||||
+ else
|
||||
+ comp = "zstd";
|
||||
ret = btrfs_set_prop(inode, "btrfs.compression",
|
||||
comp, strlen(comp), 0);
|
||||
if (ret)
|
||||
@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
|
||||
|
||||
if (range->compress_type == BTRFS_COMPRESS_LZO) {
|
||||
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
|
||||
+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
|
||||
+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
|
||||
}
|
||||
|
||||
ret = defrag_count;
|
||||
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
|
||||
index 4b23ae5..20631e9 100644
|
||||
--- a/fs/btrfs/props.c
|
||||
+++ b/fs/btrfs/props.c
|
||||
@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
|
||||
return 0;
|
||||
else if (!strncmp("zlib", value, len))
|
||||
return 0;
|
||||
+ else if (!strncmp("zstd", value, len))
|
||||
+ return 0;
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
|
||||
type = BTRFS_COMPRESS_LZO;
|
||||
else if (!strncmp("zlib", value, len))
|
||||
type = BTRFS_COMPRESS_ZLIB;
|
||||
+ else if (!strncmp("zstd", value, len))
|
||||
+ type = BTRFS_COMPRESS_ZSTD;
|
||||
else
|
||||
return -EINVAL;
|
||||
|
||||
@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
|
||||
return "zlib";
|
||||
case BTRFS_COMPRESS_LZO:
|
||||
return "lzo";
|
||||
+ case BTRFS_COMPRESS_ZSTD:
|
||||
+ return "zstd";
|
||||
}
|
||||
|
||||
return NULL;
|
||||
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
|
||||
index 12540b6..c370dea 100644
|
||||
--- a/fs/btrfs/super.c
|
||||
+++ b/fs/btrfs/super.c
|
||||
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
|
||||
btrfs_clear_opt(info->mount_opt, NODATASUM);
|
||||
btrfs_set_fs_incompat(info, COMPRESS_LZO);
|
||||
no_compress = 0;
|
||||
+ } else if (strcmp(args[0].from, "zstd") == 0) {
|
||||
+ compress_type = "zstd";
|
||||
+ info->compress_type = BTRFS_COMPRESS_ZSTD;
|
||||
+ btrfs_set_opt(info->mount_opt, COMPRESS);
|
||||
+ btrfs_clear_opt(info->mount_opt, NODATACOW);
|
||||
+ btrfs_clear_opt(info->mount_opt, NODATASUM);
|
||||
+ btrfs_set_fs_incompat(info, COMPRESS_ZSTD);
|
||||
+ no_compress = 0;
|
||||
} else if (strncmp(args[0].from, "no", 2) == 0) {
|
||||
compress_type = "no";
|
||||
btrfs_clear_opt(info->mount_opt, COMPRESS);
|
||||
@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
|
||||
if (btrfs_test_opt(info, COMPRESS)) {
|
||||
if (info->compress_type == BTRFS_COMPRESS_ZLIB)
|
||||
compress_type = "zlib";
|
||||
- else
|
||||
+ else if (info->compress_type == BTRFS_COMPRESS_LZO)
|
||||
compress_type = "lzo";
|
||||
+ else
|
||||
+ compress_type = "zstd";
|
||||
if (btrfs_test_opt(info, FORCE_COMPRESS))
|
||||
seq_printf(seq, ",compress-force=%s", compress_type);
|
||||
else
|
||||
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
|
||||
index c2d5f35..2b6d37c 100644
|
||||
--- a/fs/btrfs/sysfs.c
|
||||
+++ b/fs/btrfs/sysfs.c
|
||||
@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
|
||||
+BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
|
||||
BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
|
||||
@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = {
|
||||
BTRFS_FEAT_ATTR_PTR(default_subvol),
|
||||
BTRFS_FEAT_ATTR_PTR(mixed_groups),
|
||||
BTRFS_FEAT_ATTR_PTR(compress_lzo),
|
||||
+ BTRFS_FEAT_ATTR_PTR(compress_zstd),
|
||||
BTRFS_FEAT_ATTR_PTR(big_metadata),
|
||||
BTRFS_FEAT_ATTR_PTR(extended_iref),
|
||||
BTRFS_FEAT_ATTR_PTR(raid56),
|
||||
diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
|
||||
new file mode 100644
|
||||
index 0000000..607ce47
|
||||
--- /dev/null
|
||||
+++ b/fs/btrfs/zstd.c
|
||||
@@ -0,0 +1,432 @@
|
||||
+/*
|
||||
+ * Copyright (c) 2016-present, Facebook, Inc.
|
||||
+ * All rights reserved.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public
|
||||
+ * License v2 as published by the Free Software Foundation.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ * General Public License for more details.
|
||||
+ */
|
||||
+#include <linux/bio.h>
|
||||
+#include <linux/err.h>
|
||||
+#include <linux/init.h>
|
||||
+#include <linux/kernel.h>
|
||||
+#include <linux/mm.h>
|
||||
+#include <linux/pagemap.h>
|
||||
+#include <linux/refcount.h>
|
||||
+#include <linux/sched.h>
|
||||
+#include <linux/slab.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include "compression.h"
|
||||
+
|
||||
+#define ZSTD_BTRFS_MAX_WINDOWLOG 17
|
||||
+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
|
||||
+#define ZSTD_BTRFS_DEFAULT_LEVEL 3
|
||||
+
|
||||
+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
|
||||
+{
|
||||
+ ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
|
||||
+ src_len, 0);
|
||||
+
|
||||
+ if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
|
||||
+ params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
|
||||
+ WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
|
||||
+ return params;
|
||||
+}
|
||||
+
|
||||
+struct workspace {
|
||||
+ void *mem;
|
||||
+ size_t size;
|
||||
+ char *buf;
|
||||
+ struct list_head list;
|
||||
+};
|
||||
+
|
||||
+static void zstd_free_workspace(struct list_head *ws)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+
|
||||
+ kvfree(workspace->mem);
|
||||
+ kfree(workspace->buf);
|
||||
+ kfree(workspace);
|
||||
+}
|
||||
+
|
||||
+static struct list_head *zstd_alloc_workspace(void)
|
||||
+{
|
||||
+ ZSTD_parameters params =
|
||||
+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
|
||||
+ struct workspace *workspace;
|
||||
+
|
||||
+ workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
|
||||
+ if (!workspace)
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+
|
||||
+ workspace->size = max_t(size_t,
|
||||
+ ZSTD_CStreamWorkspaceBound(params.cParams),
|
||||
+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
|
||||
+ workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
|
||||
+ workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
|
||||
+ if (!workspace->mem || !workspace->buf)
|
||||
+ goto fail;
|
||||
+
|
||||
+ INIT_LIST_HEAD(&workspace->list);
|
||||
+
|
||||
+ return &workspace->list;
|
||||
+fail:
|
||||
+ zstd_free_workspace(&workspace->list);
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress_pages(struct list_head *ws,
|
||||
+ struct address_space *mapping,
|
||||
+ u64 start,
|
||||
+ struct page **pages,
|
||||
+ unsigned long *out_pages,
|
||||
+ unsigned long *total_in,
|
||||
+ unsigned long *total_out)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ ZSTD_CStream *stream;
|
||||
+ int ret = 0;
|
||||
+ int nr_pages = 0;
|
||||
+ struct page *in_page = NULL; /* The current page to read */
|
||||
+ struct page *out_page = NULL; /* The current page to write to */
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+ unsigned long tot_in = 0;
|
||||
+ unsigned long tot_out = 0;
|
||||
+ unsigned long len = *total_out;
|
||||
+ const unsigned long nr_dest_pages = *out_pages;
|
||||
+ unsigned long max_out = nr_dest_pages * PAGE_SIZE;
|
||||
+ ZSTD_parameters params = zstd_get_btrfs_parameters(len);
|
||||
+
|
||||
+ *out_pages = 0;
|
||||
+ *total_out = 0;
|
||||
+ *total_in = 0;
|
||||
+
|
||||
+ /* Initialize the stream */
|
||||
+ stream = ZSTD_initCStream(params, len, workspace->mem,
|
||||
+ workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_warn("BTRFS: ZSTD_initCStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* map in the first page of input data */
|
||||
+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
|
||||
+ in_buf.src = kmap(in_page);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
|
||||
+
|
||||
+
|
||||
+ /* Allocate and map in the output buffer */
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_compressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* Check to see if we are making it bigger */
|
||||
+ if (tot_in + in_buf.pos > 8192 &&
|
||||
+ tot_in + in_buf.pos <
|
||||
+ tot_out + out_buf.pos) {
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* We've reached the end of our output range */
|
||||
+ if (out_buf.pos >= max_out) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ /* Check if we need more output space */
|
||||
+ if (out_buf.pos == out_buf.size) {
|
||||
+ tot_out += PAGE_SIZE;
|
||||
+ max_out -= PAGE_SIZE;
|
||||
+ kunmap(out_page);
|
||||
+ if (nr_pages == nr_dest_pages) {
|
||||
+ out_page = NULL;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+ }
|
||||
+
|
||||
+ /* We've reached the end of the input */
|
||||
+ if (in_buf.pos >= len) {
|
||||
+ tot_in += in_buf.pos;
|
||||
+ break;
|
||||
+ }
|
||||
+
|
||||
+ /* Check if we need more input */
|
||||
+ if (in_buf.pos == in_buf.size) {
|
||||
+ tot_in += PAGE_SIZE;
|
||||
+ kunmap(in_page);
|
||||
+ put_page(in_page);
|
||||
+
|
||||
+ start += PAGE_SIZE;
|
||||
+ len -= PAGE_SIZE;
|
||||
+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
|
||||
+ in_buf.src = kmap(in_page);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
|
||||
+ }
|
||||
+ }
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_endStream(stream, &out_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_endStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ if (ret2 == 0) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ break;
|
||||
+ }
|
||||
+ if (out_buf.pos >= max_out) {
|
||||
+ tot_out += out_buf.pos;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ tot_out += PAGE_SIZE;
|
||||
+ max_out -= PAGE_SIZE;
|
||||
+ kunmap(out_page);
|
||||
+ if (nr_pages == nr_dest_pages) {
|
||||
+ out_page = NULL;
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
|
||||
+ if (out_page == NULL) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+ pages[nr_pages++] = out_page;
|
||||
+ out_buf.dst = kmap(out_page);
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
|
||||
+ }
|
||||
+
|
||||
+ if (tot_out >= tot_in) {
|
||||
+ ret = -E2BIG;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ret = 0;
|
||||
+ *total_in = tot_in;
|
||||
+ *total_out = tot_out;
|
||||
+out:
|
||||
+ *out_pages = nr_pages;
|
||||
+ /* Cleanup */
|
||||
+ if (in_page) {
|
||||
+ kunmap(in_page);
|
||||
+ put_page(in_page);
|
||||
+ }
|
||||
+ if (out_page)
|
||||
+ kunmap(out_page);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ struct page **pages_in = cb->compressed_pages;
|
||||
+ u64 disk_start = cb->start;
|
||||
+ struct bio *orig_bio = cb->orig_bio;
|
||||
+ size_t srclen = cb->compressed_len;
|
||||
+ ZSTD_DStream *stream;
|
||||
+ int ret = 0;
|
||||
+ unsigned long page_in_index = 0;
|
||||
+ unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
|
||||
+ unsigned long buf_start;
|
||||
+ unsigned long total_out = 0;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+
|
||||
+ stream = ZSTD_initDStream(
|
||||
+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_debug("BTRFS: ZSTD_initDStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+
|
||||
+ in_buf.src = kmap(pages_in[page_in_index]);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
|
||||
+
|
||||
+ out_buf.dst = workspace->buf;
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+
|
||||
+ while (1) {
|
||||
+ size_t ret2;
|
||||
+
|
||||
+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+ buf_start = total_out;
|
||||
+ total_out += out_buf.pos;
|
||||
+ out_buf.pos = 0;
|
||||
+
|
||||
+ ret = btrfs_decompress_buf2page(out_buf.dst, buf_start,
|
||||
+ total_out, disk_start, orig_bio);
|
||||
+ if (ret == 0)
|
||||
+ break;
|
||||
+
|
||||
+ if (in_buf.pos >= srclen)
|
||||
+ break;
|
||||
+
|
||||
+ /* Check if we've hit the end of a frame */
|
||||
+ if (ret2 == 0)
|
||||
+ break;
|
||||
+
|
||||
+ if (in_buf.pos == in_buf.size) {
|
||||
+ kunmap(pages_in[page_in_index++]);
|
||||
+ if (page_in_index >= total_pages_in) {
|
||||
+ in_buf.src = NULL;
|
||||
+ ret = -EIO;
|
||||
+ goto done;
|
||||
+ }
|
||||
+ srclen -= PAGE_SIZE;
|
||||
+ in_buf.src = kmap(pages_in[page_in_index]);
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
|
||||
+ }
|
||||
+ }
|
||||
+ ret = 0;
|
||||
+ zero_fill_bio(orig_bio);
|
||||
+done:
|
||||
+ if (in_buf.src)
|
||||
+ kunmap(pages_in[page_in_index]);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
|
||||
+ struct page *dest_page,
|
||||
+ unsigned long start_byte,
|
||||
+ size_t srclen, size_t destlen)
|
||||
+{
|
||||
+ struct workspace *workspace = list_entry(ws, struct workspace, list);
|
||||
+ ZSTD_DStream *stream;
|
||||
+ int ret = 0;
|
||||
+ size_t ret2;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+ unsigned long total_out = 0;
|
||||
+ unsigned long pg_offset = 0;
|
||||
+ char *kaddr;
|
||||
+
|
||||
+ stream = ZSTD_initDStream(
|
||||
+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
||||
+ if (!stream) {
|
||||
+ pr_warn("BTRFS: ZSTD_initDStream failed\n");
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+
|
||||
+ destlen = min_t(size_t, destlen, PAGE_SIZE);
|
||||
+
|
||||
+ in_buf.src = data_in;
|
||||
+ in_buf.pos = 0;
|
||||
+ in_buf.size = srclen;
|
||||
+
|
||||
+ out_buf.dst = workspace->buf;
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+
|
||||
+ ret2 = 1;
|
||||
+ while (pg_offset < destlen && in_buf.pos < in_buf.size) {
|
||||
+ unsigned long buf_start;
|
||||
+ unsigned long buf_offset;
|
||||
+ unsigned long bytes;
|
||||
+
|
||||
+ /* Check if the frame is over and we still need more input */
|
||||
+ if (ret2 == 0) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ if (ZSTD_isError(ret2)) {
|
||||
+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
|
||||
+ ZSTD_getErrorCode(ret2));
|
||||
+ ret = -EIO;
|
||||
+ goto finish;
|
||||
+ }
|
||||
+
|
||||
+ buf_start = total_out;
|
||||
+ total_out += out_buf.pos;
|
||||
+ out_buf.pos = 0;
|
||||
+
|
||||
+ if (total_out <= start_byte)
|
||||
+ continue;
|
||||
+
|
||||
+ if (total_out > start_byte && buf_start < start_byte)
|
||||
+ buf_offset = start_byte - buf_start;
|
||||
+ else
|
||||
+ buf_offset = 0;
|
||||
+
|
||||
+ bytes = min_t(unsigned long, destlen - pg_offset,
|
||||
+ out_buf.size - buf_offset);
|
||||
+
|
||||
+ kaddr = kmap_atomic(dest_page);
|
||||
+ memcpy(kaddr + pg_offset, out_buf.dst + buf_offset, bytes);
|
||||
+ kunmap_atomic(kaddr);
|
||||
+
|
||||
+ pg_offset += bytes;
|
||||
+ }
|
||||
+ ret = 0;
|
||||
+finish:
|
||||
+ if (pg_offset < destlen) {
|
||||
+ kaddr = kmap_atomic(dest_page);
|
||||
+ memset(kaddr + pg_offset, 0, destlen - pg_offset);
|
||||
+ kunmap_atomic(kaddr);
|
||||
+ }
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+const struct btrfs_compress_op btrfs_zstd_compress = {
|
||||
+ .alloc_workspace = zstd_alloc_workspace,
|
||||
+ .free_workspace = zstd_free_workspace,
|
||||
+ .compress_pages = zstd_compress_pages,
|
||||
+ .decompress_bio = zstd_decompress_bio,
|
||||
+ .decompress = zstd_decompress,
|
||||
+};
|
||||
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
|
||||
index 9aa74f3..378230c 100644
|
||||
--- a/include/uapi/linux/btrfs.h
|
||||
+++ b/include/uapi/linux/btrfs.h
|
||||
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
|
||||
#define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1)
|
||||
#define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2)
|
||||
#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3)
|
||||
-/*
|
||||
- * some patches floated around with a second compression method
|
||||
- * lets save that incompat here for when they do get in
|
||||
- * Note we don't actually support it, we're just reserving the
|
||||
- * number
|
||||
- */
|
||||
-#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
|
||||
+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
|
||||
|
||||
/*
|
||||
* older kernels tried to do bigger metadata blocks, but the
|
||||
--
|
||||
2.9.3
|
||||
@@ -0,0 +1,306 @@
|
||||
From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001
|
||||
From: Sean Purcell <me@seanp.xyz>
|
||||
Date: Mon, 17 Jul 2017 17:08:59 -0700
|
||||
Subject: [PATCH v5 4/5] squashfs: Add zstd support
|
||||
|
||||
Add zstd compression and decompression support to SquashFS. zstd is a
|
||||
great fit for SquashFS because it can compress at ratios approaching xz,
|
||||
while decompressing twice as fast as zlib. For SquashFS in particular,
|
||||
it can decompress as fast as lzo and lz4. It also has the flexibility
|
||||
to turn down the compression ratio for faster compression times.
|
||||
|
||||
The compression benchmark is run on the file tree from the SquashFS archive
|
||||
found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
|
||||
default block size (128 KB) and and various compression algorithms/levels.
|
||||
xz and zstd are also benchmarked with 256 KB blocks. The decompression
|
||||
benchmark times how long it takes to `tar` the file tree into `/dev/null`.
|
||||
See the benchmark file in the upstream zstd source repository located under
|
||||
`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
|
||||
|
||||
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||
16 GB of RAM, and a SSD.
|
||||
|
||||
| Method | Ratio | Compression MB/s | Decompression MB/s |
|
||||
|----------------|-------|------------------|--------------------|
|
||||
| gzip | 2.92 | 15 | 128 |
|
||||
| lzo | 2.64 | 9.5 | 217 |
|
||||
| lz4 | 2.12 | 94 | 218 |
|
||||
| xz | 3.43 | 5.5 | 35 |
|
||||
| xz 256 KB | 3.53 | 5.4 | 40 |
|
||||
| zstd 1 | 2.71 | 96 | 210 |
|
||||
| zstd 5 | 2.93 | 69 | 198 |
|
||||
| zstd 10 | 3.01 | 41 | 225 |
|
||||
| zstd 15 | 3.13 | 11.4 | 224 |
|
||||
| zstd 16 256 KB | 3.24 | 8.1 | 210 |
|
||||
|
||||
This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
|
||||
taking over the submission process.
|
||||
|
||||
[1] http://releases.ubuntu.com/16.10/
|
||||
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
|
||||
|
||||
zstd source repository: https://github.com/facebook/zstd
|
||||
|
||||
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v3 -> v4:
|
||||
- Fix minor linter warnings
|
||||
|
||||
v4 -> v5:
|
||||
- Fix ZSTD_DStream initialization code in squashfs
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
|
||||
fs/squashfs/Kconfig | 14 +++++
|
||||
fs/squashfs/Makefile | 1 +
|
||||
fs/squashfs/decompressor.c | 7 +++
|
||||
fs/squashfs/decompressor.h | 4 ++
|
||||
fs/squashfs/squashfs_fs.h | 1 +
|
||||
fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++
|
||||
6 files changed, 178 insertions(+)
|
||||
create mode 100644 fs/squashfs/zstd_wrapper.c
|
||||
|
||||
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
|
||||
index ffb093e..1adb334 100644
|
||||
--- a/fs/squashfs/Kconfig
|
||||
+++ b/fs/squashfs/Kconfig
|
||||
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
|
||||
|
||||
If unsure, say N.
|
||||
|
||||
+config SQUASHFS_ZSTD
|
||||
+ bool "Include support for ZSTD compressed file systems"
|
||||
+ depends on SQUASHFS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
+ help
|
||||
+ Saying Y here includes support for reading Squashfs file systems
|
||||
+ compressed with ZSTD compression. ZSTD gives better compression than
|
||||
+ the default ZLIB compression, while using less CPU.
|
||||
+
|
||||
+ ZSTD is not the standard compression used in Squashfs and so most
|
||||
+ file systems will be readable without selecting this option.
|
||||
+
|
||||
+ If unsure, say N.
|
||||
+
|
||||
config SQUASHFS_4K_DEVBLK_SIZE
|
||||
bool "Use 4K device block size?"
|
||||
depends on SQUASHFS
|
||||
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
|
||||
index 246a6f3..6655631 100644
|
||||
--- a/fs/squashfs/Makefile
|
||||
+++ b/fs/squashfs/Makefile
|
||||
@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
|
||||
squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o
|
||||
+squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o
|
||||
diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
|
||||
index d2bc136..8366398 100644
|
||||
--- a/fs/squashfs/decompressor.c
|
||||
+++ b/fs/squashfs/decompressor.c
|
||||
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
|
||||
};
|
||||
#endif
|
||||
|
||||
+#ifndef CONFIG_SQUASHFS_ZSTD
|
||||
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
||||
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
|
||||
+};
|
||||
+#endif
|
||||
+
|
||||
static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
|
||||
NULL, NULL, NULL, NULL, 0, "unknown", 0
|
||||
};
|
||||
@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = {
|
||||
&squashfs_lzo_comp_ops,
|
||||
&squashfs_xz_comp_ops,
|
||||
&squashfs_lzma_unsupported_comp_ops,
|
||||
+ &squashfs_zstd_comp_ops,
|
||||
&squashfs_unknown_comp_ops
|
||||
};
|
||||
|
||||
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
|
||||
index a25713c..0f5a8e4 100644
|
||||
--- a/fs/squashfs/decompressor.h
|
||||
+++ b/fs/squashfs/decompressor.h
|
||||
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
|
||||
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
|
||||
#endif
|
||||
|
||||
+#ifdef CONFIG_SQUASHFS_ZSTD
|
||||
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
|
||||
+#endif
|
||||
+
|
||||
#endif
|
||||
diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
|
||||
index 506f4ba..24d12fd 100644
|
||||
--- a/fs/squashfs/squashfs_fs.h
|
||||
+++ b/fs/squashfs/squashfs_fs.h
|
||||
@@ -241,6 +241,7 @@ struct meta_index {
|
||||
#define LZO_COMPRESSION 3
|
||||
#define XZ_COMPRESSION 4
|
||||
#define LZ4_COMPRESSION 5
|
||||
+#define ZSTD_COMPRESSION 6
|
||||
|
||||
struct squashfs_super_block {
|
||||
__le32 s_magic;
|
||||
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
|
||||
new file mode 100644
|
||||
index 0000000..eeaabf8
|
||||
--- /dev/null
|
||||
+++ b/fs/squashfs/zstd_wrapper.c
|
||||
@@ -0,0 +1,151 @@
|
||||
+/*
|
||||
+ * Squashfs - a compressed read only filesystem for Linux
|
||||
+ *
|
||||
+ * Copyright (c) 2016-present, Facebook, Inc.
|
||||
+ * All rights reserved.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.c
|
||||
+ */
|
||||
+
|
||||
+#include <linux/mutex.h>
|
||||
+#include <linux/buffer_head.h>
|
||||
+#include <linux/slab.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include <linux/vmalloc.h>
|
||||
+
|
||||
+#include "squashfs_fs.h"
|
||||
+#include "squashfs_fs_sb.h"
|
||||
+#include "squashfs.h"
|
||||
+#include "decompressor.h"
|
||||
+#include "page_actor.h"
|
||||
+
|
||||
+struct workspace {
|
||||
+ void *mem;
|
||||
+ size_t mem_size;
|
||||
+ size_t window_size;
|
||||
+};
|
||||
+
|
||||
+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
|
||||
+{
|
||||
+ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL);
|
||||
+
|
||||
+ if (wksp == NULL)
|
||||
+ goto failed;
|
||||
+ wksp->window_size = max_t(size_t,
|
||||
+ msblk->block_size, SQUASHFS_METADATA_SIZE);
|
||||
+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
|
||||
+ wksp->mem = vmalloc(wksp->mem_size);
|
||||
+ if (wksp->mem == NULL)
|
||||
+ goto failed;
|
||||
+
|
||||
+ return wksp;
|
||||
+
|
||||
+failed:
|
||||
+ ERROR("Failed to allocate zstd workspace\n");
|
||||
+ kfree(wksp);
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+}
|
||||
+
|
||||
+
|
||||
+static void zstd_free(void *strm)
|
||||
+{
|
||||
+ struct workspace *wksp = strm;
|
||||
+
|
||||
+ if (wksp)
|
||||
+ vfree(wksp->mem);
|
||||
+ kfree(wksp);
|
||||
+}
|
||||
+
|
||||
+
|
||||
+static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,
|
||||
+ struct buffer_head **bh, int b, int offset, int length,
|
||||
+ struct squashfs_page_actor *output)
|
||||
+{
|
||||
+ struct workspace *wksp = strm;
|
||||
+ ZSTD_DStream *stream;
|
||||
+ size_t total_out = 0;
|
||||
+ size_t zstd_err;
|
||||
+ int k = 0;
|
||||
+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
|
||||
+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
|
||||
+
|
||||
+ stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
|
||||
+
|
||||
+ if (!stream) {
|
||||
+ ERROR("Failed to initialize zstd decompressor\n");
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+ out_buf.dst = squashfs_first_page(output);
|
||||
+
|
||||
+ do {
|
||||
+ if (in_buf.pos == in_buf.size && k < b) {
|
||||
+ int avail = min(length, msblk->devblksize - offset);
|
||||
+
|
||||
+ length -= avail;
|
||||
+ in_buf.src = bh[k]->b_data + offset;
|
||||
+ in_buf.size = avail;
|
||||
+ in_buf.pos = 0;
|
||||
+ offset = 0;
|
||||
+ }
|
||||
+
|
||||
+ if (out_buf.pos == out_buf.size) {
|
||||
+ out_buf.dst = squashfs_next_page(output);
|
||||
+ if (out_buf.dst == NULL) {
|
||||
+ /* Shouldn't run out of pages
|
||||
+ * before stream is done.
|
||||
+ */
|
||||
+ squashfs_finish_page(output);
|
||||
+ goto out;
|
||||
+ }
|
||||
+ out_buf.pos = 0;
|
||||
+ out_buf.size = PAGE_SIZE;
|
||||
+ }
|
||||
+
|
||||
+ total_out -= out_buf.pos;
|
||||
+ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf);
|
||||
+ total_out += out_buf.pos; /* add the additional data produced */
|
||||
+
|
||||
+ if (in_buf.pos == in_buf.size && k < b)
|
||||
+ put_bh(bh[k++]);
|
||||
+ } while (zstd_err != 0 && !ZSTD_isError(zstd_err));
|
||||
+
|
||||
+ squashfs_finish_page(output);
|
||||
+
|
||||
+ if (ZSTD_isError(zstd_err)) {
|
||||
+ ERROR("zstd decompression error: %d\n",
|
||||
+ (int)ZSTD_getErrorCode(zstd_err));
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ if (k < b)
|
||||
+ goto out;
|
||||
+
|
||||
+ return (int)total_out;
|
||||
+
|
||||
+out:
|
||||
+ for (; k < b; k++)
|
||||
+ put_bh(bh[k]);
|
||||
+
|
||||
+ return -EIO;
|
||||
+}
|
||||
+
|
||||
+const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
||||
+ .init = zstd_init,
|
||||
+ .free = zstd_free,
|
||||
+ .decompress = zstd_uncompress,
|
||||
+ .id = ZSTD_COMPRESSION,
|
||||
+ .name = "zstd",
|
||||
+ .supported = 1
|
||||
+};
|
||||
--
|
||||
2.9.3
|
||||
@@ -0,0 +1,424 @@
|
||||
From 308795a7713ca6fcd468b60fba9a2fca99cee6a0 Mon Sep 17 00:00:00 2001
|
||||
From: Nick Terrell <terrelln@fb.com>
|
||||
Date: Wed, 2 Aug 2017 18:02:13 -0700
|
||||
Subject: [PATCH v5 5/5] crypto: Add zstd support
|
||||
|
||||
Adds zstd support to crypto and scompress. Only supports the default
|
||||
level.
|
||||
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
crypto/Kconfig | 9 ++
|
||||
crypto/Makefile | 1 +
|
||||
crypto/testmgr.c | 10 +++
|
||||
crypto/testmgr.h | 71 +++++++++++++++
|
||||
crypto/zstd.c | 265 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
5 files changed, 356 insertions(+)
|
||||
create mode 100644 crypto/zstd.c
|
||||
|
||||
diff --git a/crypto/Kconfig b/crypto/Kconfig
|
||||
index caa770e..4fc3936 100644
|
||||
--- a/crypto/Kconfig
|
||||
+++ b/crypto/Kconfig
|
||||
@@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC
|
||||
help
|
||||
This is the LZ4 high compression mode algorithm.
|
||||
|
||||
+config CRYPTO_ZSTD
|
||||
+ tristate "Zstd compression algorithm"
|
||||
+ select CRYPTO_ALGAPI
|
||||
+ select CRYPTO_ACOMP2
|
||||
+ select ZSTD_COMPRESS
|
||||
+ select ZSTD_DECOMPRESS
|
||||
+ help
|
||||
+ This is the zstd algorithm.
|
||||
+
|
||||
comment "Random Number Generation"
|
||||
|
||||
config CRYPTO_ANSI_CPRNG
|
||||
diff --git a/crypto/Makefile b/crypto/Makefile
|
||||
index d41f033..b22e1e8 100644
|
||||
--- a/crypto/Makefile
|
||||
+++ b/crypto/Makefile
|
||||
@@ -133,6 +133,7 @@ obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
|
||||
obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
|
||||
+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
|
||||
|
||||
ecdh_generic-y := ecc.o
|
||||
ecdh_generic-y += ecdh.o
|
||||
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
|
||||
index 7125ba3..8a124d3 100644
|
||||
--- a/crypto/testmgr.c
|
||||
+++ b/crypto/testmgr.c
|
||||
@@ -3603,6 +3603,16 @@ static const struct alg_test_desc alg_test_descs[] = {
|
||||
.decomp = __VECS(zlib_deflate_decomp_tv_template)
|
||||
}
|
||||
}
|
||||
+ }, {
|
||||
+ .alg = "zstd",
|
||||
+ .test = alg_test_comp,
|
||||
+ .fips_allowed = 1,
|
||||
+ .suite = {
|
||||
+ .comp = {
|
||||
+ .comp = __VECS(zstd_comp_tv_template),
|
||||
+ .decomp = __VECS(zstd_decomp_tv_template)
|
||||
+ }
|
||||
+ }
|
||||
}
|
||||
};
|
||||
|
||||
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
|
||||
index 6ceb0e2..e6b5920 100644
|
||||
--- a/crypto/testmgr.h
|
||||
+++ b/crypto/testmgr.h
|
||||
@@ -34631,4 +34631,75 @@ static const struct comp_testvec lz4hc_decomp_tv_template[] = {
|
||||
},
|
||||
};
|
||||
|
||||
+static const struct comp_testvec zstd_comp_tv_template[] = {
|
||||
+ {
|
||||
+ .inlen = 68,
|
||||
+ .outlen = 39,
|
||||
+ .input = "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd.",
|
||||
+ .output = "\x28\xb5\x2f\xfd\x00\x50\xf5\x00\x00\xb8\x54\x68\x65"
|
||||
+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
|
||||
+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
|
||||
+ ,
|
||||
+ },
|
||||
+ {
|
||||
+ .inlen = 244,
|
||||
+ .outlen = 151,
|
||||
+ .input = "zstd, short for Zstandard, is a fast lossless "
|
||||
+ "compression algorithm, targeting real-time "
|
||||
+ "compression scenarios at zlib-level and better "
|
||||
+ "compression ratios. The zstd compression library "
|
||||
+ "provides in-memory compression and decompression "
|
||||
+ "functions.",
|
||||
+ .output = "\x28\xb5\x2f\xfd\x00\x50\x75\x04\x00\x42\x4b\x1e\x17"
|
||||
+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
|
||||
+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
|
||||
+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
|
||||
+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
|
||||
+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
|
||||
+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
|
||||
+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
|
||||
+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
|
||||
+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
|
||||
+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
|
||||
+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01",
|
||||
+ },
|
||||
+};
|
||||
+
|
||||
+static const struct comp_testvec zstd_decomp_tv_template[] = {
|
||||
+ {
|
||||
+ .inlen = 43,
|
||||
+ .outlen = 68,
|
||||
+ .input = "\x28\xb5\x2f\xfd\x04\x50\xf5\x00\x00\xb8\x54\x68\x65"
|
||||
+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
|
||||
+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
|
||||
+ "\x6b\xf4\x13\x35",
|
||||
+ .output = "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd. "
|
||||
+ "The algorithm is zstd.",
|
||||
+ },
|
||||
+ {
|
||||
+ .inlen = 155,
|
||||
+ .outlen = 244,
|
||||
+ .input = "\x28\xb5\x2f\xfd\x04\x50\x75\x04\x00\x42\x4b\x1e\x17"
|
||||
+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
|
||||
+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
|
||||
+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
|
||||
+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
|
||||
+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
|
||||
+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
|
||||
+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
|
||||
+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
|
||||
+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
|
||||
+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
|
||||
+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01\xaa\x6d\xda\x0d",
|
||||
+ .output = "zstd, short for Zstandard, is a fast lossless "
|
||||
+ "compression algorithm, targeting real-time "
|
||||
+ "compression scenarios at zlib-level and better "
|
||||
+ "compression ratios. The zstd compression library "
|
||||
+ "provides in-memory compression and decompression "
|
||||
+ "functions.",
|
||||
+ },
|
||||
+};
|
||||
#endif /* _CRYPTO_TESTMGR_H */
|
||||
diff --git a/crypto/zstd.c b/crypto/zstd.c
|
||||
new file mode 100644
|
||||
index 0000000..9a76b3e
|
||||
--- /dev/null
|
||||
+++ b/crypto/zstd.c
|
||||
@@ -0,0 +1,265 @@
|
||||
+/*
|
||||
+ * Cryptographic API.
|
||||
+ *
|
||||
+ * Copyright (c) 2017-present, Facebook, Inc.
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or modify it
|
||||
+ * under the terms of the GNU General Public License version 2 as published by
|
||||
+ * the Free Software Foundation.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
+ * more details.
|
||||
+ */
|
||||
+#include <linux/crypto.h>
|
||||
+#include <linux/init.h>
|
||||
+#include <linux/interrupt.h>
|
||||
+#include <linux/mm.h>
|
||||
+#include <linux/module.h>
|
||||
+#include <linux/net.h>
|
||||
+#include <linux/vmalloc.h>
|
||||
+#include <linux/zstd.h>
|
||||
+#include <crypto/internal/scompress.h>
|
||||
+
|
||||
+
|
||||
+#define ZSTD_DEF_LEVEL 3
|
||||
+
|
||||
+struct zstd_ctx {
|
||||
+ ZSTD_CCtx *cctx;
|
||||
+ ZSTD_DCtx *dctx;
|
||||
+ void *cwksp;
|
||||
+ void *dwksp;
|
||||
+};
|
||||
+
|
||||
+static ZSTD_parameters zstd_params(void)
|
||||
+{
|
||||
+ return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0);
|
||||
+}
|
||||
+
|
||||
+static int zstd_comp_init(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ int ret = 0;
|
||||
+ const ZSTD_parameters params = zstd_params();
|
||||
+ const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams);
|
||||
+
|
||||
+ ctx->cwksp = vzalloc(wksp_size);
|
||||
+ if (!ctx->cwksp) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size);
|
||||
+ if (!ctx->cctx) {
|
||||
+ ret = -EINVAL;
|
||||
+ goto out_free;
|
||||
+ }
|
||||
+out:
|
||||
+ return ret;
|
||||
+out_free:
|
||||
+ vfree(ctx->cwksp);
|
||||
+ goto out;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decomp_init(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ int ret = 0;
|
||||
+ const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
|
||||
+
|
||||
+ ctx->dwksp = vzalloc(wksp_size);
|
||||
+ if (!ctx->dwksp) {
|
||||
+ ret = -ENOMEM;
|
||||
+ goto out;
|
||||
+ }
|
||||
+
|
||||
+ ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size);
|
||||
+ if (!ctx->dctx) {
|
||||
+ ret = -EINVAL;
|
||||
+ goto out_free;
|
||||
+ }
|
||||
+out:
|
||||
+ return ret;
|
||||
+out_free:
|
||||
+ vfree(ctx->dwksp);
|
||||
+ goto out;
|
||||
+}
|
||||
+
|
||||
+static void zstd_comp_exit(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ vfree(ctx->cwksp);
|
||||
+ ctx->cwksp = NULL;
|
||||
+ ctx->cctx = NULL;
|
||||
+}
|
||||
+
|
||||
+static void zstd_decomp_exit(struct zstd_ctx *ctx)
|
||||
+{
|
||||
+ vfree(ctx->dwksp);
|
||||
+ ctx->dwksp = NULL;
|
||||
+ ctx->dctx = NULL;
|
||||
+}
|
||||
+
|
||||
+static int __zstd_init(void *ctx)
|
||||
+{
|
||||
+ int ret;
|
||||
+
|
||||
+ ret = zstd_comp_init(ctx);
|
||||
+ if (ret)
|
||||
+ return ret;
|
||||
+ ret = zstd_decomp_init(ctx);
|
||||
+ if (ret)
|
||||
+ zstd_comp_exit(ctx);
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static void *zstd_alloc_ctx(struct crypto_scomp *tfm)
|
||||
+{
|
||||
+ int ret;
|
||||
+ struct zstd_ctx *ctx;
|
||||
+
|
||||
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
|
||||
+ if (!ctx)
|
||||
+ return ERR_PTR(-ENOMEM);
|
||||
+
|
||||
+ ret = __zstd_init(ctx);
|
||||
+ if (ret) {
|
||||
+ kfree(ctx);
|
||||
+ return ERR_PTR(ret);
|
||||
+ }
|
||||
+
|
||||
+ return ctx;
|
||||
+}
|
||||
+
|
||||
+static int zstd_init(struct crypto_tfm *tfm)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_init(ctx);
|
||||
+}
|
||||
+
|
||||
+static void __zstd_exit(void *ctx)
|
||||
+{
|
||||
+ zstd_comp_exit(ctx);
|
||||
+ zstd_decomp_exit(ctx);
|
||||
+}
|
||||
+
|
||||
+static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx)
|
||||
+{
|
||||
+ __zstd_exit(ctx);
|
||||
+ kzfree(ctx);
|
||||
+}
|
||||
+
|
||||
+static void zstd_exit(struct crypto_tfm *tfm)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ __zstd_exit(ctx);
|
||||
+}
|
||||
+
|
||||
+static int __zstd_compress(const u8 *src, unsigned int slen,
|
||||
+ u8 *dst, unsigned int *dlen, void *ctx)
|
||||
+{
|
||||
+ size_t out_len;
|
||||
+ struct zstd_ctx *zctx = ctx;
|
||||
+ const ZSTD_parameters params = zstd_params();
|
||||
+
|
||||
+ out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params);
|
||||
+ if (ZSTD_isError(out_len))
|
||||
+ return -EINVAL;
|
||||
+ *dlen = out_len;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress(struct crypto_tfm *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_compress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen,
|
||||
+ void *ctx)
|
||||
+{
|
||||
+ return __zstd_compress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int __zstd_decompress(const u8 *src, unsigned int slen,
|
||||
+ u8 *dst, unsigned int *dlen, void *ctx)
|
||||
+{
|
||||
+ size_t out_len;
|
||||
+ struct zstd_ctx *zctx = ctx;
|
||||
+
|
||||
+ out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen);
|
||||
+ if (ZSTD_isError(out_len))
|
||||
+ return -EINVAL;
|
||||
+ *dlen = out_len;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen)
|
||||
+{
|
||||
+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
|
||||
+
|
||||
+ return __zstd_decompress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
|
||||
+ unsigned int slen, u8 *dst, unsigned int *dlen,
|
||||
+ void *ctx)
|
||||
+{
|
||||
+ return __zstd_decompress(src, slen, dst, dlen, ctx);
|
||||
+}
|
||||
+
|
||||
+static struct crypto_alg alg = {
|
||||
+ .cra_name = "zstd",
|
||||
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
|
||||
+ .cra_ctxsize = sizeof(struct zstd_ctx),
|
||||
+ .cra_module = THIS_MODULE,
|
||||
+ .cra_init = zstd_init,
|
||||
+ .cra_exit = zstd_exit,
|
||||
+ .cra_u = { .compress = {
|
||||
+ .coa_compress = zstd_compress,
|
||||
+ .coa_decompress = zstd_decompress } }
|
||||
+};
|
||||
+
|
||||
+static struct scomp_alg scomp = {
|
||||
+ .alloc_ctx = zstd_alloc_ctx,
|
||||
+ .free_ctx = zstd_free_ctx,
|
||||
+ .compress = zstd_scompress,
|
||||
+ .decompress = zstd_sdecompress,
|
||||
+ .base = {
|
||||
+ .cra_name = "zstd",
|
||||
+ .cra_driver_name = "zstd-scomp",
|
||||
+ .cra_module = THIS_MODULE,
|
||||
+ }
|
||||
+};
|
||||
+
|
||||
+static int __init zstd_mod_init(void)
|
||||
+{
|
||||
+ int ret;
|
||||
+
|
||||
+ ret = crypto_register_alg(&alg);
|
||||
+ if (ret)
|
||||
+ return ret;
|
||||
+
|
||||
+ ret = crypto_register_scomp(&scomp);
|
||||
+ if (ret)
|
||||
+ crypto_unregister_alg(&alg);
|
||||
+
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+static void __exit zstd_mod_fini(void)
|
||||
+{
|
||||
+ crypto_unregister_alg(&alg);
|
||||
+ crypto_unregister_scomp(&scomp);
|
||||
+}
|
||||
+
|
||||
+module_init(zstd_mod_init);
|
||||
+module_exit(zstd_mod_fini);
|
||||
+
|
||||
+MODULE_LICENSE("GPL");
|
||||
+MODULE_DESCRIPTION("Zstd Compression Algorithm");
|
||||
+MODULE_ALIAS_CRYPTO("zstd");
|
||||
--
|
||||
2.9.3
|
||||
@@ -0,0 +1,420 @@
|
||||
From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001
|
||||
From: Sean Purcell <me@seanp.xyz>
|
||||
Date: Thu, 3 Aug 2017 17:47:03 -0700
|
||||
Subject: [PATCH v6] squashfs-tools: Add zstd support
|
||||
|
||||
This patch adds zstd support to squashfs-tools. It works with zstd
|
||||
versions >= 1.0.0. It was originally written by Sean Purcell.
|
||||
|
||||
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
v4 -> v5:
|
||||
- Fix patch documentation to reflect that Sean Purcell is the author
|
||||
- Don't strip trailing whitespace of unreleated code
|
||||
- Make zstd_display_options() static
|
||||
|
||||
v5 -> v6:
|
||||
- Fix build instructions in Makefile
|
||||
|
||||
squashfs-tools/Makefile | 20 ++++
|
||||
squashfs-tools/compressor.c | 8 ++
|
||||
squashfs-tools/squashfs_fs.h | 1 +
|
||||
squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++
|
||||
squashfs-tools/zstd_wrapper.h | 48 ++++++++
|
||||
5 files changed, 331 insertions(+)
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.c
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.h
|
||||
|
||||
diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile
|
||||
index 52d2582..22fc559 100644
|
||||
--- a/squashfs-tools/Makefile
|
||||
+++ b/squashfs-tools/Makefile
|
||||
@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1
|
||||
#LZMA_SUPPORT = 1
|
||||
#LZMA_DIR = ../../../../LZMA/lzma465
|
||||
|
||||
+
|
||||
+########### Building ZSTD support ############
|
||||
+#
|
||||
+# The ZSTD library is supported
|
||||
+# ZSTD homepage: http://zstd.net
|
||||
+# ZSTD source repository: https://github.com/facebook/zstd
|
||||
+#
|
||||
+# To build using the ZSTD library - install the library and uncomment the
|
||||
+# ZSTD_SUPPORT line below.
|
||||
+#
|
||||
+#ZSTD_SUPPORT = 1
|
||||
+
|
||||
######## Specifying default compression ########
|
||||
#
|
||||
# The next line specifies which compression algorithm is used by default
|
||||
@@ -177,6 +189,14 @@ LIBS += -llz4
|
||||
COMPRESSORS += lz4
|
||||
endif
|
||||
|
||||
+ifeq ($(ZSTD_SUPPORT),1)
|
||||
+CFLAGS += -DZSTD_SUPPORT
|
||||
+MKSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+UNSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+LIBS += -lzstd
|
||||
+COMPRESSORS += zstd
|
||||
+endif
|
||||
+
|
||||
ifeq ($(XATTR_SUPPORT),1)
|
||||
ifeq ($(XATTR_DEFAULT),1)
|
||||
CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
|
||||
diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c
|
||||
index 525e316..02b5e90 100644
|
||||
--- a/squashfs-tools/compressor.c
|
||||
+++ b/squashfs-tools/compressor.c
|
||||
@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
|
||||
extern struct compressor xz_comp_ops;
|
||||
#endif
|
||||
|
||||
+#ifndef ZSTD_SUPPORT
|
||||
+static struct compressor zstd_comp_ops = {
|
||||
+ ZSTD_COMPRESSION, "zstd"
|
||||
+};
|
||||
+#else
|
||||
+extern struct compressor zstd_comp_ops;
|
||||
+#endif
|
||||
|
||||
static struct compressor unknown_comp_ops = {
|
||||
0, "unknown"
|
||||
@@ -77,6 +84,7 @@ struct compressor *compressor[] = {
|
||||
&lzo_comp_ops,
|
||||
&lz4_comp_ops,
|
||||
&xz_comp_ops,
|
||||
+ &zstd_comp_ops,
|
||||
&unknown_comp_ops
|
||||
};
|
||||
|
||||
diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
|
||||
index 791fe12..afca918 100644
|
||||
--- a/squashfs-tools/squashfs_fs.h
|
||||
+++ b/squashfs-tools/squashfs_fs.h
|
||||
@@ -277,6 +277,7 @@ typedef long long squashfs_inode;
|
||||
#define LZO_COMPRESSION 3
|
||||
#define XZ_COMPRESSION 4
|
||||
#define LZ4_COMPRESSION 5
|
||||
+#define ZSTD_COMPRESSION 6
|
||||
|
||||
struct squashfs_super_block {
|
||||
unsigned int s_magic;
|
||||
diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
|
||||
new file mode 100644
|
||||
index 0000000..dcab75a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.c
|
||||
@@ -0,0 +1,254 @@
|
||||
+/*
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.c
|
||||
+ *
|
||||
+ * Support for ZSTD compression http://zstd.net
|
||||
+ */
|
||||
+
|
||||
+#include <stdio.h>
|
||||
+#include <string.h>
|
||||
+#include <stdlib.h>
|
||||
+#include <zstd.h>
|
||||
+#include <zstd_errors.h>
|
||||
+
|
||||
+#include "squashfs_fs.h"
|
||||
+#include "zstd_wrapper.h"
|
||||
+#include "compressor.h"
|
||||
+
|
||||
+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by the options parsing code in mksquashfs.c
|
||||
+ * to parse any -X compressor option.
|
||||
+ *
|
||||
+ * This function returns:
|
||||
+ * >=0 (number of additional args parsed) on success
|
||||
+ * -1 if the option was unrecognised, or
|
||||
+ * -2 if the option was recognised, but otherwise bad in
|
||||
+ * some way (e.g. invalid parameter)
|
||||
+ *
|
||||
+ * Note: this function sets internal compressor state, but does not
|
||||
+ * pass back the results of the parsing other than success/failure.
|
||||
+ * The zstd_dump_options() function is called later to get the options in
|
||||
+ * a format suitable for writing to the filesystem.
|
||||
+ */
|
||||
+static int zstd_options(char *argv[], int argc)
|
||||
+{
|
||||
+ if (strcmp(argv[0], "-Xcompression-level") == 0) {
|
||||
+ if (argc < 2) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level missing "
|
||||
+ "compression level\n");
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level it should "
|
||||
+ "be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = atoi(argv[1]);
|
||||
+ if (compression_level < 1 ||
|
||||
+ compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level invalid, it "
|
||||
+ "should be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ return 1;
|
||||
+ }
|
||||
+
|
||||
+ return -1;
|
||||
+failed:
|
||||
+ return -2;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to dump the parsed
|
||||
+ * compressor options in a format suitable for writing to the
|
||||
+ * compressor options field in the filesystem (stored immediately
|
||||
+ * after the superblock).
|
||||
+ *
|
||||
+ * This function returns a pointer to the compression options structure
|
||||
+ * to be stored (and the size), or NULL if there are no compression
|
||||
+ * options.
|
||||
+ */
|
||||
+static void *zstd_dump_options(int block_size, int *size)
|
||||
+{
|
||||
+ static struct zstd_comp_opts comp_opts;
|
||||
+
|
||||
+ /* don't return anything if the options are all default */
|
||||
+ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
|
||||
+ return NULL;
|
||||
+
|
||||
+ comp_opts.compression_level = compression_level;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
|
||||
+
|
||||
+ *size = sizeof(comp_opts);
|
||||
+ return &comp_opts;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is a helper specifically for the append mode of
|
||||
+ * mksquashfs. Its purpose is to set the internal compressor state
|
||||
+ * to the stored compressor options in the passed compressor options
|
||||
+ * structure.
|
||||
+ *
|
||||
+ * In effect this function sets up the compressor options
|
||||
+ * to the same state they were when the filesystem was originally
|
||||
+ * generated, this is to ensure on appending, the compressor uses
|
||||
+ * the same compression options that were used to generate the
|
||||
+ * original filesystem.
|
||||
+ *
|
||||
+ * Note, even if there are no compressor options, this function is still
|
||||
+ * called with an empty compressor structure (size == 0), to explicitly
|
||||
+ * set the default options, this is to ensure any user supplied
|
||||
+ * -X options on the appending mksquashfs command line are over-ridden.
|
||||
+ *
|
||||
+ * This function returns 0 on sucessful extraction of options, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_extract_options(int block_size, void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ if (size == 0) {
|
||||
+ /* Set default values */
|
||||
+ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = comp_opts->compression_level;
|
||||
+
|
||||
+ return 0;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+
|
||||
+ return -1;
|
||||
+}
|
||||
+
|
||||
+static void zstd_display_options(void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ printf("\tcompression-level %d\n", comp_opts->compression_level);
|
||||
+
|
||||
+ return;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to initialise the
|
||||
+ * compressor, before compress() is called.
|
||||
+ *
|
||||
+ * This function returns 0 on success, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_init(void **strm, int block_size, int datablock)
|
||||
+{
|
||||
+ ZSTD_CCtx *cctx = ZSTD_createCCtx();
|
||||
+
|
||||
+ if (!cctx) {
|
||||
+ fprintf(stderr, "zstd: failed to allocate compression "
|
||||
+ "context!\n");
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ *strm = cctx;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress(void *strm, void *dest, void *src, int size,
|
||||
+ int block_size, int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
|
||||
+ src, size, compression_level);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ /* FIXME:
|
||||
+ * zstd does not expose stable error codes. The error enum may
|
||||
+ * change between versions. Until upstream zstd stablizes the
|
||||
+ * error codes, we have no way of knowing why the error occurs.
|
||||
+ * zstd shouldn't fail to compress any input unless there isn't
|
||||
+ * enough output space. We assume that is the cause and return
|
||||
+ * the special error code for not enough output space.
|
||||
+ */
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static int zstd_uncompress(void *dest, void *src, int size, int outsize,
|
||||
+ int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_decompress(dest, outsize, src, size);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ fprintf(stderr, "\t%d %d\n", outsize, size);
|
||||
+
|
||||
+ *error = (int)ZSTD_getErrorCode(res);
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static void zstd_usage(void)
|
||||
+{
|
||||
+ fprintf(stderr, "\t -Xcompression-level <compression-level>\n");
|
||||
+ fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "
|
||||
+ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);
|
||||
+}
|
||||
+
|
||||
+struct compressor zstd_comp_ops = {
|
||||
+ .init = zstd_init,
|
||||
+ .compress = zstd_compress,
|
||||
+ .uncompress = zstd_uncompress,
|
||||
+ .options = zstd_options,
|
||||
+ .dump_options = zstd_dump_options,
|
||||
+ .extract_options = zstd_extract_options,
|
||||
+ .display_options = zstd_display_options,
|
||||
+ .usage = zstd_usage,
|
||||
+ .id = ZSTD_COMPRESSION,
|
||||
+ .name = "zstd",
|
||||
+ .supported = 1
|
||||
+};
|
||||
diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h
|
||||
new file mode 100644
|
||||
index 0000000..4fbef0a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.h
|
||||
@@ -0,0 +1,48 @@
|
||||
+#ifndef ZSTD_WRAPPER_H
|
||||
+#define ZSTD_WRAPPER_H
|
||||
+/*
|
||||
+ * Squashfs
|
||||
+ *
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.h
|
||||
+ *
|
||||
+ */
|
||||
+
|
||||
+#ifndef linux
|
||||
+#define __BYTE_ORDER BYTE_ORDER
|
||||
+#define __BIG_ENDIAN BIG_ENDIAN
|
||||
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
||||
+#else
|
||||
+#include <endian.h>
|
||||
+#endif
|
||||
+
|
||||
+#if __BYTE_ORDER == __BIG_ENDIAN
|
||||
+extern unsigned int inswap_le16(unsigned short);
|
||||
+extern unsigned int inswap_le32(unsigned int);
|
||||
+
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
|
||||
+ (s)->compression_level = inswap_le32((s)->compression_level); \
|
||||
+}
|
||||
+#else
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s)
|
||||
+#endif
|
||||
+
|
||||
+/* Default compression */
|
||||
+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
|
||||
+
|
||||
+struct zstd_comp_opts {
|
||||
+ int compression_level;
|
||||
+};
|
||||
+#endif
|
||||
--
|
||||
2.9.5
|
||||
@@ -0,0 +1,339 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
@@ -1,96 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
.PHONY: libzstd
|
||||
libzstd:
|
||||
rm -rf linux
|
||||
mkdir -p linux
|
||||
mkdir -p linux/include/linux
|
||||
mkdir -p linux/lib/zstd
|
||||
../freestanding_lib/freestanding.py \
|
||||
--source-lib ../../lib \
|
||||
--output-lib linux/lib/zstd \
|
||||
--xxhash '<linux/xxhash.h>' \
|
||||
--xxh64-state 'struct xxh64_state' \
|
||||
--xxh64-prefix 'xxh64' \
|
||||
--rewrite-include '<limits\.h>=<linux/limits.h>' \
|
||||
--rewrite-include '<stddef\.h>=<linux/types.h>' \
|
||||
--rewrite-include '"\.\./zstd.h"=<linux/zstd.h>' \
|
||||
--rewrite-include '"(\.\./common/)?zstd_errors.h"=<linux/zstd_errors.h>' \
|
||||
--sed 's,/\*\*\*,/* *,g' \
|
||||
--sed 's,/\*\*,/*,g' \
|
||||
-DZSTD_NO_INTRINSICS \
|
||||
-DZSTD_NO_UNUSED_FUNCTIONS \
|
||||
-DZSTD_LEGACY_SUPPORT=0 \
|
||||
-DZSTD_STATIC_LINKING_ONLY \
|
||||
-DFSE_STATIC_LINKING_ONLY \
|
||||
-DHUF_STATIC_LINKING_ONLY \
|
||||
-DXXH_STATIC_LINKING_ONLY \
|
||||
-DMEM_FORCE_MEMORY_ACCESS=0 \
|
||||
-D__GNUC__ \
|
||||
-DSTATIC_BMI2=0 \
|
||||
-DZSTD_ADDRESS_SANITIZER=0 \
|
||||
-DZSTD_MEMORY_SANITIZER=0 \
|
||||
-DZSTD_COMPRESS_HEAPMODE=1 \
|
||||
-UNO_PREFETCH \
|
||||
-U__cplusplus \
|
||||
-UZSTD_DLL_EXPORT \
|
||||
-UZSTD_DLL_IMPORT \
|
||||
-U__ICCARM__ \
|
||||
-UZSTD_MULTITHREAD \
|
||||
-U_MSC_VER \
|
||||
-U_WIN32 \
|
||||
-RZSTDLIB_VISIBILITY= \
|
||||
-RZSTDERRORLIB_VISIBILITY= \
|
||||
-RZSTD_FALLTHROUGH=fallthrough \
|
||||
-DZSTD_HAVE_WEAK_SYMBOLS=0 \
|
||||
-DZSTD_TRACE=0 \
|
||||
-DZSTD_NO_TRACE \
|
||||
-DZSTD_LINUX_KERNEL
|
||||
mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
|
||||
mv linux/lib/zstd/common/zstd_errors.h linux/include/linux/
|
||||
cp linux_zstd.h linux/include/linux/zstd.h
|
||||
cp zstd_compress_module.c linux/lib/zstd
|
||||
cp zstd_decompress_module.c linux/lib/zstd
|
||||
cp decompress_sources.h linux/lib/zstd
|
||||
cp linux.mk linux/lib/zstd/Makefile
|
||||
|
||||
LINUX ?= $(HOME)/repos/linux
|
||||
|
||||
.PHONY: import
|
||||
import: libzstd
|
||||
rm -f $(LINUX)/include/linux/zstd.h
|
||||
rm -f $(LINUX)/include/linux/zstd_errors.h
|
||||
rm -rf $(LINUX)/lib/zstd
|
||||
cp linux/include/linux/zstd.h $(LINUX)/include/linux
|
||||
cp linux/include/linux/zstd_lib.h $(LINUX)/include/linux
|
||||
cp linux/include/linux/zstd_errors.h $(LINUX)/include/linux
|
||||
cp -r linux/lib/zstd $(LINUX)/lib
|
||||
|
||||
import-upstream:
|
||||
rm -rf $(LINUX)/lib/zstd
|
||||
mkdir $(LINUX)/lib/zstd
|
||||
cp ../../lib/zstd.h $(LINUX)/include/linux/zstd_lib.h
|
||||
cp -r ../../lib/common $(LINUX)/lib/zstd
|
||||
cp -r ../../lib/compress $(LINUX)/lib/zstd
|
||||
cp -r ../../lib/decompress $(LINUX)/lib/zstd
|
||||
mv $(LINUX)/lib/zstd/common/zstd_errors.h $(LINUX)/include/linux
|
||||
rm $(LINUX)/lib/zstd/common/threading.*
|
||||
rm $(LINUX)/lib/zstd/common/pool.*
|
||||
rm $(LINUX)/lib/zstd/common/xxhash.*
|
||||
rm $(LINUX)/lib/zstd/compress/zstdmt_*
|
||||
|
||||
.PHONY: test
|
||||
test: libzstd
|
||||
$(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS)" -j
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) -rf linux
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user