Compare commits

...
6 Commits
Author SHA1 Message Date
Yann Collet 35132477c0 Merge pull request #270 from inikep/dev07
decoder for v0.8
2016-08-01 13:25:55 +02:00
inikep 3bae910a09 updated appveyor.yml 2016-08-01 12:24:44 +02:00
inikep 2a70f54e48 fixed ZSTD_compressBlock_opt_extDict_generic 2016-08-01 12:19:21 +02:00
inikep 08951e9d54 updated project to support legacy v0.8 decoder 2016-08-01 12:14:10 +02:00
inikep 357c34b90b added legacy decoder for v0.8 2016-08-01 12:12:15 +02:00
Yann Collet d97262a3a2 updated library number 2016-08-01 10:00:49 +02:00
10 changed files with 5269 additions and 7 deletions
+4 -1
View File
@@ -27,7 +27,8 @@ install:
SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
) else (
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
)
@@ -50,6 +51,8 @@ build_script:
ECHO *** &&
ECHO *** Building %PLATFORM% &&
ECHO *** &&
make -v &&
cc -v &&
ECHO make %MAKE_PARAMS% &&
make %MAKE_PARAMS% &&
make clean
+1 -1
View File
@@ -61,7 +61,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 7
#define ZSTD_VERSION_RELEASE 4
#define ZSTD_VERSION_RELEASE 5
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
+4 -2
View File
@@ -780,7 +780,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const U32 repIndex = (U32)(current - rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
if ( (rep[i] <= current)
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
@@ -882,7 +883,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
if ( (opt[cur].rep[i] <= current+cur)
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
+16
View File
@@ -48,6 +48,7 @@ extern "C" {
#include "zstd_v04.h"
#include "zstd_v05.h"
#include "zstd_v06.h"
#include "zstd_v08.h"
/** ZSTD_isLegacy() :
@@ -67,6 +68,7 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
case ZSTDv04_magicNumber : return 4;
case ZSTDv05_MAGICNUMBER : return 5;
case ZSTDv06_MAGICNUMBER : return 6;
case ZSTDv08_MAGICNUMBER : return 8;
default : return 0;
}
}
@@ -90,6 +92,12 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s
if (frResult != 0) return 0;
return fParams.frameContentSize;
}
if (version==8) {
ZSTDv08_frameParams fParams;
size_t const frResult = ZSTDv08_getFrameParams(&fParams, src, srcSize);
if (frResult != 0) return 0;
return fParams.frameContentSize;
}
return 0; /* should not be possible */
}
}
@@ -126,6 +134,14 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
ZSTDv06_freeDCtx(zd);
return result;
}
case 8 :
{ size_t result;
ZSTDv08_DCtx* const zd = ZSTDv08_createDCtx();
if (zd==NULL) return ERROR(memory_allocation);
result = ZSTDv08_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
ZSTDv08_freeDCtx(zd);
return result;
}
default :
return ERROR(prefix_unknown);
}
File diff suppressed because it is too large Load Diff
+225
View File
@@ -0,0 +1,225 @@
/*
zstd_v08 - decoder for 0.8 format
Header File
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
*/
#ifndef ZSTDv08_H_235446
#define ZSTDv08_H_235446
#if defined (__cplusplus)
extern "C" {
#endif
/*====== Dependency ======*/
#include <stddef.h> /* size_t */
/*====== Export for Windows ======*/
/*!
* ZSTDv08_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport)
#else
# define ZSTDLIB_API
#endif
/* *************************************
* Simple API
***************************************/
/*! ZSTDv08_getDecompressedSize() :
* @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 2 : decompressed size is an optional field, that may not be present.
* When `return==0`, consider data to decompress could have any size.
* In which case, it's necessary to use streaming mode to decompress data,
* or rely on application's implied limits.
* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits !
* Each application can have its own set of conditions.
* If the intention is to decompress public data compressed by zstd command line utility,
* it is recommended to support at least 8 MB for extended compatibility.
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTDv08_getFrameParams() to know more. */
unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTDv08_decompress() :
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
`dstCapacity` must be equal or larger than originalSize (see ZSTDv08_getDecompressedSize() ).
If originalSize is unknown, and if there is no implied application-specific limitations,
it's necessary to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv08_isError()) */
ZSTDLIB_API size_t ZSTDv08_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*====== Helper functions ======*/
ZSTDLIB_API unsigned ZSTDv08_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTDv08_getErrorName(size_t code); /*!< provides readable string from an error code */
/*-*************************************
* Explicit memory management
***************************************/
/** Decompression context */
typedef struct ZSTDv08_DCtx_s ZSTDv08_DCtx; /*< incomplete type */
ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx(void);
ZSTDLIB_API size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx);
/** ZSTDv08_decompressDCtx() :
* Same as ZSTDv08_decompress(), requires an allocated ZSTDv08_DCtx (see ZSTDv08_createDCtx()) */
ZSTDLIB_API size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-************************
* Simple dictionary API
***************************/
/*! ZSTDv08_decompress_usingDict() :
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
* Note : This function load the dictionary, resulting in a significant startup time */
ZSTDLIB_API size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
/*-**************************
* Fast Dictionary API
****************************/
/*! ZSTDv08_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
typedef struct ZSTDv08_DDict_s ZSTDv08_DDict;
ZSTDLIB_API ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict);
/*! ZSTDv08_decompress_usingDDict() :
* Decompression using a digested Dictionary
* Faster startup than ZSTDv08_decompress_usingDict(), recommended when same dictionary is used multiple times. */
ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTDv08_DDict* ddict);
typedef struct {
unsigned long long frameContentSize;
unsigned windowSize;
unsigned dictID;
unsigned checksumFlag;
} ZSTDv08_frameParams;
ZSTDLIB_API size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
/* ***************************************************************
* Compiler specifics
*****************************************************************/
/* ZSTDv08_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL */
#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport)
#else
# define ZSTDLIB_API
#endif
/* *************************************
* Streaming functions
***************************************/
/* This is the easier "buffered" streaming API,
* using an internal buffer to lift all restrictions on user-provided buffers
* which can be any size, any place, for both input and output.
* ZBUFF and ZSTD are 100% interoperable,
* frames created by one can be decoded by the other one */
typedef struct ZBUFFv08_DCtx_s ZBUFFv08_DCtx;
ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void);
ZSTDLIB_API size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* dctx);
ZSTDLIB_API size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* dctx);
ZSTDLIB_API size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression howto
*
* A ZBUFFv08_DCtx object is required to track streaming operations.
* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources.
* Use ZBUFFv08_decompressInit() to start a new decompression operation,
* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFFv08_DCtx objects can be re-init multiple times.
*
* Use ZBUFFv08_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
* @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush,
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
* or an error code, which can be tested using ZBUFFv08_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize()
* output : ZBUFFv08_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFFv08_recommendedDInSize == 128KB + 3;
* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
/* *************************************
* Tool functions
***************************************/
ZSTDLIB_API unsigned ZBUFFv08_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFFv08_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
ZSTDLIB_API size_t ZBUFFv08_recommendedDInSize(void);
ZSTDLIB_API size_t ZBUFFv08_recommendedDOutSize(void);
#define ZSTDv08_MAGICNUMBER 0xFD2FB528 /* v0.8 */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv08_H_235446 */
+84
View File
@@ -548,6 +548,82 @@ unsigned long long FIOv06_decompressFrame(dRessv06_t ress,
}
/*===== v0.8.x =====*/
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
const void* dictBuffer;
size_t dictBufferSize;
ZBUFFv08_DCtx* dctx;
} dRessv08_t;
static dRessv08_t FIOv08_createDResources(void)
{
dRessv08_t ress;
/* init */
ress.dctx = ZBUFFv08_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.dictBuffer = NULL; ress.dictBufferSize=0;
/* Allocate Memory */
ress.srcBufferSize = ZBUFFv08_recommendedDInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFFv08_recommendedDOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
return ress;
}
static void FIOv08_freeDResources(dRessv08_t ress)
{
size_t const errorCode = ZBUFFv08_freeDCtx(ress.dctx);
if (ZBUFFv08_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv08_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
}
unsigned long long FIOv08_decompressFrame(dRessv08_t ress,
FILE* foutput, FILE* finput)
{
U64 frameSize = 0;
size_t readSize = 4;
MEM_writeLE32(ress.srcBuffer, ZSTDv08_MAGICNUMBER);
ZBUFFv08_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFFv08_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFFv08_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv08_getErrorName(toRead));
readSize -= inSize;
/* Write block */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
}
return frameSize;
}
/*===== General legacy dispatcher =====*/
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
@@ -584,6 +660,14 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
FIOv06_freeDResources(r);
return s;
} }
case ZSTDv08_MAGICNUMBER :
{ dRessv08_t r = FIOv08_createDResources();
r.dictBuffer = dictBuffer;
r.dictBufferSize = dictSize;
{ unsigned long long const s = FIOv08_decompressFrame(r, foutput, finput);
FIOv08_freeDResources(r);
return s;
} }
default :
return ERROR(prefix_unknown);
}
+8
View File
@@ -428,6 +428,10 @@
RelativePath="..\..\..\lib\legacy\zstd_v06.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v08.c"
>
</File>
<File
RelativePath="..\..\..\programs\zstdcli.c"
>
@@ -538,6 +542,10 @@
RelativePath="..\..\..\lib\legacy\zstd_v06.h"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v08.h"
>
</File>
</Filter>
</Files>
<Globals>
+3 -1
View File
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
@@ -38,6 +38,7 @@
<ClCompile Include="..\..\..\lib\legacy\zstd_v04.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v08.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\dibio.c" />
@@ -62,6 +63,7 @@
<ClInclude Include="..\..\..\lib\legacy\zstd_v04.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v08.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\dibio.h" />
+4 -2
View File
@@ -93,7 +93,8 @@ IF (ZSTD_LEGACY_SUPPORT)
${LIBRARY_LEGACY_DIR}/zstd_v03.c
${LIBRARY_LEGACY_DIR}/zstd_v04.c
${LIBRARY_LEGACY_DIR}/zstd_v05.c
${LIBRARY_LEGACY_DIR}/zstd_v06.c)
${LIBRARY_LEGACY_DIR}/zstd_v06.c
${LIBRARY_LEGACY_DIR}/zstd_v08.c)
SET(Headers ${Headers}
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
@@ -102,7 +103,8 @@ IF (ZSTD_LEGACY_SUPPORT)
${LIBRARY_LEGACY_DIR}/zstd_v03.h
${LIBRARY_LEGACY_DIR}/zstd_v04.h
${LIBRARY_LEGACY_DIR}/zstd_v05.h
${LIBRARY_LEGACY_DIR}/zstd_v06.h)
${LIBRARY_LEGACY_DIR}/zstd_v06.h
${LIBRARY_LEGACY_DIR}/zstd_v08.h)
ENDIF (ZSTD_LEGACY_SUPPORT)
IF (MSVC)