Compare commits

...
7 Commits
Author SHA1 Message Date
Yann Collet b8b7b96e69 Merge branch 'dev' into ultra_init 2021-09-29 20:44:12 -07:00
Yann Collet 7a27793c94 Merge branch 'dev' into ultra_init 2021-09-15 23:03:55 -07:00
Yann Collet 72549784c5 settle for an init boundary at 8 KB
btultra2 uses btultra as seed for statistics
Then btultra uses greedy as seed for statistics.
It turns out, this gives worse results for btultra2 on small blocks.
Differentiate small blocks from larger ones (bounday 8 KB),
initialize btultra with new default stats on small blocks.
2021-09-12 22:12:42 -07:00
Yann Collet 1bc623612f improve initial statistics of sequence symbols for btultra2 2021-09-12 20:31:59 -07:00
Yann Collet 0b6fa7a11c improve initial literals stats 2021-09-12 19:18:21 -07:00
Yann Collet ab1905cfa0 Merge branch 'opt_investigation' into ultra_init 2021-09-12 12:14:44 -07:00
Yann Collet ccde29d407 initialize btultra stats with greedy
improves compression ratio on first block
at a speed cost of ~-5%.
2021-09-12 00:54:23 -07:00
7 changed files with 190 additions and 46 deletions
+10 -10
View File
@@ -723,36 +723,36 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_windowLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_windowLog, value);
CCtxParams->cParams.windowLog = (U32)value;
CCtxParams->cParams.windowLog = (unsigned)value;
return CCtxParams->cParams.windowLog;
case ZSTD_c_hashLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_hashLog, value);
CCtxParams->cParams.hashLog = (U32)value;
CCtxParams->cParams.hashLog = (unsigned)value;
return CCtxParams->cParams.hashLog;
case ZSTD_c_chainLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_chainLog, value);
CCtxParams->cParams.chainLog = (U32)value;
CCtxParams->cParams.chainLog = (unsigned)value;
return CCtxParams->cParams.chainLog;
case ZSTD_c_searchLog :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_searchLog, value);
CCtxParams->cParams.searchLog = (U32)value;
CCtxParams->cParams.searchLog = (unsigned)value;
return (size_t)value;
case ZSTD_c_minMatch :
if (value!=0) /* 0 => use default */
BOUNDCHECK(ZSTD_c_minMatch, value);
CCtxParams->cParams.minMatch = value;
CCtxParams->cParams.minMatch = (unsigned)value;
return CCtxParams->cParams.minMatch;
case ZSTD_c_targetLength :
BOUNDCHECK(ZSTD_c_targetLength, value);
CCtxParams->cParams.targetLength = value;
CCtxParams->cParams.targetLength = (unsigned)value;
return CCtxParams->cParams.targetLength;
case ZSTD_c_strategy :
@@ -765,12 +765,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
/* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(4, "set content size flag = %u", (value!=0));
CCtxParams->fParams.contentSizeFlag = value != 0;
return CCtxParams->fParams.contentSizeFlag;
return (size_t)CCtxParams->fParams.contentSizeFlag;
case ZSTD_c_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
CCtxParams->fParams.checksumFlag = value != 0;
return CCtxParams->fParams.checksumFlag;
return (size_t)CCtxParams->fParams.checksumFlag;
case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
@@ -779,11 +779,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_forceMaxWindow :
CCtxParams->forceWindow = (value != 0);
return CCtxParams->forceWindow;
return (size_t)CCtxParams->forceWindow;
case ZSTD_c_forceAttachDict : {
const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
CCtxParams->attachDictPref = pref;
return CCtxParams->attachDictPref;
}
+4 -4
View File
@@ -676,7 +676,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0;
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (__builtin_ctzll((U64)val) >> 3);
return (unsigned)(__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
0, 3, 1, 3, 1, 4, 2, 7,
@@ -693,7 +693,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
unsigned long r=0;
return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0;
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
return (unsigned)(__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
@@ -712,7 +712,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
return _BitScanReverse64(&r, (U64)val) ? (unsigned)(r >> 3) : 0;
# endif
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (__builtin_clzll(val) >> 3);
return (unsigned)(__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
@@ -726,7 +726,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
unsigned long r = 0;
return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0;
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
return (unsigned)(__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+1 -2
View File
@@ -93,7 +93,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
/* small ? don't even attempt compression (speed opt) */
# define COMPRESS_LITERALS_SIZE_MIN 63
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
}
@@ -136,7 +135,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
switch(lhSize)
{
case 3: /* 2 - 2 - 10 - 10 */
{ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
{ U32 const lhc = hType + (((U32)!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
MEM_writeLE24(ostart, lhc);
break;
}
+5
View File
@@ -14,6 +14,11 @@
#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */
/* Below this limit, the heuristic does not even attempt compression of literals,
* as the cost of the headers is expected to outweight the benefits.
* This limit is not applicable when re-using statistics from dictionary or previous block */
#define COMPRESS_LITERALS_SIZE_MIN 63
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+167 -28
View File
@@ -113,6 +113,21 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
}
static unsigned const k_baseLLfreqs[MaxLL+1] = {
4, 2, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1
};
static unsigned const k_baseOFCfreqs[MaxOff+1] = {
6, 2, 1, 1, 2, 3, 4, 4,
4, 3, 2, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1
};
/* ZSTD_rescaleFreqs() :
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
* take hints from dictionary if there is one
@@ -192,20 +207,13 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
assert(optPtr->litFreq != NULL);
if (compressedLiterals) {
unsigned lit = MaxLit;
HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
unsigned maxlit = MaxLit;
HIST_count_simple(optPtr->litFreq, &maxlit, src, srcSize); /* use raw first block to init statistics */
optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
}
{ unsigned const baseLLfreqs[MaxLL+1] = {
4, 2, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1
};
ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
}
ZSTD_memcpy(optPtr->litLengthFreq, k_baseLLfreqs, sizeof(k_baseLLfreqs));
optPtr->litLengthSum = sum_u32(k_baseLLfreqs, MaxLL+1);
{ unsigned ml;
for (ml=0; ml<=MaxML; ml++)
@@ -213,15 +221,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
}
optPtr->matchLengthSum = MaxML+1;
{ unsigned const baseOFCfreqs[MaxOff+1] = {
6, 2, 1, 1, 2, 3, 4, 4,
4, 3, 2, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1
};
ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
}
ZSTD_memcpy(optPtr->offCodeFreq, k_baseOFCfreqs, sizeof(k_baseOFCfreqs));
optPtr->offCodeSum = sum_u32(k_baseOFCfreqs, MaxOff+1);
}
@@ -1056,8 +1057,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
opt[pos].price = (int)sequencePrice;
} }
last_pos = pos-1;
}
}
} }
/* check further positions */
for (cur = 1; cur <= last_pos; cur++) {
@@ -1084,8 +1084,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price),
opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]);
}
}
} }
/* Set the repcodes of the current position. We must do it here
* because we rely on the repcodes of the 2nd to last sequence being
@@ -1252,15 +1251,76 @@ size_t ZSTD_compressBlock_btopt(
}
#include "zstd_compress_literals.h" /* COMPRESS_LITERALS_SIZE_MIN */
static void transfer_literalStats(optState_t* opt, const seqStore_t* seqStore)
{
size_t const nbLiterals = (size_t)(seqStore->lit - seqStore->litStart);
assert(seqStore->lit >= seqStore->litStart);
if (nbLiterals < COMPRESS_LITERALS_SIZE_MIN) {
/* literals won't be compressed, so give them an all-flat cost */
/* note : it would be better if it was also possible to extend this category
* to non-compressible literals, even when they are more numerous than threshold.
* However, this requires additional cpu and memory workspace */
unsigned u;
for (u=0; u<=MaxLit; u++) opt->litFreq[u]=2;
} else {
unsigned maxlit = MaxLit;
HIST_count_simple(opt->litFreq, &maxlit, seqStore->litStart, nbLiterals);
}
opt->litSum = ZSTD_downscaleStats(opt->litFreq, MaxLit, 0); /* flatten stats, by providing at least 1 to every symbol */
}
/* ZSTD_initStats_ultra():
typedef enum { fix_greedy_bias, from_btultra } biasfix_e;
static void transfer_seqStats(optState_t* opt, const seqStore_t* seqStore, biasfix_e biasfix)
{
U32 const nbSeq = (U32)(seqStore->sequences - seqStore->sequencesStart);
assert(seqStore->sequences >= seqStore->sequencesStart);
ZSTD_seqToCodes(seqStore);
{ const BYTE* codePtr = seqStore->ofCode;
U32 u;
memset(opt->offCodeFreq, 0, sizeof(U32) * (MaxOff+1));
ZSTD_STATIC_ASSERT(MaxOff >= 17);
for (u=0; u<17; u++) opt->offCodeFreq[u]=1; /* flatten stats; some offcode may not be produced by greedy but still be present */
for (u=0; u<nbSeq; u++) opt->offCodeFreq[codePtr[u]]++;
if (biasfix == fix_greedy_bias) {
assert(opt->offCodeFreq[1] == 1); /* greedy can't find rep1/rep2 */
opt->offCodeFreq[1] = (opt->offCodeFreq[0] / 3) + 1; /* bias correction */
}
opt->offCodeSum = sum_u32(opt->offCodeFreq, 18);
}
{ const BYTE* codePtr = seqStore->mlCode;
U32 u;
for (u=0; u<MaxML+1; u++) opt->matchLengthFreq[u]=1; /* flatten stats; some match length not produced by greedy might end up present */
for (u=0; u<nbSeq; u++) opt->matchLengthFreq[codePtr[u]]++;
if (biasfix == fix_greedy_bias) {
assert(opt->matchLengthFreq[0] == 1); /* greedy can't find mml=3 */
opt->matchLengthFreq[0] = opt->matchLengthFreq[1] + 1; /* bias correction */
}
opt->matchLengthSum = sum_u32(opt->matchLengthFreq, MaxML+1);
}
{ const BYTE* codePtr = seqStore->llCode;
U32 u;
ZSTD_memcpy(opt->litLengthFreq, k_baseLLfreqs, sizeof(k_baseLLfreqs));
for (u=0; u<nbSeq; u++) opt->litLengthFreq[codePtr[u]]++;
opt->litLengthSum = sum_u32(opt->litLengthFreq, MaxLL+1);
}
}
#include "zstd_lazy.h"
/* ZSTD_initStats_greedy():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function cannot error, hence its contract must be respected.
*/
static void
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
ZSTD_initStats_greedy(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
@@ -1268,13 +1328,22 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
DEBUGLOG(4, "ZSTD_initStats_greedy (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
{ size_t const lastLits = ZSTD_compressBlock_greedy(ms, seqStore, tmpRep, src, srcSize); /* generate stats into seqstore */
/* add last lits into literals buffers for proper accounting */
assert(lastLits <= srcSize);
ZSTD_memcpy(seqStore->lit , (const char*)src + srcSize - lastLits, lastLits);
seqStore->lit += lastLits;
}
/* transfer stats from seqStore into ms-opt */
transfer_literalStats(&ms->opt, seqStore);
transfer_seqStats(&ms->opt, seqStore, fix_greedy_bias);
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
@@ -1289,10 +1358,80 @@ size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 const curr = (U32)((const BYTE*)src - ms->window.base);
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
/* 2-pass strategy (when possible):
* this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.4% on first block),
* the cost is +5% cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
&& (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
&& (srcSize > ZSTD_PREDEF_THRESHOLD)
) {
ZSTD_initStats_greedy(ms, seqStore, rep, src, srcSize);
}
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
/* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function cannot error, hence its contract must be respected.
*/
static void
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
size_t lastLits;
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
if (srcSize < 8 KB) {
/* use raw btultra, initialized by default starting stats
* generally preferable for small blocks
*/
lastLits = ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
} else {
/* in this mode, btultra is initialized with greedy strategy;
* it's generally better for larger block sizes */
lastLits = ZSTD_compressBlock_btultra(ms, seqStore, tmpRep, src, srcSize); /* generate stats into ms->opt*/
}
/* transfer stats from seqStore into ms-opt */
assert(lastLits <= srcSize);
ZSTD_memcpy(seqStore->lit , (const char*)src + srcSize - lastLits, lastLits);
seqStore->lit += lastLits;
transfer_literalStats(&ms->opt, seqStore);
transfer_seqStats(&ms->opt, seqStore, from_btultra);
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
+2
View File
@@ -20,6 +20,8 @@ extern "C" {
/* used in ZSTD_loadDictionaryContent() */
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
/* All parsers @return the size of "last literals" segment */
size_t ZSTD_compressBlock_btopt(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
+1 -2
View File
@@ -627,7 +627,6 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
@@ -637,7 +636,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
/* literals stats */
{ const BYTE* bytePtr;
for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
for (bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
countLit[*bytePtr]++;
}