Lines Matching +full:sub +full:- +full:blocks
1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
6 * This source code is licensed under both the BSD-style license (found in the
9 * You may select, at your option, one of the above-listed licenses.
12 /*-*************************************
24 * Compresses literals section for a sub-block.
33 * hufMetadata->hType has literals block type info.
34 * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block.
35 * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block.
36 …* If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Blo…
37 * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
38 * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
39 * @return : compressed size of literals section of a sub-block
50 size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); in ZSTD_compressSubBlock_literal()
55 SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; in ZSTD_compressSubBlock_literal()
61 if (litSize == 0 || hufMetadata->hType == set_basic) { in ZSTD_compressSubBlock_literal()
64 } else if (hufMetadata->hType == set_rle) { in ZSTD_compressSubBlock_literal()
70 assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); in ZSTD_compressSubBlock_literal()
72 if (writeEntropy && hufMetadata->hType == set_compressed) { in ZSTD_compressSubBlock_literal()
73 ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); in ZSTD_compressSubBlock_literal()
74 op += hufMetadata->hufDesSize; in ZSTD_compressSubBlock_literal()
75 cLitSize += hufMetadata->hufDesSize; in ZSTD_compressSubBlock_literal()
76 DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); in ZSTD_compressSubBlock_literal()
80 …const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, li… in ZSTD_compressSubBlock_literal()
81 … : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags); in ZSTD_compressSubBlock_literal()
105 case 3: /* 2 - 2 - 10 - 10 */ in ZSTD_compressSubBlock_literal()
110 case 4: /* 2 - 2 - 14 - 14 */ in ZSTD_compressSubBlock_literal()
115 case 5: /* 2 - 2 - 18 - 18 */ in ZSTD_compressSubBlock_literal()
125 DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); in ZSTD_compressSubBlock_literal()
126 return (size_t)(op-ostart); in ZSTD_compressSubBlock_literal()
154 * Compresses sequences section for a sub-block.
155 * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have
156 * symbol compression modes for the super-block.
159 * The following sub-blocks will always have repeat mode.
160 * @return : compressed size of sequences section of a sub-block
172 const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; in ZSTD_compressSubBlock_sequences()
182 RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, in ZSTD_compressSubBlock_sequences()
189 op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; in ZSTD_compressSubBlock_sequences()
191 return (size_t)(op - ostart); in ZSTD_compressSubBlock_sequences()
197 DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); in ZSTD_compressSubBlock_sequences()
200 const U32 LLtype = fseMetadata->llType; in ZSTD_compressSubBlock_sequences()
201 const U32 Offtype = fseMetadata->ofType; in ZSTD_compressSubBlock_sequences()
202 const U32 MLtype = fseMetadata->mlType; in ZSTD_compressSubBlock_sequences()
203 … DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); in ZSTD_compressSubBlock_sequences()
205 ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); in ZSTD_compressSubBlock_sequences()
206 op += fseMetadata->fseTablesSize; in ZSTD_compressSubBlock_sequences()
213 op, (size_t)(oend - op), in ZSTD_compressSubBlock_sequences()
214 fseTables->matchlengthCTable, mlCode, in ZSTD_compressSubBlock_sequences()
215 fseTables->offcodeCTable, ofCode, in ZSTD_compressSubBlock_sequences()
216 fseTables->litlengthCTable, llCode, in ZSTD_compressSubBlock_sequences()
230 …if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { in ZSTD_compressSubBlock_sequences()
232 assert(fseMetadata->lastCountSize + bitstreamSize == 3); in ZSTD_compressSubBlock_sequences()
249 if (op-seqHead < 4) { in ZSTD_compressSubBlock_sequences()
257 return (size_t)(op - ostart); in ZSTD_compressSubBlock_sequences()
261 * Compresses a single sub-block.
262 * @return : compressed size of the sub-block
281 { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, in ZSTD_compressSubBlock()
282 … &entropyMetadata->hufMetadata, literals, litSize, in ZSTD_compressSubBlock()
283 op, (size_t)(oend-op), in ZSTD_compressSubBlock()
289 { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, in ZSTD_compressSubBlock()
290 &entropyMetadata->fseMetadata, in ZSTD_compressSubBlock()
294 op, (size_t)(oend-op), in ZSTD_compressSubBlock()
301 { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize; in ZSTD_compressSubBlock()
305 return (size_t)(op-ostart); in ZSTD_compressSubBlock()
318 if (hufMetadata->hType == set_basic) return litSize; in ZSTD_estimateSubBlockSize_literal()
319 else if (hufMetadata->hType == set_rle) return 1; in ZSTD_estimateSubBlockSize_literal()
320 else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { in ZSTD_estimateSubBlockSize_literal()
323 …{ size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, … in ZSTD_estimateSubBlockSize_literal()
324 if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; in ZSTD_estimateSubBlockSize_literal()
378 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, in ZSTD_estimateSubBlockSize_sequences()
379 nbSeq, fseTables->offcodeCTable, NULL, in ZSTD_estimateSubBlockSize_sequences()
382 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, in ZSTD_estimateSubBlockSize_sequences()
383 nbSeq, fseTables->litlengthCTable, LL_bits, in ZSTD_estimateSubBlockSize_sequences()
386 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, in ZSTD_estimateSubBlockSize_sequences()
387 nbSeq, fseTables->matchlengthCTable, ML_bits, in ZSTD_estimateSubBlockSize_sequences()
390 if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; in ZSTD_estimateSubBlockSize_sequences()
410 … &entropy->huf, &entropyMetadata->hufMetadata, in ZSTD_estimateSubBlockSize()
413 … nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, in ZSTD_estimateSubBlockSize()
421 if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) in ZSTD_needSequenceEntropyTables()
423 if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) in ZSTD_needSequenceEntropyTables()
425 if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) in ZSTD_needSequenceEntropyTables()
463 /* stop when sub-block budget is reached */ in sizeBlockSequences()
465 /* though continue to expand until the sub-block is deemed compressible */ in sizeBlockSequences()
474 * Breaks super-block into multiple sub-blocks and compresses them.
476 * The following blocks use repeat_mode to compress.
477 * Sub-blocks are all compressed, except the last one when beneficial.
478 * @return : compressed size of the super block (which features multiple ZSTD blocks)
490 const SeqDef* const sstart = seqStorePtr->sequencesStart; in ZSTD_compressSubBlock_multi()
491 const SeqDef* const send = seqStorePtr->sequences; in ZSTD_compressSubBlock_multi()
492 const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */ in ZSTD_compressSubBlock_multi()
493 size_t const nbSeqs = (size_t)(send - sstart); in ZSTD_compressSubBlock_multi()
494 const BYTE* const lstart = seqStorePtr->litStart; in ZSTD_compressSubBlock_multi()
495 const BYTE* const lend = seqStorePtr->lit; in ZSTD_compressSubBlock_multi()
497 size_t const nbLiterals = (size_t)(lend - lstart); in ZSTD_compressSubBlock_multi()
503 const BYTE* llCodePtr = seqStorePtr->llCode; in ZSTD_compressSubBlock_multi()
504 const BYTE* mlCodePtr = seqStorePtr->mlCode; in ZSTD_compressSubBlock_multi()
505 const BYTE* ofCodePtr = seqStorePtr->ofCode; in ZSTD_compressSubBlock_multi()
507 size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize); in ZSTD_compressSubBlock_multi()
508 int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed); in ZSTD_compressSubBlock_multi()
512 (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart)); in ZSTD_compressSubBlock_multi()
519 &nextCBlock->entropy, entropyMetadata, in ZSTD_compressSubBlock_multi()
524 size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs; in ZSTD_compressSubBlock_multi()
535 /* compress and write sub-blocks */ in ZSTD_compressSubBlock_multi()
537 for (n=0; n < nbSubBlocks-1; n++) { in ZSTD_compressSubBlock_multi()
538 /* determine nb of sequences for current sub-block + nbLiterals from next sequence */ in ZSTD_compressSubBlock_multi()
539 size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp), in ZSTD_compressSubBlock_multi()
541 /* if reached last sequence : break to last sub-block (simplification) */ in ZSTD_compressSubBlock_multi()
542 assert(seqCount <= (size_t)(send-sp)); in ZSTD_compressSubBlock_multi()
545 /* compress sub-block */ in ZSTD_compressSubBlock_multi()
551 size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, in ZSTD_compressSubBlock_multi()
556 op, (size_t)(oend-op), in ZSTD_compressSubBlock_multi()
564 DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes", in ZSTD_compressSubBlock_multi()
583 /* otherwise : do not compress yet, coalesce current sub-block with following one */ in ZSTD_compressSubBlock_multi()
588 DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp)); in ZSTD_compressSubBlock_multi()
591 size_t litSize = (size_t)(lend - lp); in ZSTD_compressSubBlock_multi()
592 size_t seqCount = (size_t)(send - sp); in ZSTD_compressSubBlock_multi()
595 size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, in ZSTD_compressSubBlock_multi()
600 op, (size_t)(oend-op), in ZSTD_compressSubBlock_multi()
608 DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes", in ZSTD_compressSubBlock_multi()
631 … ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); in ZSTD_compressSubBlock_multi()
633 if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { in ZSTD_compressSubBlock_multi()
643 size_t const rSize = (size_t)((iend - ip)); in ZSTD_compressSubBlock_multi()
644 size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock); in ZSTD_compressSubBlock_multi()
645 DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize)); in ZSTD_compressSubBlock_multi()
653 ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); in ZSTD_compressSubBlock_multi()
655 … ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); in ZSTD_compressSubBlock_multi()
657 ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); in ZSTD_compressSubBlock_multi()
662 (unsigned)(op-ostart)); in ZSTD_compressSubBlock_multi()
663 return (size_t)(op-ostart); in ZSTD_compressSubBlock_multi()
673 FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, in ZSTD_compressSuperBlock()
674 &zc->blockState.prevCBlock->entropy, in ZSTD_compressSuperBlock()
675 &zc->blockState.nextCBlock->entropy, in ZSTD_compressSuperBlock()
676 &zc->appliedParams, in ZSTD_compressSuperBlock()
678 zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), ""); in ZSTD_compressSuperBlock()
680 return ZSTD_compressSubBlock_multi(&zc->seqStore, in ZSTD_compressSuperBlock()
681 zc->blockState.prevCBlock, in ZSTD_compressSuperBlock()
682 zc->blockState.nextCBlock, in ZSTD_compressSuperBlock()
684 &zc->appliedParams, in ZSTD_compressSuperBlock()
687 zc->bmi2, lastBlock, in ZSTD_compressSuperBlock()
688 zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); in ZSTD_compressSuperBlock()