Lines Matching +full:assert +full:- +full:falling +full:- +full:edge
1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
6 * This source code is licensed under both the BSD-style license (found in the
9 * You may select, at your option, one of the above-listed licenses.
15 /*-*******************************************************
51 /*-*************************************************************
57 …size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSI… in ZSTD_blockSizeMax()
58 assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); in ZSTD_blockSizeMax()
71 bpPtr->lastBlock = cBlockHeader & 1; in ZSTD_getcBlockSize()
72 bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); in ZSTD_getcBlockSize()
73 bpPtr->origSize = cSize; /* only useful for RLE */ in ZSTD_getcBlockSize()
74 if (bpPtr->blockType == bt_rle) return 1; in ZSTD_getcBlockSize()
75 RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); in ZSTD_getcBlockSize()
85 assert(litSize <= blockSizeMax); in ZSTD_allocateLiteralsBuffer()
86 assert(dctx->isFrameDecompression || streaming == not_streaming); in ZSTD_allocateLiteralsBuffer()
87 assert(expectedWriteSize <= blockSizeMax); in ZSTD_allocateLiteralsBuffer()
94 dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; in ZSTD_allocateLiteralsBuffer()
95 dctx->litBufferEnd = dctx->litBuffer + litSize; in ZSTD_allocateLiteralsBuffer()
96 dctx->litBufferLocation = ZSTD_in_dst; in ZSTD_allocateLiteralsBuffer()
101 dctx->litBuffer = dctx->litExtraBuffer; in ZSTD_allocateLiteralsBuffer()
102 dctx->litBufferEnd = dctx->litBuffer + litSize; in ZSTD_allocateLiteralsBuffer()
103 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_allocateLiteralsBuffer()
105 assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); in ZSTD_allocateLiteralsBuffer()
115 …dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OV… in ZSTD_allocateLiteralsBuffer()
116 dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; in ZSTD_allocateLiteralsBuffer()
119 dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; in ZSTD_allocateLiteralsBuffer()
120 dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; in ZSTD_allocateLiteralsBuffer()
122 dctx->litBufferLocation = ZSTD_split; in ZSTD_allocateLiteralsBuffer()
123 assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); in ZSTD_allocateLiteralsBuffer()
131 * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
149 DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); in ZSTD_decodeLiteralsBlock()
150 RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); in ZSTD_decodeLiteralsBlock()
163 | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); in ZSTD_decodeLiteralsBlock()
167 /* 2 - 2 - 10 - 10 */ in ZSTD_decodeLiteralsBlock()
174 /* 2 - 2 - 14 - 14 */ in ZSTD_decodeLiteralsBlock()
180 /* 2 - 2 - 18 - 18 */ in ZSTD_decodeLiteralsBlock()
190 "Not enough literals (%zu) for the 4-streams mode (min %u)", in ZSTD_decodeLiteralsBlock()
197 if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { in ZSTD_decodeLiteralsBlock()
198 PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); in ZSTD_decodeLiteralsBlock()
204 dctx->litBuffer, litSize, istart+lhSize, litCSize, in ZSTD_decodeLiteralsBlock()
205 dctx->HUFptr, flags); in ZSTD_decodeLiteralsBlock()
207 assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); in ZSTD_decodeLiteralsBlock()
209 dctx->litBuffer, litSize, istart+lhSize, litCSize, in ZSTD_decodeLiteralsBlock()
210 dctx->HUFptr, flags); in ZSTD_decodeLiteralsBlock()
216 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
217 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
218 sizeof(dctx->workspace), flags); in ZSTD_decodeLiteralsBlock()
221 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
222 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
223 sizeof(dctx->workspace), flags); in ZSTD_decodeLiteralsBlock()
227 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
228 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
229 sizeof(dctx->workspace), flags); in ZSTD_decodeLiteralsBlock()
232 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
234 assert(litSize > ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
235 …ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTR… in ZSTD_decodeLiteralsBlock()
236 …ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, lit… in ZSTD_decodeLiteralsBlock()
237 dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; in ZSTD_decodeLiteralsBlock()
238 dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; in ZSTD_decodeLiteralsBlock()
239 assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); in ZSTD_decodeLiteralsBlock()
244 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
245 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
246 dctx->litEntropy = 1; in ZSTD_decodeLiteralsBlock()
247 if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; in ZSTD_decodeLiteralsBlock()
278 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
280 … ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
281 …ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUF… in ZSTD_decodeLiteralsBlock()
285 ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); in ZSTD_decodeLiteralsBlock()
287 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
288 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
292 dctx->litPtr = istart+lhSize; in ZSTD_decodeLiteralsBlock()
293 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
294 dctx->litBufferEnd = dctx->litPtr + litSize; in ZSTD_decodeLiteralsBlock()
295 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decodeLiteralsBlock()
324 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
326 ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
327 ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
331 ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); in ZSTD_decodeLiteralsBlock()
333 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
334 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
351 dctx->isFrameDecompression = 0; in ZSTD_decodeLiteralsBlock_wrapper()
356 …* These are pre-calculated FSE decoding tables using default distributions as defined in specifica…
357 …* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributio…
359 * - start from default distributions, present in /lib/common/zstd_internal.h
360 * - generate tables normally, using ZSTD_buildFSETable()
361 * - printout the content of tables
362 * - prettify output, report below, test with fuzzer to ensure it's correct */
470 DTableH->tableLog = 0; in ZSTD_buildSeqTable_rle()
471 DTableH->fastMode = 0; in ZSTD_buildSeqTable_rle()
473 cell->nbBits = 0; in ZSTD_buildSeqTable_rle()
474 cell->nextState = 0; in ZSTD_buildSeqTable_rle()
475 assert(nbAddBits < 255); in ZSTD_buildSeqTable_rle()
476 cell->nbAdditionalBits = nbAddBits; in ZSTD_buildSeqTable_rle()
477 cell->baseValue = baseValue; in ZSTD_buildSeqTable_rle()
497 U32 highThreshold = tableSize - 1; in ZSTD_buildFSETable_body()
501 assert(maxSymbolValue <= MaxSeq); in ZSTD_buildFSETable_body()
502 assert(tableLog <= MaxFSELog); in ZSTD_buildFSETable_body()
503 assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); in ZSTD_buildFSETable_body()
509 { S16 const largeLimit= (S16)(1 << (tableLog-1)); in ZSTD_buildFSETable_body()
512 if (normalizedCounter[s]==-1) { in ZSTD_buildFSETable_body()
513 tableDecode[highThreshold--].baseValue = s; in ZSTD_buildFSETable_body()
517 assert(normalizedCounter[s]>=0); in ZSTD_buildFSETable_body()
524 assert(tableSize <= 512); in ZSTD_buildFSETable_body()
526 * no low probability (-1 count) symbols. When compressing in ZSTD_buildFSETable_body()
530 if (highThreshold == tableSize - 1) { in ZSTD_buildFSETable_body()
531 size_t const tableMask = tableSize-1; in ZSTD_buildFSETable_body()
537 * our buffer to handle the over-write. in ZSTD_buildFSETable_body()
551 assert(n>=0); in ZSTD_buildFSETable_body()
565 assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ in ZSTD_buildFSETable_body()
574 assert(position == 0); in ZSTD_buildFSETable_body()
577 U32 const tableMask = tableSize-1; in ZSTD_buildFSETable_body()
588 …assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incor… in ZSTD_buildFSETable_body()
597 tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) ); in ZSTD_buildFSETable_body()
598 tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); in ZSTD_buildFSETable_body()
599 assert(nbAdditionalBits[symbol] < 255); in ZSTD_buildFSETable_body()
691 assert(0); in ZSTD_buildSeqTable()
717 nbSeq = ((nbSeq-0x80)<<8) + *ip++; in ZSTD_decodeSeqHeaders()
726 return (size_t)(ip - istart); in ZSTD_decodeSeqHeaders()
731 …RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes… in ZSTD_decodeSeqHeaders()
738 { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, in ZSTD_decodeSeqHeaders()
740 ip, iend-ip, in ZSTD_decodeSeqHeaders()
742 LL_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
743 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
744 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
750 { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, in ZSTD_decodeSeqHeaders()
752 ip, iend-ip, in ZSTD_decodeSeqHeaders()
754 OF_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
755 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
756 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
762 { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, in ZSTD_decodeSeqHeaders()
764 ip, iend-ip, in ZSTD_decodeSeqHeaders()
766 ML_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
767 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
768 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
775 return ip-istart; in ZSTD_decodeSeqHeaders()
803 * Postcondition: *op - *op >= 8
806 assert(*ip <= *op); in ZSTD_overlapCopy8()
818 *ip -= sub2; in ZSTD_overlapCopy8()
824 assert(*op - *ip >= 8); in ZSTD_overlapCopy8()
834 …* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VEC…
835 * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
839 ptrdiff_t const diff = op - ip; in ZSTD_safecopy()
842 assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || in ZSTD_safecopy()
852 assert(length >= 8); in ZSTD_safecopy()
854 length -= 8; in ZSTD_safecopy()
855 assert(op - ip >= 8); in ZSTD_safecopy()
856 assert(op <= oend); in ZSTD_safecopy()
866 assert(oend > oend_w); in ZSTD_safecopy()
867 ZSTD_wildcopy(op, ip, oend_w - op, ovtype); in ZSTD_safecopy()
868 ip += oend_w - op; in ZSTD_safecopy()
869 op += oend_w - op; in ZSTD_safecopy()
876 …* This version allows overlap with dst before src, or handles the non-overlap case with dst after …
879 ptrdiff_t const diff = op - ip; in ZSTD_safecopyDstBeforeSrc()
882 if (length < 8 || diff > -8) { in ZSTD_safecopyDstBeforeSrc()
888 if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { in ZSTD_safecopyDstBeforeSrc()
889 ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); in ZSTD_safecopyDstBeforeSrc()
890 ip += oend - WILDCOPY_OVERLENGTH - op; in ZSTD_safecopyDstBeforeSrc()
891 op += oend - WILDCOPY_OVERLENGTH - op; in ZSTD_safecopyDstBeforeSrc()
916 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceEnd()
917 BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; in ZSTD_execSequenceEnd()
919 /* bounds checks : careful of address space overflow in 32-bit mode */ in ZSTD_execSequenceEnd()
920 …RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit withi… in ZSTD_execSequenceEnd()
921 …RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to re… in ZSTD_execSequenceEnd()
922 assert(op < op + sequenceLength); in ZSTD_execSequenceEnd()
923 assert(oLitEnd < op + sequenceLength); in ZSTD_execSequenceEnd()
931 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceEnd()
933 … RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); in ZSTD_execSequenceEnd()
934 match = dictEnd - (prefixStart - match); in ZSTD_execSequenceEnd()
940 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceEnd()
943 sequence.matchLength -= length1; in ZSTD_execSequenceEnd()
964 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceEndSplitLitBuffer()
967 /* bounds checks : careful of address space overflow in 32-bit mode */ in ZSTD_execSequenceEndSplitLitBuffer()
968 …RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit withi… in ZSTD_execSequenceEndSplitLitBuffer()
969 …RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to re… in ZSTD_execSequenceEndSplitLitBuffer()
970 assert(op < op + sequenceLength); in ZSTD_execSequenceEndSplitLitBuffer()
971 assert(oLitEnd < op + sequenceLength); in ZSTD_execSequenceEndSplitLitBuffer()
980 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceEndSplitLitBuffer()
982 … RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); in ZSTD_execSequenceEndSplitLitBuffer()
983 match = dictEnd - (prefixStart - match); in ZSTD_execSequenceEndSplitLitBuffer()
989 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceEndSplitLitBuffer()
992 sequence.matchLength -= length1; in ZSTD_execSequenceEndSplitLitBuffer()
1009 BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ in ZSTD_execSequence()
1010 …BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL … in ZSTD_execSequence()
1012 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequence()
1014 assert(op != NULL /* Precondition */); in ZSTD_execSequence()
1015 assert(oend_w < oend /* No underflow */); in ZSTD_execSequence()
1021 /* Handle edge cases in a slow path: in ZSTD_execSequence()
1022 * - Read beyond end of literals in ZSTD_execSequence()
1023 * - Match end is within WILDCOPY_OVERLIMIT of oend in ZSTD_execSequence()
1024 * - 32-bit mode and the match length overflows in ZSTD_execSequence()
1029 (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) in ZSTD_execSequence()
1033 assert(op <= oLitEnd /* No overflow */); in ZSTD_execSequence()
1034 assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); in ZSTD_execSequence()
1035 assert(oMatchEnd <= oend /* No underflow */); in ZSTD_execSequence()
1036 assert(iLitEnd <= litLimit /* Literal length is in bounds */); in ZSTD_execSequence()
1037 assert(oLitEnd <= oend_w /* Can wildcopy literals */); in ZSTD_execSequence()
1038 assert(oMatchEnd <= oend_w /* Can wildcopy matches */); in ZSTD_execSequence()
1041 * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. in ZSTD_execSequence()
1042 * We likely don't need the full 32-byte wildcopy. in ZSTD_execSequence()
1044 assert(WILDCOPY_OVERLENGTH >= 16); in ZSTD_execSequence()
1047 ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); in ZSTD_execSequence()
1053 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequence()
1054 /* offset beyond prefix -> go into extDict */ in ZSTD_execSequence()
1055 …RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected,… in ZSTD_execSequence()
1056 match = dictEnd + (match - prefixStart); in ZSTD_execSequence()
1062 { size_t const length1 = dictEnd - match; in ZSTD_execSequence()
1065 sequence.matchLength -= length1; in ZSTD_execSequence()
1070 assert(op <= oMatchEnd); in ZSTD_execSequence()
1071 assert(oMatchEnd <= oend_w); in ZSTD_execSequence()
1072 assert(match >= prefixStart); in ZSTD_execSequence()
1073 assert(sequence.matchLength >= 1); in ZSTD_execSequence()
1086 assert(sequence.offset < WILDCOPY_VECLEN); in ZSTD_execSequence()
1093 assert(op < oMatchEnd); in ZSTD_execSequence()
1094 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); in ZSTD_execSequence()
1108 BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ in ZSTD_execSequenceSplitLitBuffer()
1110 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceSplitLitBuffer()
1112 assert(op != NULL /* Precondition */); in ZSTD_execSequenceSplitLitBuffer()
1113 assert(oend_w < oend /* No underflow */); in ZSTD_execSequenceSplitLitBuffer()
1114 /* Handle edge cases in a slow path: in ZSTD_execSequenceSplitLitBuffer()
1115 * - Read beyond end of literals in ZSTD_execSequenceSplitLitBuffer()
1116 * - Match end is within WILDCOPY_OVERLIMIT of oend in ZSTD_execSequenceSplitLitBuffer()
1117 * - 32-bit mode and the match length overflows in ZSTD_execSequenceSplitLitBuffer()
1122 (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) in ZSTD_execSequenceSplitLitBuffer()
1126 assert(op <= oLitEnd /* No overflow */); in ZSTD_execSequenceSplitLitBuffer()
1127 assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); in ZSTD_execSequenceSplitLitBuffer()
1128 assert(oMatchEnd <= oend /* No underflow */); in ZSTD_execSequenceSplitLitBuffer()
1129 assert(iLitEnd <= litLimit /* Literal length is in bounds */); in ZSTD_execSequenceSplitLitBuffer()
1130 assert(oLitEnd <= oend_w /* Can wildcopy literals */); in ZSTD_execSequenceSplitLitBuffer()
1131 assert(oMatchEnd <= oend_w /* Can wildcopy matches */); in ZSTD_execSequenceSplitLitBuffer()
1134 * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. in ZSTD_execSequenceSplitLitBuffer()
1135 * We likely don't need the full 32-byte wildcopy. in ZSTD_execSequenceSplitLitBuffer()
1137 assert(WILDCOPY_OVERLENGTH >= 16); in ZSTD_execSequenceSplitLitBuffer()
1140 ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); in ZSTD_execSequenceSplitLitBuffer()
1146 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceSplitLitBuffer()
1147 /* offset beyond prefix -> go into extDict */ in ZSTD_execSequenceSplitLitBuffer()
1148 …RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected,… in ZSTD_execSequenceSplitLitBuffer()
1149 match = dictEnd + (match - prefixStart); in ZSTD_execSequenceSplitLitBuffer()
1155 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceSplitLitBuffer()
1158 sequence.matchLength -= length1; in ZSTD_execSequenceSplitLitBuffer()
1162 assert(op <= oMatchEnd); in ZSTD_execSequenceSplitLitBuffer()
1163 assert(oMatchEnd <= oend_w); in ZSTD_execSequenceSplitLitBuffer()
1164 assert(match >= prefixStart); in ZSTD_execSequenceSplitLitBuffer()
1165 assert(sequence.matchLength >= 1); in ZSTD_execSequenceSplitLitBuffer()
1178 assert(sequence.offset < WILDCOPY_VECLEN); in ZSTD_execSequenceSplitLitBuffer()
1185 assert(op < oMatchEnd); in ZSTD_execSequenceSplitLitBuffer()
1186 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); in ZSTD_execSequenceSplitLitBuffer()
1197 DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); in ZSTD_initFseState()
1199 (U32)DStatePtr->state, DTableH->tableLog); in ZSTD_initFseState()
1201 DStatePtr->table = dt + 1; in ZSTD_initFseState()
1208 DStatePtr->state = nextState + lowBits; in ZSTD_updateFseStateWithDInfo()
1211 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
1218 ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
1226 * only used in 32-bit mode
1236 * and its fields extracted by simply shifting or bit-extracting on aarch64. in ZSTD_decodeSequence()
1246 ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); in ZSTD_decodeSequence()
1247 ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); in ZSTD_decodeSequence()
1248 …ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)… in ZSTD_decodeSequence()
1250 const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; in ZSTD_decodeSequence()
1251 const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; in ZSTD_decodeSequence()
1252 const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; in ZSTD_decodeSequence()
1254 seq.matchLength = mlDInfo->baseValue; in ZSTD_decodeSequence()
1255 seq.litLength = llDInfo->baseValue; in ZSTD_decodeSequence()
1256 { U32 const ofBase = ofDInfo->baseValue; in ZSTD_decodeSequence()
1257 BYTE const llBits = llDInfo->nbAdditionalBits; in ZSTD_decodeSequence()
1258 BYTE const mlBits = mlDInfo->nbAdditionalBits; in ZSTD_decodeSequence()
1259 BYTE const ofBits = ofDInfo->nbAdditionalBits; in ZSTD_decodeSequence()
1262 U16 const llNext = llDInfo->nextState; in ZSTD_decodeSequence()
1263 U16 const mlNext = mlDInfo->nextState; in ZSTD_decodeSequence()
1264 U16 const ofNext = ofDInfo->nextState; in ZSTD_decodeSequence()
1265 U32 const llnbBits = llDInfo->nbBits; in ZSTD_decodeSequence()
1266 U32 const mlnbBits = mlDInfo->nbBits; in ZSTD_decodeSequence()
1267 U32 const ofnbBits = ofDInfo->nbBits; in ZSTD_decodeSequence()
1269 assert(llBits <= MaxLLBits); in ZSTD_decodeSequence()
1270 assert(mlBits <= MaxMLBits); in ZSTD_decodeSequence()
1271 assert(ofBits <= MaxOff); in ZSTD_decodeSequence()
1274 * valuable to mark likeliness for clang, it gives around 3-4% of in ZSTD_decodeSequence()
1284 … ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); in ZSTD_decodeSequence()
1290 … offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); in ZSTD_decodeSequence()
1291 BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1292 offset += BIT_readBitsFast(&seqState->DStream, extraBits); in ZSTD_decodeSequence()
1294 …offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX… in ZSTD_decodeSequence()
1295 if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1297 seqState->prevOffset[2] = seqState->prevOffset[1]; in ZSTD_decodeSequence()
1298 seqState->prevOffset[1] = seqState->prevOffset[0]; in ZSTD_decodeSequence()
1299 seqState->prevOffset[0] = offset; in ZSTD_decodeSequence()
1301 U32 const ll0 = (llDInfo->baseValue == 0); in ZSTD_decodeSequence()
1303 offset = seqState->prevOffset[ll0]; in ZSTD_decodeSequence()
1304 seqState->prevOffset[1] = seqState->prevOffset[!ll0]; in ZSTD_decodeSequence()
1305 seqState->prevOffset[0] = offset; in ZSTD_decodeSequence()
1307 offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); in ZSTD_decodeSequence()
1308 … { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; in ZSTD_decodeSequence()
1309 …temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at … in ZSTD_decodeSequence()
1310 if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; in ZSTD_decodeSequence()
1311 seqState->prevOffset[1] = seqState->prevOffset[0]; in ZSTD_decodeSequence()
1312 seqState->prevOffset[0] = offset = temp; in ZSTD_decodeSequence()
1318 seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); in ZSTD_decodeSequence()
1320 … if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) in ZSTD_decodeSequence()
1321 BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1322 …if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) in ZSTD_decodeSequence()
1323 BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1324 /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ in ZSTD_decodeSequence()
1328 seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); in ZSTD_decodeSequence()
1331 BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1338 …ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= … in ZSTD_decodeSequence()
1339 …ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= … in ZSTD_decodeSequence()
1340 if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ in ZSTD_decodeSequence()
1341 …ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= … in ZSTD_decodeSequence()
1342 BIT_reloadDStream(&seqState->DStream); in ZSTD_decodeSequence()
1353 size_t const windowSize = dctx->fParams.windowSize; in ZSTD_dictionaryIsActive()
1355 if (dctx->dictContentEndForFuzzing == NULL) return 0; in ZSTD_dictionaryIsActive()
1357 if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; in ZSTD_dictionaryIsActive()
1358 /* Dictionary is not our ext-dict. */ in ZSTD_dictionaryIsActive()
1359 if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; in ZSTD_dictionaryIsActive()
1361 if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; in ZSTD_dictionaryIsActive()
1374 if (dctx->isFrameDecompression) { in ZSTD_assertValidSequence()
1375 size_t const windowSize = dctx->fParams.windowSize; in ZSTD_assertValidSequence()
1380 assert(op <= oend); in ZSTD_assertValidSequence()
1381 assert((size_t)(oend - op) >= sequenceSize); in ZSTD_assertValidSequence()
1382 assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); in ZSTD_assertValidSequence()
1384 …size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->… in ZSTD_assertValidSequence()
1386 assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); in ZSTD_assertValidSequence()
1387 assert(seq.offset <= windowSize + dictSize); in ZSTD_assertValidSequence()
1390 assert(seq.offset <= windowSize); in ZSTD_assertValidSequence()
1414 const BYTE* litPtr = dctx->litPtr; in ZSTD_decompressSequences_bodySplitLitBuffer()
1415 const BYTE* litBufferEnd = dctx->litBufferEnd; in ZSTD_decompressSequences_bodySplitLitBuffer()
1416 const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); in ZSTD_decompressSequences_bodySplitLitBuffer()
1417 const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); in ZSTD_decompressSequences_bodySplitLitBuffer()
1418 const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); in ZSTD_decompressSequences_bodySplitLitBuffer()
1424 dctx->fseEntropy = 1; in ZSTD_decompressSequences_bodySplitLitBuffer()
1425 { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } in ZSTD_decompressSequences_bodySplitLitBuffer()
1427 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), in ZSTD_decompressSequences_bodySplitLitBuffer()
1429 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); in ZSTD_decompressSequences_bodySplitLitBuffer()
1430 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); in ZSTD_decompressSequences_bodySplitLitBuffer()
1431 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); in ZSTD_decompressSequences_bodySplitLitBuffer()
1432 assert(dst != NULL); in ZSTD_decompressSequences_bodySplitLitBuffer()
1443 * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression in ZSTD_decompressSequences_bodySplitLitBuffer()
1445 * performance swing is caused by parts of the decompression loop falling in ZSTD_decompressSequences_bodySplitLitBuffer()
1451 * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ in ZSTD_decompressSequences_bodySplitLitBuffer()
1452 * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst in ZSTD_decompressSequences_bodySplitLitBuffer()
1459 * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 in ZSTD_decompressSequences_bodySplitLitBuffer()
1460 * Use Instruments->Counters to get DSB/MITE cycles. in ZSTD_decompressSequences_bodySplitLitBuffer()
1464 * - Coffeelake: Intel i9-9900k in ZSTD_decompressSequences_bodySplitLitBuffer()
1465 * - Coffeelake: Intel i7-9700k in ZSTD_decompressSequences_bodySplitLitBuffer()
1469 * - Haswell in ZSTD_decompressSequences_bodySplitLitBuffer()
1470 * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH in ZSTD_decompressSequences_bodySplitLitBuffer()
1471 * - Skylake in ZSTD_decompressSequences_bodySplitLitBuffer()
1474 … * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer in ZSTD_decompressSequences_bodySplitLitBuffer()
1475 … * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer in ZSTD_decompressSequences_bodySplitLitBuffer()
1476 * - ZSTD_decompressSequences_body in ZSTD_decompressSequences_bodySplitLitBuffer()
1488 /* good for gcc-7, gcc-9, and gcc-11 */ in ZSTD_decompressSequences_bodySplitLitBuffer()
1494 /* good for gcc-8 and gcc-10 */ in ZSTD_decompressSequences_bodySplitLitBuffer()
1502 for ( ; nbSeq; nbSeq--) { in ZSTD_decompressSequences_bodySplitLitBuffer()
1504 if (litPtr + sequence.litLength > dctx->litBufferEnd) break; in ZSTD_decompressSequences_bodySplitLitBuffer()
1505 …ize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH,… in ZSTD_decompressSequences_bodySplitLitBuffer()
1507 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequences_bodySplitLitBuffer()
1515 DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); in ZSTD_decompressSequences_bodySplitLitBuffer()
1519 const size_t leftoverLit = dctx->litBufferEnd - litPtr; in ZSTD_decompressSequences_bodySplitLitBuffer()
1522 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi… in ZSTD_decompressSequences_bodySplitLitBuffer()
1524 sequence.litLength -= leftoverLit; in ZSTD_decompressSequences_bodySplitLitBuffer()
1527 litPtr = dctx->litExtraBuffer; in ZSTD_decompressSequences_bodySplitLitBuffer()
1528 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; in ZSTD_decompressSequences_bodySplitLitBuffer()
1529 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decompressSequences_bodySplitLitBuffer()
1532 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequences_bodySplitLitBuffer()
1540 nbSeq--; in ZSTD_decompressSequences_bodySplitLitBuffer()
1551 /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ in ZSTD_decompressSequences_bodySplitLitBuffer()
1564 for ( ; nbSeq ; nbSeq--) { in ZSTD_decompressSequences_bodySplitLitBuffer()
1568 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequences_bodySplitLitBuffer()
1584 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } in ZSTD_decompressSequences_bodySplitLitBuffer()
1588 if (dctx->litBufferLocation == ZSTD_split) { in ZSTD_decompressSequences_bodySplitLitBuffer()
1590 size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); in ZSTD_decompressSequences_bodySplitLitBuffer()
1592 RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); in ZSTD_decompressSequences_bodySplitLitBuffer()
1597 litPtr = dctx->litExtraBuffer; in ZSTD_decompressSequences_bodySplitLitBuffer()
1598 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; in ZSTD_decompressSequences_bodySplitLitBuffer()
1599 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decompressSequences_bodySplitLitBuffer()
1602 { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); in ZSTD_decompressSequences_bodySplitLitBuffer()
1604 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); in ZSTD_decompressSequences_bodySplitLitBuffer()
1610 DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); in ZSTD_decompressSequences_bodySplitLitBuffer()
1611 return (size_t)(op - ostart); in ZSTD_decompressSequences_bodySplitLitBuffer()
1624 …BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDs… in ZSTD_decompressSequences_body()
1626 const BYTE* litPtr = dctx->litPtr; in ZSTD_decompressSequences_body()
1627 const BYTE* const litEnd = litPtr + dctx->litSize; in ZSTD_decompressSequences_body()
1628 const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); in ZSTD_decompressSequences_body()
1629 const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); in ZSTD_decompressSequences_body()
1630 const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); in ZSTD_decompressSequences_body()
1636 dctx->fseEntropy = 1; in ZSTD_decompressSequences_body()
1637 { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } in ZSTD_decompressSequences_body()
1639 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), in ZSTD_decompressSequences_body()
1641 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); in ZSTD_decompressSequences_body()
1642 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); in ZSTD_decompressSequences_body()
1643 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); in ZSTD_decompressSequences_body()
1644 assert(dst != NULL); in ZSTD_decompressSequences_body()
1660 for ( ; nbSeq ; nbSeq--) { in ZSTD_decompressSequences_body()
1664 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequences_body()
1674 assert(nbSeq == 0); in ZSTD_decompressSequences_body()
1677 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } in ZSTD_decompressSequences_body()
1681 { size_t const lastLLSize = (size_t)(litEnd - litPtr); in ZSTD_decompressSequences_body()
1683 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); in ZSTD_decompressSequences_body()
1689 DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); in ZSTD_decompressSequences_body()
1690 return (size_t)(op - ostart); in ZSTD_decompressSequences_body()
1731 * It's generally employed when block contains a significant portion of long-distance matches
1743 …BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd… in ZSTD_decompressSequencesLong_body()
1745 const BYTE* litPtr = dctx->litPtr; in ZSTD_decompressSequencesLong_body()
1746 const BYTE* litBufferEnd = dctx->litBufferEnd; in ZSTD_decompressSequencesLong_body()
1747 const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); in ZSTD_decompressSequencesLong_body()
1748 const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); in ZSTD_decompressSequencesLong_body()
1749 const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); in ZSTD_decompressSequencesLong_body()
1754 #define STORED_SEQS_MASK (STORED_SEQS-1) in ZSTD_decompressSequencesLong_body()
1760 size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ in ZSTD_decompressSequencesLong_body()
1762 dctx->fseEntropy = 1; in ZSTD_decompressSequencesLong_body()
1763 { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } in ZSTD_decompressSequencesLong_body()
1764 assert(dst != NULL); in ZSTD_decompressSequencesLong_body()
1765 assert(iend >= ip); in ZSTD_decompressSequencesLong_body()
1767 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), in ZSTD_decompressSequencesLong_body()
1769 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); in ZSTD_decompressSequencesLong_body()
1770 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); in ZSTD_decompressSequencesLong_body()
1771 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); in ZSTD_decompressSequencesLong_body()
1775 seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1); in ZSTD_decompressSequencesLong_body()
1782 seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1); in ZSTD_decompressSequencesLong_body()
1784 …if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_S… in ZSTD_decompressSequencesLong_body()
1786 const size_t leftoverLit = dctx->litBufferEnd - litPtr; in ZSTD_decompressSequencesLong_body()
1789 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi… in ZSTD_decompressSequencesLong_body()
1791 sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; in ZSTD_decompressSequencesLong_body()
1794 litPtr = dctx->litExtraBuffer; in ZSTD_decompressSequencesLong_body()
1795 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; in ZSTD_decompressSequencesLong_body()
1796 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decompressSequencesLong_body()
1797 …{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOR… in ZSTD_decompressSequencesLong_body()
1799 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequencesLong_body()
1800 …ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], pr… in ZSTD_decompressSequencesLong_body()
1811 size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? in ZSTD_decompressSequencesLong_body()
1812 …oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENG… in ZSTD_decompressSequencesLong_body()
1813 …ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBuf… in ZSTD_decompressSequencesLong_body()
1815 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequencesLong_body()
1816 …ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], pr… in ZSTD_decompressSequencesLong_body()
1828 seqNb -= seqAdvance; in ZSTD_decompressSequencesLong_body()
1831 … if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { in ZSTD_decompressSequencesLong_body()
1832 const size_t leftoverLit = dctx->litBufferEnd - litPtr; in ZSTD_decompressSequencesLong_body()
1834 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi… in ZSTD_decompressSequencesLong_body()
1836 sequence->litLength -= leftoverLit; in ZSTD_decompressSequencesLong_body()
1839 litPtr = dctx->litExtraBuffer; in ZSTD_decompressSequencesLong_body()
1840 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; in ZSTD_decompressSequencesLong_body()
1841 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decompressSequencesLong_body()
1844 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequencesLong_body()
1853 size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? in ZSTD_decompressSequencesLong_body()
1854 …ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *seq… in ZSTD_decompressSequencesLong_body()
1857 assert(!ZSTD_isError(oneSeqSize)); in ZSTD_decompressSequencesLong_body()
1866 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } in ZSTD_decompressSequencesLong_body()
1870 …if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy lit… in ZSTD_decompressSequencesLong_body()
1871 size_t const lastLLSize = litBufferEnd - litPtr; in ZSTD_decompressSequencesLong_body()
1872 RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); in ZSTD_decompressSequencesLong_body()
1877 litPtr = dctx->litExtraBuffer; in ZSTD_decompressSequencesLong_body()
1878 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; in ZSTD_decompressSequencesLong_body()
1880 { size_t const lastLLSize = litBufferEnd - litPtr; in ZSTD_decompressSequencesLong_body()
1881 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); in ZSTD_decompressSequencesLong_body()
1888 return (size_t)(op - ostart); in ZSTD_decompressSequencesLong_body()
1999 return (size_t)(op - virtualStart); in ZSTD_totalHistorySize()
2028 assert(max <= (1 << OffFSELog)); /* max not too large */ in ZSTD_getOffsetInfo()
2034 assert(tableLog <= OffFSELog); in ZSTD_getOffsetInfo()
2035 info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ in ZSTD_getOffsetInfo()
2053 return (size_t)-1; in ZSTD_maxShortOffset()
2055 /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. in ZSTD_maxShortOffset()
2059 size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; in ZSTD_maxShortOffset()
2060 size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; in ZSTD_maxShortOffset()
2061 assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); in ZSTD_maxShortOffset()
2079 * would consider this edge case as an error.
2086 … DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
2089 srcSize -= litCSize;
2098 …_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart);
2101 * We don't expect that to be the case in 64-bit mode.
2111 /* These macros control at build-time which decompressor implementation
2117 int usePrefetchDecoder = dctx->ddictIsCold;
2128 srcSize -= seqHSize;
2131 …RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (s…
2139 ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
2153 dctx->ddictIsCold = 0;
2169 if (dctx->litBufferLocation == ZSTD_split)
2181 if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
2182 dctx->dictEnd = dctx->previousDstEnd;
2183 …dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx-…
2184 dctx->prefixStart = dst;
2185 dctx->previousDstEnd = dst;
2195 dctx->isFrameDecompression = 0;
2199 dctx->previousDstEnd = (char*)dst + dSize;