diff --git a/README b/README index 71a5026..4e05352 100644 --- a/README +++ b/README @@ -244,7 +244,7 @@ A. It has been tested on gcc, ekopath and the intel compiler successfully previously. Whether the commercial compilers help or not, I could not tell you. Q. What codebase are you basing this on? -A. rzip v2.1 and lzma sdk907, but it should be possible to stay in sync with +A. rzip v2.1 and lzma sdk920, but it should be possible to stay in sync with each of these in the future. Q. Do we really need yet another compression format? diff --git a/lrzip_private.h b/lrzip_private.h index be63ac5..e30a586 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -371,6 +371,7 @@ struct stream_info { i64 bufsize; i64 cur_pos; i64 initial_pos; + i64 total_read; i64 ram_alloced; i64 size; long thread_no; diff --git a/lzma/C/Alloc.c b/lzma/C/Alloc.c index 152b623..358a7b5 100644 --- a/lzma/C/Alloc.c +++ b/lzma/C/Alloc.c @@ -99,7 +99,7 @@ void *BigAlloc(size_t size) #ifdef _SZ_ALLOC_DEBUG fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); #endif - + #ifdef _7ZIP_LARGE_PAGES if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) { @@ -118,7 +118,7 @@ void BigFree(void *address) if (address != 0) fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); #endif - + if (address == 0) return; VirtualFree(address, 0, MEM_RELEASE); diff --git a/lzma/C/Alloc.h b/lzma/C/Alloc.h index 78eb1f8..b8e4143 100644 --- a/lzma/C/Alloc.h +++ b/lzma/C/Alloc.h @@ -1,18 +1,19 @@ /* Alloc.h -- Memory allocation functions -2008-03-13 -Igor Pavlov -Public domain */ +2009-02-07 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H #include -#ifdef _WIN32 +#ifdef __cplusplus +extern "C" { +#endif void *MyAlloc(size_t size); void MyFree(void *address); +#ifdef _WIN32 void SetLargePageSize(); @@ -23,15 +24,15 @@ void BigFree(void *address); #else -#include /* malloc */ - -#define MyAlloc(size) malloc(size) -#define MyFree(address) free(address) -#define MidAlloc(size) malloc(size) -#define MidFree(address) free(address) -#define BigAlloc(size) malloc(size) -#define BigFree(address) free(address) +#define MidAlloc(size) MyAlloc(size) +#define MidFree(address) MyFree(address) +#define BigAlloc(size) MyAlloc(size) +#define BigFree(address) MyFree(address) #endif +#ifdef __cplusplus +} +#endif + #endif diff --git a/lzma/C/LzFind.c b/lzma/C/LzFind.c index e753d04..e3ecb05 100644 --- a/lzma/C/LzFind.c +++ b/lzma/C/LzFind.c @@ -512,7 +512,7 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) delta2 = p->pos - p->hash[hash2Value]; curMatch = p->hash[kFix3HashSize + hashValue]; - + p->hash[hash2Value] = p->hash[kFix3HashSize + hashValue] = p->pos; @@ -546,7 +546,7 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) delta2 = p->pos - p->hash[ hash2Value]; delta3 = p->pos - p->hash[kFix3HashSize + hash3Value]; curMatch = p->hash[kFix4HashSize + hashValue]; - + p->hash[ hash2Value] = p->hash[kFix3HashSize + hash3Value] = p->hash[kFix4HashSize + hashValue] = p->pos; diff --git a/lzma/C/LzFindMt.c b/lzma/C/LzFindMt.c index 3fcfb88..aa41ed9 100644 --- a/lzma/C/LzFindMt.c +++ b/lzma/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2009-05-26 : Igor Pavlov : Public domain */ +2009-09-20 : Igor Pavlov : Public domain */ #include "LzHash.h" @@ -58,7 +58,7 @@ void MtSync_StopWriting(CMtSync *p) p->csWasEntered = False; } Semaphore_Release1(&p->freeSemaphore); - + Event_Wait(&p->wasStopped); while (myNumBlocks++ != p->numProcessedBlocks) @@ -108,12 +108,12 @@ static SRes MtSync_Create2(CMtSync *p, unsigned (MY_STD_CALL *startAddress)(void RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)); RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStarted)); RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)); - + RINOK_THREAD(Semaphore_Create(&p->freeSemaphore, numBlocks, numBlocks)); RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks)); p->needStart = True; - + RINOK_THREAD(Thread_Create(&p->thread, startAddress, obj)); p->wasCreated = True; return SZ_OK; @@ -385,7 +385,7 @@ void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) CriticalSection_Enter(&sync->cs); sync->csWasEntered = True; } - + BtGetMatches(p, p->btBuf + (globalBlockIndex & kMtBtNumBlocksMask) * kMtBtBlockSize); if (p->pos > kMtMaxValForNormalize - kMtBtBlockSize) @@ -561,7 +561,7 @@ UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) const Byte *cur = p->pointerToCurPos; UInt32 lzPos = p->lzPos; MT_HASH2_CALC - + curMatch2 = hash[hash2Value]; hash[hash2Value] = lzPos; @@ -584,7 +584,7 @@ UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) curMatch2 = hash[ hash2Value]; curMatch3 = hash[kFix3HashSize + hash3Value]; - + hash[ hash2Value] = hash[kFix3HashSize + hash3Value] = lzPos; @@ -616,11 +616,11 @@ UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) const Byte *cur = p->pointerToCurPos; UInt32 lzPos = p->lzPos; MT_HASH4_CALC - + curMatch2 = hash[ hash2Value]; curMatch3 = hash[kFix3HashSize + hash3Value]; curMatch4 = hash[kFix4HashSize + hash4Value]; - + hash[ hash2Value] = hash[kFix3HashSize + hash3Value] = hash[kFix4HashSize + hash4Value] = @@ -711,47 +711,47 @@ UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) return len; } -#define SKIP_HEADER2 do { GET_NEXT_BLOCK_IF_REQUIRED -#define SKIP_HEADER(n) SKIP_HEADER2 if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; -#define SKIP_FOOTER } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); +#define SKIP_HEADER2_MT do { GET_NEXT_BLOCK_IF_REQUIRED +#define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; +#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) { - SKIP_HEADER2 { p->btNumAvailBytes--; - SKIP_FOOTER + SKIP_HEADER2_MT { p->btNumAvailBytes--; + SKIP_FOOTER_MT } void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) { - SKIP_HEADER(2) + SKIP_HEADER_MT(2) UInt32 hash2Value; MT_HASH2_CALC hash[hash2Value] = p->lzPos; - SKIP_FOOTER + SKIP_FOOTER_MT } void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) { - SKIP_HEADER(3) + SKIP_HEADER_MT(3) UInt32 hash2Value, hash3Value; MT_HASH3_CALC hash[kFix3HashSize + hash3Value] = hash[ hash2Value] = p->lzPos; - SKIP_FOOTER + SKIP_FOOTER_MT } /* void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) { - SKIP_HEADER(4) + SKIP_HEADER_MT(4) UInt32 hash2Value, hash3Value, hash4Value; MT_HASH4_CALC hash[kFix4HashSize + hash4Value] = hash[kFix3HashSize + hash3Value] = hash[ hash2Value] = p->lzPos; - SKIP_FOOTER + SKIP_FOOTER_MT } */ diff --git a/lzma/C/LzFindMt.h b/lzma/C/LzFindMt.h index 63133b1..b985af5 100644 --- a/lzma/C/LzFindMt.h +++ b/lzma/C/LzFindMt.h @@ -62,7 +62,7 @@ typedef struct _CMatchFinderMt const UInt32 *crc; Mf_Mix_Matches MixMatchesFunc; - + /* LZ + BT */ CMtSync btSync; Byte btDummy[kMtCacheLineDummy]; @@ -85,7 +85,7 @@ typedef struct _CMatchFinderMt /* BT + Hash */ CMtSync hashSync; /* Byte hashDummy[kMtCacheLineDummy]; */ - + /* Hash */ Mf_GetHeads GetHeadsFunc; CMatchFinder *MatchFinder; diff --git a/lzma/C/LzmaDec.c b/lzma/C/LzmaDec.c index 89d934a..2036761 100644 --- a/lzma/C/LzmaDec.c +++ b/lzma/C/LzmaDec.c @@ -1,5 +1,5 @@ /* LzmaDec.c -- LZMA Decoder -2008-11-06 : Igor Pavlov : Public domain */ +2009-09-20 : Igor Pavlov : Public domain */ #include "LzmaDec.h" @@ -113,12 +113,6 @@ StopCompilingDueBUG #endif -static const Byte kLiteralNextStates[kNumStates * 2] = -{ - 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5, - 7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10 -}; - #define LZMA_DIC_MIN (1 << 12) /* First LZMA-symbol is always decoded. @@ -147,7 +141,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; SizeT dicPos = p->dicPos; - + UInt32 processedPos = p->processedPos; UInt32 checkDicSize = p->checkDicSize; unsigned len = 0; @@ -175,6 +169,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (state < kNumLitStates) { + state -= (state < 4) ? state : 3; symbol = 1; do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100); } @@ -182,6 +177,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte { unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; unsigned offs = 0x100; + state -= (state < 10) ? 3 : 6; symbol = 1; do { @@ -196,9 +192,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } dic[dicPos++] = (Byte)symbol; processedPos++; - - state = kLiteralNextStates[state]; - /* if (state < 4) state = 0; else if (state < 10) state -= 3; else state -= 6; */ continue; } else @@ -331,7 +324,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte { NORMALIZE range >>= 1; - + { UInt32 t; code -= range; @@ -378,7 +371,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else if (distance >= checkDicSize) return SZ_ERROR_DATA; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; - /* state = kLiteralNextStates[state]; */ } len += kMatchMinLen; @@ -730,7 +722,7 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr SizeT inSize = *srcLen; (*srcLen) = 0; LzmaDec_WriteRem(p, dicLimit); - + *status = LZMA_STATUS_NOT_SPECIFIED; while (p->remainLen != kMatchSpecLenStart) @@ -776,7 +768,7 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr if (p->needInitState) LzmaDec_InitStateReal(p); - + if (p->tempBufSize == 0) { SizeT processed; @@ -907,12 +899,12 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) { UInt32 dicSize; Byte d; - + if (size < LZMA_PROPS_SIZE) return SZ_ERROR_UNSUPPORTED; else dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); - + if (dicSize < LZMA_DIC_MIN) dicSize = LZMA_DIC_MIN; p->dicSize = dicSize; @@ -994,7 +986,7 @@ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, p.dicBufSize = outSize; LzmaDec_Init(&p); - + *srcLen = inSize; res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); diff --git a/lzma/C/LzmaDec.h b/lzma/C/LzmaDec.h index 7927acd..bf7f084 100644 --- a/lzma/C/LzmaDec.h +++ b/lzma/C/LzmaDec.h @@ -130,7 +130,7 @@ LzmaDec_Allocate* can return: SZ_ERROR_MEM - Memory allocation error SZ_ERROR_UNSUPPORTED - Unsupported properties */ - + SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); @@ -159,7 +159,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); */ /* LzmaDec_DecodeToDic - + The decoding to internal dictionary buffer (CLzmaDec::dic). You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! diff --git a/lzma/C/LzmaEnc.c b/lzma/C/LzmaEnc.c index 4ddd3dc..cf13138 100644 --- a/lzma/C/LzmaEnc.c +++ b/lzma/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2009-04-22 : Igor Pavlov : Public domain */ +2010-04-16 : Igor Pavlov : Public domain */ #include @@ -13,7 +13,7 @@ #include "LzmaEnc.h" #include "LzFind.h" -#ifdef COMPRESS_MF_MT +#ifndef _7ZIP_ST #include "LzFindMt.h" #endif @@ -66,7 +66,7 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); if (p->numThreads < 0) p->numThreads = - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST ((p->btMode && p->algo) ? 2 : 1); #else 1; @@ -109,7 +109,7 @@ void LzmaEnc_FastPosInit(Byte *g_FastPos) int c = 2, slotFast; g_FastPos[0] = 0; g_FastPos[1] = 1; - + for (slotFast = 2; slotFast < kNumLogBits * 2; slotFast++) { UInt32 k = (1 << ((slotFast >> 1) - 1)); @@ -172,7 +172,7 @@ typedef struct #define kEndPosModelIndex 14 #define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) -#define kNumFullDistances (1 << (kEndPosModelIndex / 2)) +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) #ifdef _LZMA_PROB32 #define CLzmaProb UInt32 @@ -246,7 +246,7 @@ typedef struct CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; @@ -259,17 +259,17 @@ typedef struct IMatchFinder matchFinder; void *matchFinderObj; - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST Bool mtMode; CMatchFinderMt matchFinderMt; #endif CMatchFinder matchFinderBase; - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST Byte pad[128]; #endif - + UInt32 optimumEndIndex; UInt32 optimumCurrentIndex; @@ -277,7 +277,7 @@ typedef struct UInt32 numPairs; UInt32 numAvail; COptimal opt[kNumOpts]; - + #ifndef LZMA_LOG_BSR Byte g_FastPos[1 << kNumLogBits]; #endif @@ -311,14 +311,14 @@ typedef struct CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; unsigned lclp; Bool fastMode; - + CRangeEnc rc; Bool writeEndMark; @@ -395,7 +395,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) LzmaEncProps_Normalize(&props); if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || - props.dictSize > (unsigned)(1 << kDicLogSizeMaxCompress) || props.dictSize > (1 << 30)) + props.dictSize > ((UInt32)1 << kDicLogSizeMaxCompress) || props.dictSize > ((UInt32)1 << 30)) return SZ_ERROR_PARAM; p->dictSize = props.dictSize; p->matchFinderCycles = props.mc; @@ -428,7 +428,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->writeEndMark = props.writeEndMark; - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST /* if (newMultiThread != _multiThread) { @@ -804,7 +804,7 @@ static void MovePos(CLzmaEnc *p, UInt32 num) { #ifdef SHOW_STAT ttt += num; - fprintf(stderr, "\n MovePos %d", num); + printf("\n MovePos %d", num); #endif if (num != 0) { @@ -819,12 +819,12 @@ static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); #ifdef SHOW_STAT - fprintf(stderr, "\n i = %d numPairs = %d ", ttt, numPairs / 2); + printf("\n i = %d numPairs = %d ", ttt, numPairs / 2); ttt++; { UInt32 i; for (i = 0; i < numPairs; i += 2) - fprintf(stderr, "%2d %6d | ", p->matches[i], p->matches[i + 1]); + printf("%2d %6d | ", p->matches[i], p->matches[i + 1]); } #endif if (numPairs > 0) @@ -909,10 +909,10 @@ static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) { UInt32 posPrev = posMem; UInt32 backCur = backMem; - + backMem = p->opt[posPrev].backPrev; posMem = p->opt[posPrev].posPrev; - + p->opt[posPrev].backPrev = backCur; p->opt[posPrev].posPrev = cur; cur = posPrev; @@ -943,7 +943,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) return lenRes; } p->optimumCurrentIndex = p->optimumEndIndex = 0; - + if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else @@ -1115,9 +1115,9 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) if (position >= 0) { unsigned i; - fprintf(stderr, "\n pos = %4X", position); + printf("\n pos = %4X", position); for (i = cur; i <= lenEnd; i++) - fprintf(stderr, "\nprice[%4X] = %d", position - cur + i, p->opt[i].price); + printf("\nprice[%4X] = %d", position - cur + i, p->opt[i].price); } #endif @@ -1241,7 +1241,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - + if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) { UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); @@ -1303,7 +1303,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } } } - + startLen = 2; /* speed optimization */ { UInt32 repIndex; @@ -1334,10 +1334,10 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } while (--lenTest >= 2); lenTest = lenTestTemp; - + if (repIndex == 0) startLen = lenTest + 1; - + /* if (_maxMode) */ { UInt32 lenTest2 = lenTest + 1; @@ -1361,7 +1361,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) nextRepMatchPrice = curAndLenCharPrice + GET_PRICE_1(p->isMatch[state2][posStateNext]) + GET_PRICE_1(p->isRep[state2]); - + /* for (; lenTest2 >= 2; lenTest2--) */ { UInt32 curAndLenPrice; @@ -1416,7 +1416,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; else curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - + opt = &p->opt[cur + lenTest]; if (curAndLenPrice < opt->price) { @@ -1450,7 +1450,7 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) nextRepMatchPrice = curAndLenCharPrice + GET_PRICE_1(p->isMatch[state2][posStateNext]) + GET_PRICE_1(p->isRep[state2]); - + /* for (; lenTest2 >= 2; lenTest2--) */ { UInt32 offset = cur + lenTest + 1 + lenTest2; @@ -1562,7 +1562,7 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) MovePos(p, repLen - 1); return repLen; } - + if (mainLen < 2 || numAvail <= 2) return 1; @@ -1576,7 +1576,7 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) return 1; } - + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; for (i = 0; i < LZMA_NUM_REPS; i++) { @@ -1677,7 +1677,7 @@ void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); MatchFinder_Construct(&p->matchFinderBase); - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST MatchFinderMt_Construct(&p->matchFinderMt); p->matchFinderMt.MatchFinder = &p->matchFinderBase; #endif @@ -1716,7 +1716,7 @@ void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) { - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); #endif MatchFinder_Free(&p->matchFinderBase, allocBig); @@ -1772,7 +1772,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize len = GetOptimum(p, nowPos32, &pos); #ifdef SHOW_STAT2 - fprintf(stderr, "\n pos = %4X, len = %d pos = %d", nowPos32, len, pos); + printf("\n pos = %4X, len = %d pos = %d", nowPos32, len, pos); #endif posState = nowPos32 & p->pbMask; @@ -1837,7 +1837,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize pos -= LZMA_NUM_REPS; GetPosSlot(pos, posSlot); RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); - + if (posSlot >= kStartPosModelIndex) { UInt32 footerBits = ((posSlot >> 1) - 1); @@ -1901,7 +1901,7 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I if (!RangeEnc_Alloc(&p->rc, alloc)) return SZ_ERROR_MEM; btMode = (p->matchFinderBase.btMode != 0); - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST p->mtMode = (p->multiThread && !p->fastMode && btMode); #endif @@ -1926,7 +1926,7 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I if (beforeSize + p->dictSize < keepWindowSize) beforeSize = keepWindowSize - p->dictSize; - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST if (p->mtMode) { RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); @@ -2073,7 +2073,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, void LzmaEnc_Finish(CLzmaEncHandle pp) { - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST CLzmaEnc *p = (CLzmaEnc *)pp; if (p->mtMode) MatchFinderMt_ReleaseStream(&p->matchFinderMt); @@ -2142,7 +2142,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, p->rc.outStream = &outStream.funcTable; res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); - + *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem; if (outStream.overflow) @@ -2155,7 +2155,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) { SRes res = SZ_OK; - #ifdef COMPRESS_MF_MT + #ifndef _7ZIP_ST Byte allocaDummy[0x300]; int i = 0; for (i = 0; i < 16; i++) diff --git a/lzma/C/LzmaLib.h b/lzma/C/LzmaLib.h index 2a3efe1..76c99ce 100644 --- a/lzma/C/LzmaLib.h +++ b/lzma/C/LzmaLib.h @@ -38,7 +38,7 @@ outPropsSize - LZMA Encoder will use defult values for any parameter, if it is -1 for any from: level, loc, lp, pb, fb, numThreads 0 for dictSize - + level - compression level: 0 <= level <= 9; level dictSize algo fb @@ -50,7 +50,7 @@ level - compression level: 0 <= level <= 9; 5: 16 MB 1 32 6: 32 MB 1 32 7+: 64 MB 1 64 - + The default value for "level" is 5. algo = 0 means fast method diff --git a/lzma/C/Makefile.am b/lzma/C/Makefile.am index a059ba2..03ef6ce 100644 --- a/lzma/C/Makefile.am +++ b/lzma/C/Makefile.am @@ -11,25 +11,26 @@ endif noinst_LTLIBRARIES = liblzma.la liblzma_la_SOURCES = \ $(ASM_S) \ - Alloc.h \ - basetyps.h \ - LzFind.h \ - LzFindMt.h \ - LzHash.h \ - LzmaDec.h \ - LzmaEnc.h \ - LzmaLib.h \ - MyGuidDef.h \ - MyWindows.h \ - Threads.h \ - Types.h \ - windows.h \ - Threads.c \ - LzFind.c \ - LzFindMt.c \ - LzmaDec.c \ - LzmaEnc.c \ - LzmaLib.c + LzmaDec.h \ + LzmaEnc.h \ + LzFind.c \ + LzFind.h \ + LzFindMt.c \ + LzFindMt.h \ + LzmaDec.c \ + LzmaEnc.c \ + LzmaLib.c \ + LzmaLib.h \ + Alloc.c \ + Alloc.h \ + Threads.c \ + Threads.h \ + Types.h \ + LzHash.h \ + windows.h \ + basetyps.h \ + MyWindows.h \ + MyGuidDef.h liblzma_so_LIBS = $(ASM_LIBS) liblzma_so_CFLAGS = \ diff --git a/lzma/C/Types.h b/lzma/C/Types.h index b36c5c7..b61f534 100644 --- a/lzma/C/Types.h +++ b/lzma/C/Types.h @@ -46,6 +46,7 @@ typedef int SRes; typedef DWORD WRes; #else typedef int WRes; +typedef void * HANDLE; #endif #ifndef RINOK diff --git a/lzma/Methods.txt b/lzma/Methods.txt index 0503ca7..f52e7c3 100644 --- a/lzma/Methods.txt +++ b/lzma/Methods.txt @@ -1,4 +1,4 @@ -7-Zip method IDs (4.61) +7-Zip method IDs (9.18) ----------------------- Each compression or crypto method in 7z has unique binary value (ID). @@ -24,14 +24,22 @@ List of defined IDs ------------------- 00 - Copy -01 - Reserved -02 - Common + +03 - Delta +04 - x86 (BCJ) +05 - PPC (Big Endian) +06 - IA64 +07 - ARM (little endian) +08 - ARM Thumb (little endian) +09 - SPARC +21 - LZMA2 + +02.. - Common 03 Swap - 2 Swap2 - 4 Swap4 - 04 Delta (subject to change) -03 - 7z +03.. - 7z 01 - LZMA 01 - Version @@ -60,11 +68,8 @@ List of defined IDs 7F - 01 - experimental methods. - 80 - reserved for independent developers - E0 - Random IDs - -04 - Misc +04.. - Misc 00 - Reserved 01 - Zip 00 - Copy (not used). Use {00} instead @@ -72,7 +77,13 @@ List of defined IDs 06 - Implode 08 - Deflate 09 - Deflate64 + 10 - Imploding 12 - BZip2 (not used). Use {04 02 02} instead + 14 - LZMA + 60 - Jpeg + 61 - WavPack + 62 - PPMd + 63 - wzAES 02 - BZip 02 - BZip2 03 - Rar @@ -91,7 +102,7 @@ List of defined IDs 02 - BZip2NSIS -06 - Crypto +06.. - Crypto 00 - 01 - AES 0x - AES-128 @@ -118,7 +129,7 @@ List of defined IDs 07 - 7z 01 - AES-256 + SHA-256 -07 - Hash (subject to change) +07.. - Hash (subject to change) 00 - 01 - CRC 02 - SHA-1 diff --git a/lzma/history.txt b/lzma/history.txt index c807f32..79abd9c 100644 --- a/lzma/history.txt +++ b/lzma/history.txt @@ -1,6 +1,46 @@ HISTORY of the LZMA SDK ----------------------- +9.18 beta 2010-11-02 +------------------------- +- New small SFX module for installers (SfxSetup). + + +9.12 beta 2010-03-24 +------------------------- +- The BUG in LZMA SDK 9.* was fixed: LZMA2 codec didn't work, + if more than 10 threads were used (or more than 20 threads in some modes). + + +9.11 beta 2010-03-15 +------------------------- +- PPMd compression method support + + +9.09 2009-12-12 +------------------------- +- The bug was fixed: + Utf16_To_Utf8 funstions in UTFConvert.cpp and 7zMain.c + incorrectly converted surrogate characters (the code >= 0x10000) to UTF-8. +- Some bugs were fixed + + +9.06 2009-08-17 +------------------------- +- Some changes in ANSI-C 7z Decoder interfaces. + + +9.04 2009-05-30 +------------------------- +- LZMA2 compression method support +- xz format support + + +4.65 2009-02-03 +------------------------- +- Some minor fixes + + 4.63 2008-12-31 ------------------------- - Some minor fixes diff --git a/lzma/lzma.txt b/lzma/lzma.txt index d3cbc85..6593232 100644 --- a/lzma/lzma.txt +++ b/lzma/lzma.txt @@ -1,4 +1,4 @@ -LZMA SDK 4.63 +LZMA SDK 9.20 ------------- LZMA SDK provides the documentation, samples, header files, libraries, @@ -20,6 +20,10 @@ LICENSE LZMA SDK is written and placed in the public domain by Igor Pavlov. +Some code in LZMA SDK is based on public domain code from another developers: + 1) PPMd var.H (2001): Dmitry Shkarin + 2) SHA-256: Wei Dai (Crypto++ library) + LZMA SDK Contents ----------------- @@ -33,7 +37,7 @@ LZMA SDK includes: UNIX/Linux version ------------------ To compile C++ version of file->file LZMA encoding, go to directory -C++/7zip/Compress/LZMA_Alone +CPP/7zip/Bundles/LzmaCon and call make to recompile it: make -f makefile.gcc clean all @@ -49,6 +53,7 @@ lzma.txt - LZMA SDK description (this file) 7zC.txt - 7z ANSI-C Decoder description methods.txt - Compression method IDs for .7z lzma.exe - Compiled file->file LZMA encoder/decoder for Windows +7zr.exe - 7-Zip with 7z/lzma/xz support. history.txt - history of the LZMA SDK @@ -66,7 +71,7 @@ C/ - C files LzmaEnc.* - LZMA encoding LzmaLib.* - LZMA Library for DLL calling Types.h - Basic types for another .c files - Threads.* - The code for multithreading. + Threads.* - The code for multithreading. LzmaLib - LZMA Library (.DLL for Windows) @@ -86,12 +91,6 @@ CPP/ -- CPP files Compress - files related to compression/decompression - Copy - Copy coder - RangeCoder - Range Coder (special code of compression/decompression) - LZMA - LZMA compression/decompression on C++ - LZMA_Alone - file->file LZMA compression/decompression - Branch - Filters for x86, IA-64, ARM, ARM-Thumb, PowerPC and SPARC code - Archive - files related to archiving Common - common files for archive handling @@ -100,6 +99,7 @@ CPP/ -- CPP files Bundles - Modules that are bundles of other modules Alone7z - 7zr.exe: Standalone version of 7z.exe that supports only 7z/LZMA/BCJ/BCJ2 + LzmaCon - lzma.exe: LZMA compression/decompression Format7zR - 7zr.dll: Reduced version of 7za.dll: extracting/compressing to 7z/LZMA/BCJ/BCJ2 Format7zExtractR - 7zxr.dll: Reduced version of 7zxa.dll: extracting from 7z/LZMA/BCJ/BCJ2. @@ -369,8 +369,8 @@ Interface: propData - LZMA properties (5 bytes) propSize - size of propData buffer (5 bytes) finishMode - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - Decode just destLen bytes. - LZMA_FINISH_END - Stream must be finished after (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). You can use LZMA_FINISH_END, when you know that current output buffer covers last bytes of stream. alloc - Memory allocator. @@ -431,7 +431,7 @@ Memory Requirements: { ... int res = LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode); + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode); ... } @@ -527,7 +527,8 @@ static ISzAlloc g_Alloc = { SzAlloc, SzFree }; LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); -If callback function return some error code, LzmaEnc_Encode also returns that code. +If callback function return some error code, LzmaEnc_Encode also returns that code +or it can return the code like SZ_ERROR_READ, SZ_ERROR_WRITE or SZ_ERROR_PROGRESS. Single-call RAM->RAM Compression @@ -549,8 +550,8 @@ Return code: -LZMA Defines ------------- +Defines +------- _LZMA_SIZE_OPT - Enable some optimizations in LZMA Decoder to get smaller executable code. @@ -562,6 +563,9 @@ _LZMA_UINT32_IS_ULONG - Define it if int is 16-bit on your compiler and long is _LZMA_NO_SYSTEM_SIZE_T - Define it if you don't want to use size_t type. +_7ZIP_PPMD_SUPPPORT - Define it if you don't want to support PPMD method in AMSI-C .7z decoder. + + C++ LZMA Encoder/Decoder ~~~~~~~~~~~~~~~~~~~~~~~~ C++ LZMA code use COM-like interfaces. So if you want to use it, diff --git a/runzip.c b/runzip.c index 0a2ee82..1c2469d 100644 --- a/runzip.c +++ b/runzip.c @@ -285,6 +285,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 else if (control->major_version == 0 && control->minor_version == 4) chunk_bytes = 8; else { + print_maxverbose("Reading chunk_bytes at %lld\n", get_readseek(control, fd_in)); /* Read in the stored chunk byte width from the file */ if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1)) fatal_return(("Failed to read chunk_bytes size in runzip_chunk\n"), -1); @@ -352,8 +353,8 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 print_maxverbose("Checksum for block: 0x%08x\n", cksum); } - if (unlikely(close_stream_in(ss))) - fatal_return(("Failed to close stream!\n"), -1); + if (unlikely(close_stream_in(control, ss))) + fatal("Failed to close stream!\n"); return total; } diff --git a/stream.c b/stream.c index 58302e4..a2df99a 100644 --- a/stream.c +++ b/stream.c @@ -921,7 +921,7 @@ static i64 get_seek(rzip_control *control, int fd) return ret; } -static i64 get_readseek(rzip_control *control, int fd) +i64 get_readseek(rzip_control *control, int fd) { i64 ret; @@ -1159,12 +1159,16 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) if (control->major_version == 0 && control->minor_version > 5) { /* Read in flag that tells us if there are more chunks after * this. Ignored if we know the final file size */ + print_maxverbose("Reading eof flag at %lld\n", get_readseek(control, f)); if (unlikely(read_u8(control, f, &control->eof))) { print_err("Failed to read eof flag in open_stream_in\n"); goto failed; } + print_maxverbose("EOF: %d\n", control->eof); + /* Read in the expected chunk size */ if (!ENCRYPT) { + print_maxverbose("Reading expected chunksize at %lld\n", get_readseek(control, f)); if (unlikely(read_val(control, f, &sinfo->size, sinfo->chunk_bytes))) { print_err("Failed to read in chunk size in open_stream_in\n"); goto failed; @@ -1209,6 +1213,7 @@ again: } else { int read_len; + print_maxverbose("Reading stream %d header at %lld\n", i, get_readseek(control, f)); if ((control->major_version == 0 && control->minor_version < 6) || ENCRYPT) read_len = 8; @@ -1222,6 +1227,8 @@ again: goto failed; header_length = 1 + (read_len * 3); } + sinfo->total_read += header_length; + if (ENCRYPT) if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; @@ -1388,11 +1395,14 @@ retry: if (unlikely(!flush_tmpoutbuf(control))) goto error; } + print_maxverbose("Writing initial chunk bytes value %d at %lld\n", + ctis->chunk_bytes, get_seek(control, ctis->fd)); /* Write chunk bytes of this block */ write_u8(control, ctis->chunk_bytes); /* Write whether this is the last chunk, followed by the size * of this chunk */ + print_maxverbose("Writing EOF flag as %d\n", control->eof); write_u8(control, control->eof); if (!ENCRYPT) write_val(control, ctis->size, ctis->chunk_bytes); @@ -1401,6 +1411,7 @@ retry: ctis->initial_pos = get_seek(control, ctis->fd); if (unlikely(ctis->initial_pos == -1)) goto error; + print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos); for (j = 0; j < ctis->num_streams; j++) { /* If encrypting, we leave SALT_LEN room to write in salt * later */ @@ -1418,6 +1429,8 @@ retry: } } + print_maxverbose("Compthread %ld seeking to %lld to store length %d\n", i, ctis->s[cti->streamno].last_head, write_len); + if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) fatal_goto(("Failed to seekto in compthread %d\n", i), error); @@ -1428,6 +1441,9 @@ retry: rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17); ctis->s[cti->streamno].last_head = ctis->cur_pos + 1 + (write_len * 2) + (ENCRYPT ? SALT_LEN : 0); + + print_maxverbose("Compthread %ld seeking to %lld to write header\n", i, ctis->cur_pos); + if (unlikely(seekto(control, ctis, ctis->cur_pos))) fatal_goto(("Failed to seekto cur_pos in compthread %d\n", i), error); @@ -1455,6 +1471,9 @@ retry: if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; ctis->cur_pos += SALT_LEN; } + + print_maxverbose("Compthread %ld writing data at %lld\n", i, ctis->cur_pos); + if (unlikely(write_buf(control, cti->s_buf, padded_len))) fatal_goto(("Failed to write_buf s_buf in compthread %d\n", i), error); @@ -1588,7 +1607,7 @@ retry: /* fill a buffer from a stream - return -1 on failure */ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) { - i64 u_len, c_len, last_head, padded_len; + i64 u_len, c_len, last_head, padded_len, header_length; uchar enc_head[25 + SALT_LEN], blocksalt[SALT_LEN]; struct stream *s = &sinfo->s[streamno]; stream_thread_struct *st; @@ -1624,9 +1643,11 @@ fill_another: c_len = c_len32; u_len = u_len32; last_head = last_head32; + header_length = 13; } else { int read_len; + print_maxverbose("Reading ucomp header at %lld\n", get_readseek(control, sinfo->fd)); if ((control->major_version == 0 && control->minor_version < 6) || ENCRYPT) read_len = 8; else @@ -1637,7 +1658,9 @@ fill_another: return -1; if (unlikely(read_val(control, sinfo->fd, &last_head, read_len))) return -1; + header_length = 1 + (read_len * 3); } + sinfo->total_read += header_length; if (ENCRYPT) { if (unlikely(!decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head))) return -1; @@ -1649,6 +1672,7 @@ fill_another: last_head = le64toh(last_head); padded_len = MAX(c_len, MIN_SIZE); + sinfo->total_read += padded_len; fsync(control->fd_out); s_buf = malloc(MAX(u_len, MIN_SIZE)); @@ -1826,11 +1850,17 @@ int close_stream_out(rzip_control *control, void *ss) } /* close down an input stream */ -int close_stream_in(void *ss) +int close_stream_in(rzip_control *control, void *ss) { struct stream_info *sinfo = ss; int i; + print_maxverbose("Closing stream at %lld, want to seek to %lld\n", + get_readseek(control, control->fd_in), + sinfo->initial_pos + sinfo->total_read); + if (unlikely(read_seekto(control, sinfo, sinfo->total_read))) + return -1; + for (i = 0; i < sinfo->num_streams; i++) free(sinfo->s[i].buf); diff --git a/stream.h b/stream.h index 6967702..bb089d2 100644 --- a/stream.h +++ b/stream.h @@ -28,15 +28,16 @@ bool create_pthread(pthread_t *thread, pthread_attr_t *attr, bool join_pthread(pthread_t th, void **thread_return); ssize_t write_1g(rzip_control *control, void *buf, i64 len); ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len); -bool prepare_streamout_threads(rzip_control *control); -bool close_streamout_threads(rzip_control *control); +i64 get_readseek(rzip_control *control, int fd); +void prepare_streamout_threads(rzip_control *control); +void close_streamout_threads(rzip_control *control); void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes); void *open_stream_in(rzip_control *control, int f, int n, char cbytes); bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); int close_stream_out(rzip_control *control, void *ss); -int close_stream_in(void *ss); +int close_stream_in(rzip_control *control, void *ss); ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret); #endif