ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into liblrzip

Browse source 
Conflicts:
	runzip.c
	stream.h
This commit is contained in:
Con Kolivas 2011-09-15 08:31:00 +10:00
parent b1c5d5504a e1f2138060
commit a0fe1a8c8e
19 changed files with 237 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README
View file

@ -244,7 +244,7 @@ A. It has been tested on gcc, ekopath and the intel compiler successfully
previously. Whether the commercial compilers help or not, I could not tell you. previously. Whether the commercial compilers help or not, I could not tell you.
Q. What codebase are you basing this on? Q. What codebase are you basing this on?
A. rzip v2.1 and lzma sdk907, but it should be possible to stay in sync with A. rzip v2.1 and lzma sdk920, but it should be possible to stay in sync with
each of these in the future. each of these in the future.
Q. Do we really need yet another compression format? Q. Do we really need yet another compression format?

1
lrzip_private.h
View file

@ -371,6 +371,7 @@ struct stream_info {
i64 bufsize; i64 bufsize;
i64 cur_pos; i64 cur_pos;
i64 initial_pos; i64 initial_pos;
i64 total_read;
i64 ram_alloced; i64 ram_alloced;
i64 size; i64 size;
long thread_no; long thread_no;

25
lzma/C/Alloc.h
View file

@ -1,18 +1,19 @@
/* Alloc.h -- Memory allocation functions /* Alloc.h -- Memory allocation functions
2008-03-13 2009-02-07 : Igor Pavlov : Public domain */
Igor Pavlov
Public domain */
#ifndef __COMMON_ALLOC_H #ifndef __COMMON_ALLOC_H
#define __COMMON_ALLOC_H #define __COMMON_ALLOC_H
#include <stddef.h> #include <stddef.h>
#ifdef _WIN32 #ifdef __cplusplus
extern "C" {
#endif
void *MyAlloc(size_t size); void *MyAlloc(size_t size);
void MyFree(void *address); void MyFree(void *address);
#ifdef _WIN32
void SetLargePageSize(); void SetLargePageSize();
@ -23,15 +24,15 @@ void BigFree(void *address);
#else #else
#include <stdlib.h> /* malloc */ #define MidAlloc(size) MyAlloc(size)
#define MidFree(address) MyFree(address)
#define MyAlloc(size) malloc(size) #define BigAlloc(size) MyAlloc(size)
#define MyFree(address) free(address) #define BigFree(address) MyFree(address)
#define MidAlloc(size) malloc(size)
#define MidFree(address) free(address)
#define BigAlloc(size) malloc(size)
#define BigFree(address) free(address)
#endif #endif
#ifdef __cplusplus
}
#endif
#endif #endif

24
lzma/C/LzFindMt.c
View file

@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms /* LzFindMt.c -- multithreaded Match finder for LZ algorithms
2009-05-26 : Igor Pavlov : Public domain */ 2009-09-20 : Igor Pavlov : Public domain */
#include "LzHash.h" #include "LzHash.h"
@ -711,47 +711,47 @@ UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances)
return len; return len;
} }
#define SKIP_HEADER2 do { GET_NEXT_BLOCK_IF_REQUIRED #define SKIP_HEADER2_MT do { GET_NEXT_BLOCK_IF_REQUIRED
#define SKIP_HEADER(n) SKIP_HEADER2 if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; #define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash;
#define SKIP_FOOTER } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); #define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0);
void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num)
{ {
SKIP_HEADER2 { p->btNumAvailBytes--; SKIP_HEADER2_MT { p->btNumAvailBytes--;
SKIP_FOOTER SKIP_FOOTER_MT
} }
void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num)
{ {
SKIP_HEADER(2) SKIP_HEADER_MT(2)
UInt32 hash2Value; UInt32 hash2Value;
MT_HASH2_CALC MT_HASH2_CALC
hash[hash2Value] = p->lzPos; hash[hash2Value] = p->lzPos;
SKIP_FOOTER SKIP_FOOTER_MT
} }
void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num)
{ {
SKIP_HEADER(3) SKIP_HEADER_MT(3)
UInt32 hash2Value, hash3Value; UInt32 hash2Value, hash3Value;
MT_HASH3_CALC MT_HASH3_CALC
hash[kFix3HashSize + hash3Value] = hash[kFix3HashSize + hash3Value] =
hash[ hash2Value] = hash[ hash2Value] =
p->lzPos; p->lzPos;
SKIP_FOOTER SKIP_FOOTER_MT
} }
/* /*
void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num)
{ {
SKIP_HEADER(4) SKIP_HEADER_MT(4)
UInt32 hash2Value, hash3Value, hash4Value; UInt32 hash2Value, hash3Value, hash4Value;
MT_HASH4_CALC MT_HASH4_CALC
hash[kFix4HashSize + hash4Value] = hash[kFix4HashSize + hash4Value] =
hash[kFix3HashSize + hash3Value] = hash[kFix3HashSize + hash3Value] =
hash[ hash2Value] = hash[ hash2Value] =
p->lzPos; p->lzPos;
SKIP_FOOTER SKIP_FOOTER_MT
} }
*/ */

14
lzma/C/LzmaDec.c
View file

@ -1,5 +1,5 @@
/* LzmaDec.c -- LZMA Decoder /* LzmaDec.c -- LZMA Decoder
2008-11-06 : Igor Pavlov : Public domain */ 2009-09-20 : Igor Pavlov : Public domain */
#include "LzmaDec.h" #include "LzmaDec.h"
@ -113,12 +113,6 @@
StopCompilingDueBUG StopCompilingDueBUG
#endif #endif
static const Byte kLiteralNextStates[kNumStates * 2] =
{
0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5,
7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10
};
#define LZMA_DIC_MIN (1 << 12) #define LZMA_DIC_MIN (1 << 12)
/* First LZMA-symbol is always decoded. /* First LZMA-symbol is always decoded.
@ -175,6 +169,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
if (state < kNumLitStates) if (state < kNumLitStates)
{ {
state -= (state < 4) ? state : 3;
symbol = 1; symbol = 1;
do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100); do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100);
} }
@ -182,6 +177,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
{ {
unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)];
unsigned offs = 0x100; unsigned offs = 0x100;
state -= (state < 10) ? 3 : 6;
symbol = 1; symbol = 1;
do do
{ {
@ -196,9 +192,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
} }
dic[dicPos++] = (Byte)symbol; dic[dicPos++] = (Byte)symbol;
processedPos++; processedPos++;
state = kLiteralNextStates[state];
/* if (state < 4) state = 0; else if (state < 10) state -= 3; else state -= 6; */
continue; continue;
} }
else else
@ -378,7 +371,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
else if (distance >= checkDicSize) else if (distance >= checkDicSize)
return SZ_ERROR_DATA; return SZ_ERROR_DATA;
state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3;
/* state = kLiteralNextStates[state]; */
} }
len += kMatchMinLen; len += kMatchMinLen;

40
lzma/C/LzmaEnc.c
View file

@ -1,5 +1,5 @@
/* LzmaEnc.c -- LZMA Encoder /* LzmaEnc.c -- LZMA Encoder
2009-04-22 : Igor Pavlov : Public domain */ 2010-04-16 : Igor Pavlov : Public domain */
#include <string.h> #include <string.h>
@ -13,7 +13,7 @@
#include "LzmaEnc.h" #include "LzmaEnc.h"
#include "LzFind.h" #include "LzFind.h"
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
#include "LzFindMt.h" #include "LzFindMt.h"
#endif #endif
@ -66,7 +66,7 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p)
if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1);
if (p->numThreads < 0) if (p->numThreads < 0)
p->numThreads = p->numThreads =
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
((p->btMode && p->algo) ? 2 : 1); ((p->btMode && p->algo) ? 2 : 1);
#else #else
1; 1;
@ -172,7 +172,7 @@ typedef struct
#define kEndPosModelIndex 14 #define kEndPosModelIndex 14
#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) #define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex)
#define kNumFullDistances (1 << (kEndPosModelIndex / 2)) #define kNumFullDistances (1 << (kEndPosModelIndex >> 1))
#ifdef _LZMA_PROB32 #ifdef _LZMA_PROB32
#define CLzmaProb UInt32 #define CLzmaProb UInt32
@ -259,14 +259,14 @@ typedef struct
IMatchFinder matchFinder; IMatchFinder matchFinder;
void *matchFinderObj; void *matchFinderObj;
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
Bool mtMode; Bool mtMode;
CMatchFinderMt matchFinderMt; CMatchFinderMt matchFinderMt;
#endif #endif
CMatchFinder matchFinderBase; CMatchFinder matchFinderBase;
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
Byte pad[128]; Byte pad[128];
#endif #endif
@ -395,7 +395,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
LzmaEncProps_Normalize(&props); LzmaEncProps_Normalize(&props);
if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX ||
props.dictSize > (unsigned)(1 << kDicLogSizeMaxCompress) || props.dictSize > (1 << 30)) props.dictSize > ((UInt32)1 << kDicLogSizeMaxCompress) || props.dictSize > ((UInt32)1 << 30))
return SZ_ERROR_PARAM; return SZ_ERROR_PARAM;
p->dictSize = props.dictSize; p->dictSize = props.dictSize;
p->matchFinderCycles = props.mc; p->matchFinderCycles = props.mc;
@ -428,7 +428,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
p->writeEndMark = props.writeEndMark; p->writeEndMark = props.writeEndMark;
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
/* /*
if (newMultiThread != _multiThread) if (newMultiThread != _multiThread)
{ {
@ -804,7 +804,7 @@ static void MovePos(CLzmaEnc *p, UInt32 num)
{ {
#ifdef SHOW_STAT #ifdef SHOW_STAT
ttt += num; ttt += num;
fprintf(stderr, "\n MovePos %d", num); printf("\n MovePos %d", num);
#endif #endif
if (num != 0) if (num != 0)
{ {
@ -819,12 +819,12 @@ static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes)
p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj);
numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches);
#ifdef SHOW_STAT #ifdef SHOW_STAT
fprintf(stderr, "\n i = %d numPairs = %d ", ttt, numPairs / 2); printf("\n i = %d numPairs = %d ", ttt, numPairs / 2);
ttt++; ttt++;
{ {
UInt32 i; UInt32 i;
for (i = 0; i < numPairs; i += 2) for (i = 0; i < numPairs; i += 2)
fprintf(stderr, "%2d %6d | ", p->matches[i], p->matches[i + 1]); printf("%2d %6d | ", p->matches[i], p->matches[i + 1]);
} }
#endif #endif
if (numPairs > 0) if (numPairs > 0)
@ -1115,9 +1115,9 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes)
if (position >= 0) if (position >= 0)
{ {
unsigned i; unsigned i;
fprintf(stderr, "\n pos = %4X", position); printf("\n pos = %4X", position);
for (i = cur; i <= lenEnd; i++) for (i = cur; i <= lenEnd; i++)
fprintf(stderr, "\nprice[%4X] = %d", position - cur + i, p->opt[i].price); printf("\nprice[%4X] = %d", position - cur + i, p->opt[i].price);
} }
#endif #endif
@ -1677,7 +1677,7 @@ void LzmaEnc_Construct(CLzmaEnc *p)
{ {
RangeEnc_Construct(&p->rc); RangeEnc_Construct(&p->rc);
MatchFinder_Construct(&p->matchFinderBase); MatchFinder_Construct(&p->matchFinderBase);
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
MatchFinderMt_Construct(&p->matchFinderMt); MatchFinderMt_Construct(&p->matchFinderMt);
p->matchFinderMt.MatchFinder = &p->matchFinderBase; p->matchFinderMt.MatchFinder = &p->matchFinderBase;
#endif #endif
@ -1716,7 +1716,7 @@ void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc)
void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig)
{ {
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); MatchFinderMt_Destruct(&p->matchFinderMt, allocBig);
#endif #endif
MatchFinder_Free(&p->matchFinderBase, allocBig); MatchFinder_Free(&p->matchFinderBase, allocBig);
@ -1772,7 +1772,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize
len = GetOptimum(p, nowPos32, &pos); len = GetOptimum(p, nowPos32, &pos);
#ifdef SHOW_STAT2 #ifdef SHOW_STAT2
fprintf(stderr, "\n pos = %4X, len = %d pos = %d", nowPos32, len, pos); printf("\n pos = %4X, len = %d pos = %d", nowPos32, len, pos);
#endif #endif
posState = nowPos32 & p->pbMask; posState = nowPos32 & p->pbMask;
@ -1901,7 +1901,7 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I
if (!RangeEnc_Alloc(&p->rc, alloc)) if (!RangeEnc_Alloc(&p->rc, alloc))
return SZ_ERROR_MEM; return SZ_ERROR_MEM;
btMode = (p->matchFinderBase.btMode != 0); btMode = (p->matchFinderBase.btMode != 0);
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
p->mtMode = (p->multiThread && !p->fastMode && btMode); p->mtMode = (p->multiThread && !p->fastMode && btMode);
#endif #endif
@ -1926,7 +1926,7 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I
if (beforeSize + p->dictSize < keepWindowSize) if (beforeSize + p->dictSize < keepWindowSize)
beforeSize = keepWindowSize - p->dictSize; beforeSize = keepWindowSize - p->dictSize;
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
if (p->mtMode) if (p->mtMode)
{ {
RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig));
@ -2073,7 +2073,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
void LzmaEnc_Finish(CLzmaEncHandle pp) void LzmaEnc_Finish(CLzmaEncHandle pp)
{ {
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
CLzmaEnc *p = (CLzmaEnc *)pp; CLzmaEnc *p = (CLzmaEnc *)pp;
if (p->mtMode) if (p->mtMode)
MatchFinderMt_ReleaseStream(&p->matchFinderMt); MatchFinderMt_ReleaseStream(&p->matchFinderMt);
@ -2155,7 +2155,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress)
{ {
SRes res = SZ_OK; SRes res = SZ_OK;
#ifdef COMPRESS_MF_MT #ifndef _7ZIP_ST
Byte allocaDummy[0x300]; Byte allocaDummy[0x300];
int i = 0; int i = 0;
for (i = 0; i < 16; i++) for (i = 0; i < 16; i++)

27
lzma/C/Makefile.am
View file

@ -11,25 +11,26 @@ endif
noinst_LTLIBRARIES = liblzma.la noinst_LTLIBRARIES = liblzma.la
liblzma_la_SOURCES = \ liblzma_la_SOURCES = \
$(ASM_S) \ $(ASM_S) \
Alloc.h \
basetyps.h \
LzFind.h \
LzFindMt.h \
LzHash.h \
LzmaDec.h \ LzmaDec.h \
LzmaEnc.h \ LzmaEnc.h \
LzmaLib.h \
MyGuidDef.h \
MyWindows.h \
Threads.h \
Types.h \
windows.h \
Threads.c \
LzFind.c \ LzFind.c \
LzFind.h \
LzFindMt.c \ LzFindMt.c \
LzFindMt.h \
LzmaDec.c \ LzmaDec.c \
LzmaEnc.c \ LzmaEnc.c \
LzmaLib.c LzmaLib.c \
LzmaLib.h \
Alloc.c \
Alloc.h \
Threads.c \
Threads.h \
Types.h \
LzHash.h \
windows.h \
basetyps.h \
MyWindows.h \
MyGuidDef.h
liblzma_so_LIBS = $(ASM_LIBS) liblzma_so_LIBS = $(ASM_LIBS)
liblzma_so_CFLAGS = \ liblzma_so_CFLAGS = \

1
lzma/C/Types.h
View file

@ -46,6 +46,7 @@ typedef int SRes;
typedef DWORD WRes; typedef DWORD WRes;
#else #else
typedef int WRes; typedef int WRes;
typedef void * HANDLE;
#endif #endif
#ifndef RINOK #ifndef RINOK

33
lzma/Methods.txt
View file

@ -1,4 +1,4 @@
7-Zip method IDs (4.61) 7-Zip method IDs (9.18)
----------------------- -----------------------
Each compression or crypto method in 7z has unique binary value (ID). Each compression or crypto method in 7z has unique binary value (ID).
@ -24,14 +24,22 @@ List of defined IDs
------------------- -------------------
00 - Copy 00 - Copy
01 - Reserved
02 - Common 03 - Delta
04 - x86 (BCJ)
05 - PPC (Big Endian)
06 - IA64
07 - ARM (little endian)
08 - ARM Thumb (little endian)
09 - SPARC
21 - LZMA2
02.. - Common
03 Swap 03 Swap
- 2 Swap2 - 2 Swap2
- 4 Swap4 - 4 Swap4
04 Delta (subject to change)
03 - 7z 03.. - 7z
01 - LZMA 01 - LZMA
01 - Version 01 - Version
@ -60,11 +68,8 @@ List of defined IDs
7F - 7F -
01 - experimental methods. 01 - experimental methods.
80 - reserved for independent developers
E0 - Random IDs 04.. - Misc
04 - Misc
00 - Reserved 00 - Reserved
01 - Zip 01 - Zip
00 - Copy (not used). Use {00} instead 00 - Copy (not used). Use {00} instead
@ -72,7 +77,13 @@ List of defined IDs
06 - Implode 06 - Implode
08 - Deflate 08 - Deflate
09 - Deflate64 09 - Deflate64
10 - Imploding
12 - BZip2 (not used). Use {04 02 02} instead 12 - BZip2 (not used). Use {04 02 02} instead
14 - LZMA
60 - Jpeg
61 - WavPack
62 - PPMd
63 - wzAES
02 - BZip 02 - BZip
02 - BZip2 02 - BZip2
03 - Rar 03 - Rar
@ -91,7 +102,7 @@ List of defined IDs
02 - BZip2NSIS 02 - BZip2NSIS
06 - Crypto 06.. - Crypto
00 - 00 -
01 - AES 01 - AES
0x - AES-128 0x - AES-128
@ -118,7 +129,7 @@ List of defined IDs
07 - 7z 07 - 7z
01 - AES-256 + SHA-256 01 - AES-256 + SHA-256
07 - Hash (subject to change) 07.. - Hash (subject to change)
00 - 00 -
01 - CRC 01 - CRC
02 - SHA-1 02 - SHA-1

40
lzma/history.txt
View file

@ -1,6 +1,46 @@
HISTORY of the LZMA SDK HISTORY of the LZMA SDK
----------------------- -----------------------
9.18 beta 2010-11-02
-------------------------
- New small SFX module for installers (SfxSetup).
9.12 beta 2010-03-24
-------------------------
- The BUG in LZMA SDK 9.* was fixed: LZMA2 codec didn't work,
if more than 10 threads were used (or more than 20 threads in some modes).
9.11 beta 2010-03-15
-------------------------
- PPMd compression method support
9.09 2009-12-12
-------------------------
- The bug was fixed:
Utf16_To_Utf8 funstions in UTFConvert.cpp and 7zMain.c
incorrectly converted surrogate characters (the code >= 0x10000) to UTF-8.
- Some bugs were fixed
9.06 2009-08-17
-------------------------
- Some changes in ANSI-C 7z Decoder interfaces.
9.04 2009-05-30
-------------------------
- LZMA2 compression method support
- xz format support
4.65 2009-02-03
-------------------------
- Some minor fixes
4.63 2008-12-31 4.63 2008-12-31
------------------------- -------------------------
- Some minor fixes - Some minor fixes

26
lzma/lzma.txt
View file

@ -1,4 +1,4 @@
LZMA SDK 4.63 LZMA SDK 9.20
------------- -------------
LZMA SDK provides the documentation, samples, header files, libraries, LZMA SDK provides the documentation, samples, header files, libraries,
@ -20,6 +20,10 @@ LICENSE
LZMA SDK is written and placed in the public domain by Igor Pavlov. LZMA SDK is written and placed in the public domain by Igor Pavlov.
Some code in LZMA SDK is based on public domain code from another developers:
1) PPMd var.H (2001): Dmitry Shkarin
2) SHA-256: Wei Dai (Crypto++ library)
LZMA SDK Contents LZMA SDK Contents
----------------- -----------------
@ -33,7 +37,7 @@ LZMA SDK includes:
UNIX/Linux version UNIX/Linux version
------------------ ------------------
To compile C++ version of file->file LZMA encoding, go to directory To compile C++ version of file->file LZMA encoding, go to directory
C++/7zip/Compress/LZMA_Alone CPP/7zip/Bundles/LzmaCon
and call make to recompile it: and call make to recompile it:
make -f makefile.gcc clean all make -f makefile.gcc clean all
@ -49,6 +53,7 @@ lzma.txt - LZMA SDK description (this file)
7zC.txt - 7z ANSI-C Decoder description 7zC.txt - 7z ANSI-C Decoder description
methods.txt - Compression method IDs for .7z methods.txt - Compression method IDs for .7z
lzma.exe - Compiled file->file LZMA encoder/decoder for Windows lzma.exe - Compiled file->file LZMA encoder/decoder for Windows
7zr.exe - 7-Zip with 7z/lzma/xz support.
history.txt - history of the LZMA SDK history.txt - history of the LZMA SDK
@ -86,12 +91,6 @@ CPP/ -- CPP files
Compress - files related to compression/decompression Compress - files related to compression/decompression
Copy - Copy coder
RangeCoder - Range Coder (special code of compression/decompression)
LZMA - LZMA compression/decompression on C++
LZMA_Alone - file->file LZMA compression/decompression
Branch - Filters for x86, IA-64, ARM, ARM-Thumb, PowerPC and SPARC code
Archive - files related to archiving Archive - files related to archiving
Common - common files for archive handling Common - common files for archive handling
@ -100,6 +99,7 @@ CPP/ -- CPP files
Bundles - Modules that are bundles of other modules Bundles - Modules that are bundles of other modules
Alone7z - 7zr.exe: Standalone version of 7z.exe that supports only 7z/LZMA/BCJ/BCJ2 Alone7z - 7zr.exe: Standalone version of 7z.exe that supports only 7z/LZMA/BCJ/BCJ2
LzmaCon - lzma.exe: LZMA compression/decompression
Format7zR - 7zr.dll: Reduced version of 7za.dll: extracting/compressing to 7z/LZMA/BCJ/BCJ2 Format7zR - 7zr.dll: Reduced version of 7za.dll: extracting/compressing to 7z/LZMA/BCJ/BCJ2
Format7zExtractR - 7zxr.dll: Reduced version of 7zxa.dll: extracting from 7z/LZMA/BCJ/BCJ2. Format7zExtractR - 7zxr.dll: Reduced version of 7zxa.dll: extracting from 7z/LZMA/BCJ/BCJ2.
@ -527,7 +527,8 @@ static ISzAlloc g_Alloc = { SzAlloc, SzFree };
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
If callback function return some error code, LzmaEnc_Encode also returns that code. If callback function return some error code, LzmaEnc_Encode also returns that code
or it can return the code like SZ_ERROR_READ, SZ_ERROR_WRITE or SZ_ERROR_PROGRESS.
Single-call RAM->RAM Compression Single-call RAM->RAM Compression
@ -549,8 +550,8 @@ Return code:
LZMA Defines Defines
------------ -------
_LZMA_SIZE_OPT - Enable some optimizations in LZMA Decoder to get smaller executable code. _LZMA_SIZE_OPT - Enable some optimizations in LZMA Decoder to get smaller executable code.
@ -562,6 +563,9 @@ _LZMA_UINT32_IS_ULONG - Define it if int is 16-bit on your compiler and long is
_LZMA_NO_SYSTEM_SIZE_T - Define it if you don't want to use size_t type. _LZMA_NO_SYSTEM_SIZE_T - Define it if you don't want to use size_t type.
_7ZIP_PPMD_SUPPPORT - Define it if you don't want to support PPMD method in AMSI-C .7z decoder.
C++ LZMA Encoder/Decoder C++ LZMA Encoder/Decoder
~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~
C++ LZMA code use COM-like interfaces. So if you want to use it, C++ LZMA code use COM-like interfaces. So if you want to use it,

5
runzip.c
View file

@ -285,6 +285,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64
else if (control->major_version == 0 && control->minor_version == 4) else if (control->major_version == 0 && control->minor_version == 4)
chunk_bytes = 8; chunk_bytes = 8;
else { else {
print_maxverbose("Reading chunk_bytes at %lld\n", get_readseek(control, fd_in));
/* Read in the stored chunk byte width from the file */ /* Read in the stored chunk byte width from the file */
if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1)) if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1))
fatal_return(("Failed to read chunk_bytes size in runzip_chunk\n"), -1); fatal_return(("Failed to read chunk_bytes size in runzip_chunk\n"), -1);
@ -352,8 +353,8 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64
print_maxverbose("Checksum for block: 0x%08x\n", cksum); print_maxverbose("Checksum for block: 0x%08x\n", cksum);
} }
if (unlikely(close_stream_in(ss))) if (unlikely(close_stream_in(control, ss)))
fatal_return(("Failed to close stream!\n"), -1); fatal("Failed to close stream!\n");
return total; return total;
} }

36
stream.c
View file

@ -921,7 +921,7 @@ static i64 get_seek(rzip_control *control, int fd)
return ret; return ret;
} }
static i64 get_readseek(rzip_control *control, int fd) i64 get_readseek(rzip_control *control, int fd)
{ {
i64 ret; i64 ret;
@ -1159,12 +1159,16 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes)
if (control->major_version == 0 && control->minor_version > 5) { if (control->major_version == 0 && control->minor_version > 5) {
/* Read in flag that tells us if there are more chunks after /* Read in flag that tells us if there are more chunks after
* this. Ignored if we know the final file size */ * this. Ignored if we know the final file size */
print_maxverbose("Reading eof flag at %lld\n", get_readseek(control, f));
if (unlikely(read_u8(control, f, &control->eof))) { if (unlikely(read_u8(control, f, &control->eof))) {
print_err("Failed to read eof flag in open_stream_in\n"); print_err("Failed to read eof flag in open_stream_in\n");
goto failed; goto failed;
} }
print_maxverbose("EOF: %d\n", control->eof);
/* Read in the expected chunk size */ /* Read in the expected chunk size */
if (!ENCRYPT) { if (!ENCRYPT) {
print_maxverbose("Reading expected chunksize at %lld\n", get_readseek(control, f));
if (unlikely(read_val(control, f, &sinfo->size, sinfo->chunk_bytes))) { if (unlikely(read_val(control, f, &sinfo->size, sinfo->chunk_bytes))) {
print_err("Failed to read in chunk size in open_stream_in\n"); print_err("Failed to read in chunk size in open_stream_in\n");
goto failed; goto failed;
@ -1209,6 +1213,7 @@ again:
} else { } else {
int read_len; int read_len;
print_maxverbose("Reading stream %d header at %lld\n", i, get_readseek(control, f));
if ((control->major_version == 0 && control->minor_version < 6) || if ((control->major_version == 0 && control->minor_version < 6) ||
ENCRYPT) ENCRYPT)
read_len = 8; read_len = 8;
@ -1222,6 +1227,8 @@ again:
goto failed; goto failed;
header_length = 1 + (read_len * 3); header_length = 1 + (read_len * 3);
} }
sinfo->total_read += header_length;
if (ENCRYPT) if (ENCRYPT)
if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed;
@ -1388,11 +1395,14 @@ retry:
if (unlikely(!flush_tmpoutbuf(control))) goto error; if (unlikely(!flush_tmpoutbuf(control))) goto error;
} }
print_maxverbose("Writing initial chunk bytes value %d at %lld\n",
ctis->chunk_bytes, get_seek(control, ctis->fd));
/* Write chunk bytes of this block */ /* Write chunk bytes of this block */
write_u8(control, ctis->chunk_bytes); write_u8(control, ctis->chunk_bytes);
/* Write whether this is the last chunk, followed by the size /* Write whether this is the last chunk, followed by the size
* of this chunk */ * of this chunk */
print_maxverbose("Writing EOF flag as %d\n", control->eof);
write_u8(control, control->eof); write_u8(control, control->eof);
if (!ENCRYPT) if (!ENCRYPT)
write_val(control, ctis->size, ctis->chunk_bytes); write_val(control, ctis->size, ctis->chunk_bytes);
@ -1401,6 +1411,7 @@ retry:
ctis->initial_pos = get_seek(control, ctis->fd); ctis->initial_pos = get_seek(control, ctis->fd);
if (unlikely(ctis->initial_pos == -1)) goto error; if (unlikely(ctis->initial_pos == -1)) goto error;
print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos);
for (j = 0; j < ctis->num_streams; j++) { for (j = 0; j < ctis->num_streams; j++) {
/* If encrypting, we leave SALT_LEN room to write in salt /* If encrypting, we leave SALT_LEN room to write in salt
* later */ * later */
@ -1418,6 +1429,8 @@ retry:
} }
} }
print_maxverbose("Compthread %ld seeking to %lld to store length %d\n", i, ctis->s[cti->streamno].last_head, write_len);
if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head)))
fatal_goto(("Failed to seekto in compthread %d\n", i), error); fatal_goto(("Failed to seekto in compthread %d\n", i), error);
@ -1428,6 +1441,9 @@ retry:
rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17); rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17);
ctis->s[cti->streamno].last_head = ctis->cur_pos + 1 + (write_len * 2) + (ENCRYPT ? SALT_LEN : 0); ctis->s[cti->streamno].last_head = ctis->cur_pos + 1 + (write_len * 2) + (ENCRYPT ? SALT_LEN : 0);
print_maxverbose("Compthread %ld seeking to %lld to write header\n", i, ctis->cur_pos);
if (unlikely(seekto(control, ctis, ctis->cur_pos))) if (unlikely(seekto(control, ctis, ctis->cur_pos)))
fatal_goto(("Failed to seekto cur_pos in compthread %d\n", i), error); fatal_goto(("Failed to seekto cur_pos in compthread %d\n", i), error);
@ -1455,6 +1471,9 @@ retry:
if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error;
ctis->cur_pos += SALT_LEN; ctis->cur_pos += SALT_LEN;
} }
print_maxverbose("Compthread %ld writing data at %lld\n", i, ctis->cur_pos);
if (unlikely(write_buf(control, cti->s_buf, padded_len))) if (unlikely(write_buf(control, cti->s_buf, padded_len)))
fatal_goto(("Failed to write_buf s_buf in compthread %d\n", i), error); fatal_goto(("Failed to write_buf s_buf in compthread %d\n", i), error);
@ -1588,7 +1607,7 @@ retry:
/* fill a buffer from a stream - return -1 on failure */ /* fill a buffer from a stream - return -1 on failure */
static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int streamno)
{ {
i64 u_len, c_len, last_head, padded_len; i64 u_len, c_len, last_head, padded_len, header_length;
uchar enc_head[25 + SALT_LEN], blocksalt[SALT_LEN]; uchar enc_head[25 + SALT_LEN], blocksalt[SALT_LEN];
struct stream *s = &sinfo->s[streamno]; struct stream *s = &sinfo->s[streamno];
stream_thread_struct *st; stream_thread_struct *st;
@ -1624,9 +1643,11 @@ fill_another:
c_len = c_len32; c_len = c_len32;
u_len = u_len32; u_len = u_len32;
last_head = last_head32; last_head = last_head32;
header_length = 13;
} else { } else {
int read_len; int read_len;
print_maxverbose("Reading ucomp header at %lld\n", get_readseek(control, sinfo->fd));
if ((control->major_version == 0 && control->minor_version < 6) || ENCRYPT) if ((control->major_version == 0 && control->minor_version < 6) || ENCRYPT)
read_len = 8; read_len = 8;
else else
@ -1637,7 +1658,9 @@ fill_another:
return -1; return -1;
if (unlikely(read_val(control, sinfo->fd, &last_head, read_len))) if (unlikely(read_val(control, sinfo->fd, &last_head, read_len)))
return -1; return -1;
header_length = 1 + (read_len * 3);
} }
sinfo->total_read += header_length;
if (ENCRYPT) { if (ENCRYPT) {
if (unlikely(!decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head))) return -1; if (unlikely(!decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head))) return -1;
@ -1649,6 +1672,7 @@ fill_another:
last_head = le64toh(last_head); last_head = le64toh(last_head);
padded_len = MAX(c_len, MIN_SIZE); padded_len = MAX(c_len, MIN_SIZE);
sinfo->total_read += padded_len;
fsync(control->fd_out); fsync(control->fd_out);
s_buf = malloc(MAX(u_len, MIN_SIZE)); s_buf = malloc(MAX(u_len, MIN_SIZE));
@ -1826,11 +1850,17 @@ int close_stream_out(rzip_control *control, void *ss)
} }
/* close down an input stream */ /* close down an input stream */
int close_stream_in(void *ss) int close_stream_in(rzip_control *control, void *ss)
{ {
struct stream_info *sinfo = ss; struct stream_info *sinfo = ss;
int i; int i;
print_maxverbose("Closing stream at %lld, want to seek to %lld\n",
get_readseek(control, control->fd_in),
sinfo->initial_pos + sinfo->total_read);
if (unlikely(read_seekto(control, sinfo, sinfo->total_read)))
return -1;
for (i = 0; i < sinfo->num_streams; i++) for (i = 0; i < sinfo->num_streams; i++)
free(sinfo->s[i].buf); free(sinfo->s[i].buf);

7
stream.h
View file

@ -28,15 +28,16 @@ bool create_pthread(pthread_t *thread, pthread_attr_t *attr,
bool join_pthread(pthread_t th, void **thread_return); bool join_pthread(pthread_t th, void **thread_return);
ssize_t write_1g(rzip_control *control, void *buf, i64 len); ssize_t write_1g(rzip_control *control, void *buf, i64 len);
ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len); ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len);
bool prepare_streamout_threads(rzip_control *control); i64 get_readseek(rzip_control *control, int fd);
bool close_streamout_threads(rzip_control *control); void prepare_streamout_threads(rzip_control *control);
void close_streamout_threads(rzip_control *control);
void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes); void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes);
void *open_stream_in(rzip_control *control, int f, int n, char cbytes); void *open_stream_in(rzip_control *control, int f, int n, char cbytes);
bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream);
int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len);
i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len);
int close_stream_out(rzip_control *control, void *ss); int close_stream_out(rzip_control *control, void *ss);
int close_stream_in(void *ss); int close_stream_in(rzip_control *control, void *ss);
ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret); ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret);
#endif #endif