ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

rebase-of-function-split-and-control-additions-to-fu

Browse source
This commit is contained in:
Con Kolivas 2011-03-09 08:32:14 +11:00
parent f0828d5b38
commit f6f0a25ef6
12 changed files with 1230 additions and 1006 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Makefile.am
View file

@ -31,6 +31,9 @@ lrztar_SCRIPTS = lrztar
bin_PROGRAMS = lrzip bin_PROGRAMS = lrzip
lrzip_SOURCES = \ lrzip_SOURCES = \
zpipe.cpp \ zpipe.cpp \
liblrzip.h \
lrzip.c \
lrzip.h \
main.c \ main.c \
rzip.h \ rzip.h \
rzip.c \ rzip.c \

2
configure.ac
View file

@ -70,7 +70,7 @@ AC_CHECK_HEADERS(sys/param.h ctype.h sys/wait.h sys/ioctl.h errno.h)
AC_TYPE_OFF_T AC_TYPE_OFF_T
AC_TYPE_SIZE_T AC_TYPE_SIZE_T
AC_C___ATTRIBUTE__
AC_CHECK_SIZEOF(int) AC_CHECK_SIZEOF(int)
AC_CHECK_SIZEOF(long) AC_CHECK_SIZEOF(long)
AC_CHECK_SIZEOF(short) AC_CHECK_SIZEOF(short)

70
liblrzip.h Normal file
View file

@ -0,0 +1,70 @@
/*
Copyright (C) 2006-2011 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#if !defined(LIBLRZIP_H) && !defined(MAIN_C)
#define LIBLRZIP_H
#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS)
#define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA))
#define SHOW_PROGRESS (control->flags & FLAG_SHOW_PROGRESS)
#define KEEP_FILES (control->flags & FLAG_KEEP_FILES)
#define TEST_ONLY (control->flags & FLAG_TEST_ONLY)
#define FORCE_REPLACE (control->flags & FLAG_FORCE_REPLACE)
#define DECOMPRESS (control->flags & FLAG_DECOMPRESS)
#define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS)
#define LZO_COMPRESS (control->flags & FLAG_LZO_COMPRESS)
#define BZIP2_COMPRESS (control->flags & FLAG_BZIP2_COMPRESS)
#define ZLIB_COMPRESS (control->flags & FLAG_ZLIB_COMPRESS)
#define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS)
#define VERBOSE (control->flags & FLAG_VERBOSE)
#define VERBOSITY (control->flags & FLAG_VERBOSITY)
#define MAX_VERBOSE (control->flags & FLAG_VERBOSITY_MAX)
#define STDIN (control->flags & FLAG_STDIN)
#define STDOUT (control->flags & FLAG_STDOUT)
#define INFO (control->flags & FLAG_INFO)
#define UNLIMITED (control->flags & FLAG_UNLIMITED)
#define HASH_CHECK (control->flags & FLAG_HASH)
#define HAS_MD5 (control->flags & FLAG_MD5)
#define CHECK_FILE (control->flags & FLAG_CHECK)
#define KEEP_BROKEN (control->flags & FLAG_KEEP_BROKEN)
#define LZO_TEST (control->flags & FLAG_THRESHOLD)
#define print_output(format, args...) do {\
fprintf(control->msgout, format, ##args); \
fflush(control->msgout); \
} while (0)
#define print_progress(format, args...) do {\
if (SHOW_PROGRESS) \
print_output(format, ##args); \
} while (0)
#define print_verbose(format, args...) do {\
if (VERBOSE) \
print_output(format, ##args); \
} while (0)
#define print_maxverbose(format, args...) do {\
if (MAX_VERBOSE) \
print_output(format, ##args); \
} while (0)
#endif

668
lrzip.c Normal file
View file

@ -0,0 +1,668 @@
/*
Copyright (C) 2006-2011 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rzip.h"
#include "liblrzip.h" /* flag defines */
void write_magic(rzip_control *control, int fd_in, int fd_out)
{
struct stat st;
char magic[24];
int i;
memset(magic, 0, sizeof(magic));
strcpy(magic, "LRZI");
magic[4] = LRZIP_MAJOR_VERSION;
magic[5] = LRZIP_MINOR_VERSION;
if (unlikely(fstat(fd_in, &st)))
fatal("bad magic file descriptor!?\n");
memcpy(&magic[6], &control->st_size, 8);
/* save LZMA compression flags */
if (LZMA_COMPRESS) {
for (i = 0; i < 5; i++)
magic[i + 16] = (char)control->lzma_properties[i];
}
/* This is a flag that the archive contains an md5 sum at the end
* which can be used as an integrity check instead of crc check.
* crc is still stored for compatibility with 0.5 versions.
*/
magic[21] = 1;
if (unlikely(lseek(fd_out, 0, SEEK_SET)))
fatal("Failed to seek to BOF to write Magic Header\n");
if (unlikely(write(fd_out, magic, sizeof(magic)) != sizeof(magic)))
fatal("Failed to write magic header\n");
}
void read_magic(rzip_control *control, int fd_in, i64 *expected_size)
{
char magic[24];
uint32_t v;
int md5, i;
if (unlikely(read(fd_in, magic, sizeof(magic)) != sizeof(magic)))
fatal("Failed to read magic header\n");
*expected_size = 0;
if (unlikely(strncmp(magic, "LRZI", 4)))
failure("Not an lrzip file\n");
memcpy(&control->major_version, &magic[4], 1);
memcpy(&control->minor_version, &magic[5], 1);
/* Support the convoluted way we described size in versions < 0.40 */
if (control->major_version == 0 && control->minor_version < 4) {
memcpy(&v, &magic[6], 4);
*expected_size = ntohl(v);
memcpy(&v, &magic[10], 4);
*expected_size |= ((i64)ntohl(v)) << 32;
} else
memcpy(expected_size, &magic[6], 8);
/* restore LZMA compression flags only if stored */
if ((int) magic[16]) {
for (i = 0; i < 5; i++)
control->lzma_properties[i] = magic[i + 16];
}
/* Whether this archive contains md5 data at the end or not */
md5 = magic[21];
if (md5 == 1)
control->flags |= FLAG_MD5;
print_verbose("Detected lrzip version %d.%d file.\n", control->major_version, control->minor_version);
if (control->major_version > LRZIP_MAJOR_VERSION ||
(control->major_version == LRZIP_MAJOR_VERSION && control->minor_version > LRZIP_MINOR_VERSION))
print_output("Attempting to work with file produced by newer lrzip version %d.%d file.\n", control->major_version, control->minor_version);
}
/* preserve ownership and permissions where possible */
void preserve_perms(rzip_control *control, int fd_in, int fd_out)
{
struct stat st;
if (unlikely(fstat(fd_in, &st)))
fatal("Failed to fstat input file\n");
if (unlikely(fchmod(fd_out, (st.st_mode & 0777))))
print_err("Warning, unable to set permissions on %s\n", control->outfile);
/* chown fail is not fatal */
if (unlikely(fchown(fd_out, st.st_uid, st.st_gid)))
print_err("Warning, unable to set owner on %s\n", control->outfile);
}
/* Open a temporary outputfile to emulate stdout */
int open_tmpoutfile(rzip_control *control)
{
int fd_out;
if (STDOUT)
print_verbose("Outputting to stdout.\n");
if (control->tmpdir) {
control->outfile = realloc(NULL, strlen(control->tmpdir)+16);
if (unlikely(!control->outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control->outfile, control->tmpdir);
strcat(control->outfile, "lrzipout.XXXXXX");
} else {
control->outfile = realloc(NULL, 16);
if (unlikely(!control->outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control->outfile, "lrzipout.XXXXXX");
}
fd_out = mkstemp(control->outfile);
if (unlikely(fd_out == -1))
fatal("Failed to create out tmpfile: %s\n", strerror(errno));
register_outfile(control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN);
return fd_out;
}
/* Dump temporary outputfile to perform stdout */
void dump_tmpoutfile(rzip_control *control, int fd_out)
{
FILE *tmpoutfp;
int tmpchar;
print_progress("Dumping to stdout.\n");
/* flush anything not yet in the temporary file */
fsync(fd_out);
tmpoutfp = fdopen(fd_out, "r");
if (unlikely(tmpoutfp == NULL))
fatal("Failed to fdopen out tmpfile: %s\n", strerror(errno));
rewind(tmpoutfp);
while ((tmpchar = fgetc(tmpoutfp)) != EOF)
putchar(tmpchar);
fflush(control->msgout);
}
/* Open a temporary inputfile to perform stdin decompression */
int open_tmpinfile(rzip_control *control)
{
int fd_in;
if (control->tmpdir) {
control->infile = malloc(strlen(control->tmpdir)+15);
if (unlikely(!control->infile))
fatal("Failed to allocate infile name\n");
strcpy(control->infile, control->tmpdir);
strcat(control->infile, "lrzipin.XXXXXX");
} else {
control->infile = malloc(15);
if (unlikely(!control->infile))
fatal("Failed to allocate infile name\n");
strcpy(control->infile, "lrzipin.XXXXXX");
}
fd_in = mkstemp(control->infile);
if (unlikely(fd_in == -1))
fatal("Failed to create in tmpfile: %s\n", strerror(errno));
register_infile(control->infile, (DECOMPRESS || TEST_ONLY) && STDIN);
return fd_in;
}
/* Read data from stdin into temporary inputfile */
void read_tmpinfile(rzip_control *control, int fd_in)
{
FILE *tmpinfp;
int tmpchar;
if (control->flags & FLAG_SHOW_PROGRESS)
fprintf(control->msgout, "Copying from stdin.\n");
tmpinfp = fdopen(fd_in, "w+");
if (unlikely(tmpinfp == NULL))
fatal("Failed to fdopen in tmpfile: %s\n", strerror(errno));
while ((tmpchar = getchar()) != EOF)
fputc(tmpchar, tmpinfp);
fflush(tmpinfp);
rewind(tmpinfp);
}
/*
decompress one file from the command line
*/
void decompress_file(rzip_control *control)
{
char *tmp, *tmpoutfile, *infilecopy = NULL;
int fd_in, fd_out = -1, fd_hist = -1;
i64 expected_size, free_space;
struct statvfs fbuf;
if (!STDIN) {
if ((tmp = strrchr(control->infile, '.')) && strcmp(tmp,control->suffix)) {
/* make sure infile has an extension. If not, add it
* because manipulations may be made to input filename, set local ptr
*/
infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1);
if (unlikely(infilecopy == NULL))
fatal("Failed to allocate memory for infile suffix\n");
else {
strcpy(infilecopy, control->infile);
strcat(infilecopy, control->suffix);
}
} else
infilecopy = strdup(control->infile);
/* regardless, infilecopy has the input filename */
}
if (!STDOUT && !TEST_ONLY) {
/* if output name already set, use it */
if (control->outname) {
control->outfile = strdup(control->outname);
} else {
/* default output name from infilecopy
* test if outdir specified. If so, strip path from filename of
* infilecopy, then remove suffix.
*/
if (control->outdir && (tmp = strrchr(infilecopy, '/')))
tmpoutfile = strdup(tmp + 1);
else
tmpoutfile = strdup(infilecopy);
/* remove suffix to make outfile name */
if ((tmp = strrchr(tmpoutfile, '.')) && !strcmp(tmp, control->suffix))
*tmp='\0';
control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpoutfile) + 1);
if (unlikely(!control->outfile))
fatal("Failed to allocate outfile name\n");
if (control->outdir) { /* prepend control->outdir */
strcpy(control->outfile, control->outdir);
strcat(control->outfile, tmpoutfile);
} else
strcpy(control->outfile, tmpoutfile);
free(tmpoutfile);
}
if (!STDOUT)
print_progress("Output filename is: %s...Decompressing...\n", control->outfile);
}
if (STDIN) {
fd_in = open_tmpinfile(control);
read_tmpinfile(control, fd_in);
} else {
fd_in = open(infilecopy, O_RDONLY);
if (unlikely(fd_in == -1)) {
fatal("Failed to open %s: %s\n",
infilecopy,
strerror(errno));
}
}
if (!(TEST_ONLY | STDOUT)) {
if (FORCE_REPLACE)
fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
else
fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666);
if (unlikely(fd_out == -1)) {
/* We must ensure we don't delete a file that already
* exists just because we tried to create a new one */
control->flags |= FLAG_KEEP_BROKEN;
fatal("Failed to create %s: %s\n", control->outfile, strerror(errno));
}
preserve_perms(control, fd_in, fd_out);
} else
fd_out = open_tmpoutfile(control);
control->fd_out = fd_out;
/* Check if there's enough free space on the device chosen to fit the
* decompressed file. */
if (unlikely(fstatvfs(fd_out, &fbuf)))
fatal("Failed to fstatvfs in decompress_file\n");
free_space = fbuf.f_bsize * fbuf.f_bavail;
if (free_space < expected_size) {
if (FORCE_REPLACE)
print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n");
else
failure("Inadequate free space to decompress file, use -f to override.\n");
}
fd_hist = open(control->outfile, O_RDONLY);
if (unlikely(fd_hist == -1))
fatal("Failed to open history file %s\n", control->outfile);
read_magic(control, fd_in, &expected_size);
if (NO_MD5)
print_verbose("Not performing MD5 hash check\n");
if (HAS_MD5)
print_verbose("MD5 ");
else
print_verbose("CRC32 ");
print_verbose("being used for integrity testing.\n");
print_progress("Decompressing...");
runzip_fd(control, fd_in, fd_out, fd_hist, expected_size);
if (STDOUT)
dump_tmpoutfile(control, fd_out);
/* if we get here, no fatal errors during decompression */
print_progress("\r");
if (!(STDOUT | TEST_ONLY))
print_output("Output filename is: %s: ", control->outfile);
print_progress("[OK] - %lld bytes \n", expected_size);
if (unlikely(close(fd_hist) || close(fd_out)))
fatal("Failed to close files\n");
if (TEST_ONLY | STDOUT) {
/* Delete temporary files generated for testing or faking stdout */
if (unlikely(unlink(control->outfile)))
fatal("Failed to unlink tmpfile: %s\n", strerror(errno));
}
close(fd_in);
if (!(KEEP_FILES | TEST_ONLY) || STDIN) {
if (unlikely(unlink(control->infile)))
fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno));
}
free(control->outfile);
free(infilecopy);
}
void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head)
{
if (unlikely(read(fd_in, ctype, 1) != 1))
fatal("Failed to read in get_header_info\n");
if (control->major_version == 0 && control->minor_version < 4) {
u32 c_len32, u_len32, last_head32;
if (unlikely(read(fd_in, &c_len32, 4) != 4))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, &u_len32, 4) != 4))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, &last_head32, 4) != 4))
fatal("Failed to read in get_header_info");
*c_len = c_len32;
*u_len = u_len32;
*last_head = last_head32;
} else {
if (unlikely(read(fd_in, c_len, 8) != 8))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, u_len, 8) != 8))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, last_head, 8) != 8))
fatal("Failed to read_i64 in get_header_info");
}
}
void get_fileinfo(rzip_control *control)
{
i64 expected_size, infile_size;
int seekspot, fd_in;
long double cratio;
uchar ctype = 0;
struct stat st;
char *tmp, *infilecopy = NULL;
if (!STDIN) {
if ((tmp = strrchr(control->infile, '.')) && strcmp(tmp,control->suffix)) {
infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1);
if (unlikely(infilecopy == NULL))
fatal("Failed to allocate memory for infile suffix\n");
else {
strcpy(infilecopy, control->infile);
strcat(infilecopy, control->suffix);
}
} else
infilecopy = strdup(control->infile);
}
if (STDIN)
fd_in = 0;
else {
fd_in = open(infilecopy, O_RDONLY);
if (unlikely(fd_in == -1))
fatal("Failed to open %s: %s\n", infilecopy, strerror(errno));
}
/* Get file size */
if (unlikely(fstat(fd_in, &st)))
fatal("bad magic file descriptor!?\n");
memcpy(&infile_size, &st.st_size, 8);
/* Get decompressed size */
read_magic(control, fd_in, &expected_size);
/* Version < 0.4 had different file format */
if (control->major_version == 0 && control->minor_version < 4)
seekspot = 50;
else if (control->major_version == 0 && control->minor_version == 4)
seekspot = 74;
else
seekspot = 75;
if (unlikely(lseek(fd_in, seekspot, SEEK_SET) == -1))
fatal("Failed to lseek in get_fileinfo: %s\n", strerror(errno));
/* Read the compression type of the first block. It's possible that
not all blocks are compressed so this may not be accurate.
*/
if (unlikely(read(fd_in, &ctype, 1) != 1))
fatal("Failed to read in get_fileinfo\n");
cratio = (long double)expected_size / (long double)infile_size;
print_output("%s:\nlrzip version: %d.%d file\n", infilecopy, control->major_version, control->minor_version);
print_output("Compression: ");
if (ctype == CTYPE_NONE)
print_output("rzip alone\n");
else if (ctype == CTYPE_BZIP2)
print_output("rzip + bzip2\n");
else if (ctype == CTYPE_LZO)
print_output("rzip + lzo\n");
else if (ctype == CTYPE_LZMA)
print_output("rzip + lzma\n");
else if (ctype == CTYPE_GZIP)
print_output("rzip + gzip\n");
else if (ctype == CTYPE_ZPAQ)
print_output("rzip + zpaq\n");
else
print_output("Dunno wtf\n");
print_output("Decompressed file size: %llu\n", expected_size);
print_output("Compressed file size: %llu\n", infile_size);
print_output("Compression ratio: %.3Lf\n", cratio);
if (HAS_MD5) {
char md5_stored[MD5_DIGEST_SIZE];
int i;
print_output("MD5 used for integrity testing\n");
if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1)
fatal("Failed to seek to md5 data in runzip_fd\n");
if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
fatal("Failed to read md5 data in runzip_fd\n");
print_output("MD5: ");
for (i = 0; i < MD5_DIGEST_SIZE; i++)
print_output("%02x", md5_stored[i] & 0xFF);
print_output("\n");
} else
print_output("CRC32 used for integrity testing\n");
if (VERBOSE || MAX_VERBOSE) {
i64 u_len, c_len, last_head, utotal = 0, ctotal = 0, ofs = 25,
stream_head[2];
int header_length = 25, stream = 0, chunk = 0;
if (control->major_version == 0 && control->minor_version < 4)
header_length = 13;
next_chunk:
stream = 0;
stream_head[0] = 0;
stream_head[1] = stream_head[0] + header_length;
print_output("Rzip chunk %d:\n", ++chunk);
while (stream < NUM_STREAMS) {
int block = 1;
if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET)) == -1)
fatal("Failed to seek to header data in get_fileinfo\n");
get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head);
print_output("Stream: %d\n", stream);
print_maxverbose("Offset: %lld\n", ofs);
print_output("Block\tComp\tPercent\tSize\n");
do {
i64 head_off;
if (unlikely(last_head + ofs > infile_size))
failure("Offset greater than archive size, likely corrupted/truncated archive.\n");
if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1)
fatal("Failed to seek to header data in get_fileinfo\n");
get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head);
if (unlikely(last_head < 0 || c_len < 0 || u_len < 0))
failure("Entry negative, likely corrupted archive.\n");
print_output("%d\t", block);
if (ctype == CTYPE_NONE)
print_output("none");
else if (ctype == CTYPE_BZIP2)
print_output("bzip2");
else if (ctype == CTYPE_LZO)
print_output("lzo");
else if (ctype == CTYPE_LZMA)
print_output("lzma");
else if (ctype == CTYPE_GZIP)
print_output("gzip");
else if (ctype == CTYPE_ZPAQ)
print_output("zpaq");
else
print_output("Dunno wtf");
utotal += u_len;
ctotal += c_len;
print_output("\t%.1f%%\t%lld / %lld", (double)c_len / (double)(u_len / 100), c_len, u_len);
print_maxverbose("\tOffset: %lld\tHead: %lld", head_off, last_head);
print_output("\n");
block++;
} while (last_head);
++stream;
}
ofs = lseek(fd_in, 0, SEEK_CUR) + c_len;
/* Chunk byte entry */
if (control->major_version == 0 && control->minor_version > 4)
ofs++;
if (ofs < infile_size - MD5_DIGEST_SIZE)
goto next_chunk;
if (unlikely(ofs > infile_size))
failure("Offset greater than archive size, likely corrupted/truncated archive.\n");
print_output("Rzip compression: %.1f%% %lld / %lld\n",
(double)utotal / (double)(expected_size / 100),
utotal, expected_size);
print_output("Back end compression: %.1f%% %lld / %lld\n",
(double)ctotal / (double)(utotal / 100),
ctotal, utotal);
print_output("Overall compression: %.1f%% %lld / %lld\n",
(double)ctotal / (double)(expected_size / 100),
ctotal, expected_size);
}
if (STDIN) {
if (unlikely(unlink(control->infile)))
fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno));
} else
if (unlikely(close(fd_in)))
fatal("Failed to close fd_in in get_fileinfo\n");
free(control->outfile);
free(infilecopy);
}
/*
compress one file from the command line
*/
void compress_file(rzip_control *control)
{
const char *tmp, *tmpinfile; /* we're just using this as a proxy for control->infile.
* Spares a compiler warning
*/
int fd_in, fd_out;
char header[24];
memset(header, 0, sizeof(header));
if (!STDIN) {
/* is extension at end of infile? */
if ((tmp = strrchr(control->infile, '.')) && !strcmp(tmp, control->suffix)) {
print_err("%s: already has %s suffix. Skipping...\n", control->infile, control->suffix);
return;
}
fd_in = open(control->infile, O_RDONLY);
if (unlikely(fd_in == -1))
fatal("Failed to open %s: %s\n", control->infile, strerror(errno));
} else
fd_in = 0;
if (!STDOUT) {
if (control->outname) {
/* check if outname has control->suffix */
if (*(control->suffix) == '\0') /* suffix is empty string */
control->outfile = strdup(control->outname);
else if ((tmp=strrchr(control->outname, '.')) && strcmp(tmp, control->suffix)) {
control->outfile = malloc(strlen(control->outname) + strlen(control->suffix) + 1);
if (unlikely(!control->outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control->outfile, control->outname);
strcat(control->outfile, control->suffix);
print_output("Suffix added to %s.\nFull pathname is: %s\n", control->outname, control->outfile);
} else /* no, already has suffix */
control->outfile = strdup(control->outname);
} else {
/* default output name from control->infile
* test if outdir specified. If so, strip path from filename of
* control->infile
*/
if (control->outdir && (tmp = strrchr(control->infile, '/')))
tmpinfile = tmp + 1;
else
tmpinfile = control->infile;
control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpinfile) + strlen(control->suffix) + 1);
if (unlikely(!control->outfile))
fatal("Failed to allocate outfile name\n");
if (control->outdir) { /* prepend control->outdir */
strcpy(control->outfile, control->outdir);
strcat(control->outfile, tmpinfile);
} else
strcpy(control->outfile, tmpinfile);
strcat(control->outfile, control->suffix);
print_progress("Output filename is: %s\n", control->outfile);
}
if (FORCE_REPLACE)
fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
else
fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666);
if (unlikely(fd_out == -1)) {
/* We must ensure we don't delete a file that already
* exists just because we tried to create a new one */
control->flags |= FLAG_KEEP_BROKEN;
fatal("Failed to create %s: %s\n", control->outfile, strerror(errno));
}
} else
fd_out = open_tmpoutfile(control);
control->fd_out = fd_out;
preserve_perms(control, fd_in, fd_out);
/* write zeroes to 24 bytes at beginning of file */
if (unlikely(write(fd_out, header, sizeof(header)) != sizeof(header)))
fatal("Cannot write file header\n");
rzip_fd(control, fd_in, fd_out);
/* write magic at end b/c lzma does not tell us properties until it is done */
write_magic(control, fd_in, fd_out);
if (STDOUT)
dump_tmpoutfile(control, fd_out);
if (unlikely(close(fd_in) || close(fd_out)))
fatal("Failed to close files\n");
if (STDOUT) {
/* Delete temporary files generated for testing or faking stdout */
if (unlikely(unlink(control->outfile)))
fatal("Failed to unlink tmpfile: %s\n", strerror(errno));
}
if (!KEEP_FILES) {
if (unlikely(unlink(control->infile)))
fatal("Failed to unlink %s: %s\n", control->infile, strerror(errno));
}
free(control->outfile);
}

115
lrzip.h Normal file
View file

@ -0,0 +1,115 @@
/*
Copyright (C) 2006-2011 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef LRZIP_H
#define LRZIP_H
#define LRZIP_MAJOR_VERSION 0
#define LRZIP_MINOR_VERSION 5
#define LRZIP_MINOR_SUBVERSION 70
#define NUM_STREAMS 2
#define STREAM_BUFSIZE (1024 * 1024 * 10)
#include "config.h"
#include <stdint.h>
#ifndef uchar
#define uchar unsigned char
#endif
#ifndef int32
#if (SIZEOF_INT == 4)
#define int32 int
#elif (SIZEOF_LONG == 4)
#define int32 long
#elif (SIZEOF_SHORT == 4)
#define int32 short
#endif
#endif
#ifndef int16
#if (SIZEOF_INT == 2)
#define int16 int
#elif (SIZEOF_SHORT == 2)
#define int16 short
#endif
#endif
#ifndef uint32
#define uint32 unsigned int32
#endif
#ifndef uint16
#define uint16 unsigned int16
#endif
#ifndef MIN
#define MIN(a, b) ((a) < (b)? (a): (b))
#endif
#ifndef MAX
#define MAX(a, b) ((a) > (b)? (a): (b))
#endif
#if !HAVE_STRERROR
extern char *sys_errlist[];
#define strerror(i) sys_errlist[i]
#endif
#ifndef HAVE_ERRNO_H
extern int errno;
#endif
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
typedef long long int i64;
typedef uint16_t u16;
typedef uint32_t u32;
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN)
#define PROCESSORS (1)
#else
#define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
#endif
#ifdef _SC_PAGE_SIZE
#define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
#else
#define PAGE_SIZE (4096)
#endif
typedef struct rzip_control rzip_control;
void write_magic(rzip_control *control, int fd_in, int fd_out);
void read_magic(rzip_control *control, int fd_in, i64 *expected_size);
void preserve_perms(rzip_control *control, int fd_in, int fd_out);
int open_tmpoutfile(rzip_control *control);
void dump_tmpoutfile(rzip_control *control, int fd_out);
int open_tmpinfile(rzip_control *control);
void read_tmpinfile(rzip_control *control, int fd_in);
void decompress_file(rzip_control *control);
void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head);
void get_fileinfo(rzip_control *control);
void compress_file(rzip_control *control);
#endif

47
m4/ac_attribute.m4 Normal file
View file

@ -0,0 +1,47 @@
dnl Copyright (C) 2004-2008 Kim Woelders
dnl Copyright (C) 2008 Vincent Torri <vtorri at univ-evry dot fr>
dnl That code is public domain and can be freely used or copied.
dnl Originally snatched from somewhere...
dnl Macro for checking if the compiler supports __attribute__
dnl Usage: AC_C___ATTRIBUTE__
dnl call AC_DEFINE for HAVE___ATTRIBUTE__ and __UNUSED__
dnl if the compiler supports __attribute__, HAVE___ATTRIBUTE__ is
dnl defined to 1 and __UNUSED__ is defined to __attribute__((unused))
dnl otherwise, HAVE___ATTRIBUTE__ is not defined and __UNUSED__ is
dnl defined to nothing.
AC_DEFUN([AC_C___ATTRIBUTE__],
[
AC_MSG_CHECKING([for __attribute__])
AC_CACHE_VAL([ac_cv___attribute__],
[AC_TRY_COMPILE(
[
#include <stdlib.h>
int func(int x);
int foo(int x __attribute__ ((unused)))
{
exit(1);
}
],
[],
[ac_cv___attribute__="yes"],
[ac_cv___attribute__="no"]
)])
AC_MSG_RESULT($ac_cv___attribute__)
if test "x${ac_cv___attribute__}" = "xyes" ; then
AC_DEFINE([HAVE___ATTRIBUTE__], [1], [Define to 1 if your compiler has __attribute__])
AC_DEFINE([__UNUSED__], [__attribute__((unused))], [Macro declaring a function argument to be unused])
else
AC_DEFINE([__UNUSED__], [], [Macro declaring a function argument to be unused])
fi
])
dnl End of ac_attribute.m4

707
main.c
View file

@ -17,7 +17,55 @@
along with this program. If not, see <http://www.gnu.org/licenses/>. along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
/* lrzip compression - main program */ /* lrzip compression - main program */
#define MAIN_C
#include "rzip.h" #include "rzip.h"
/* main() defines, different from liblrzip defines */
#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS)
#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA))
#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS)
#define KEEP_FILES (control.flags & FLAG_KEEP_FILES)
#define TEST_ONLY (control.flags & FLAG_TEST_ONLY)
#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE)
#define DECOMPRESS (control.flags & FLAG_DECOMPRESS)
#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS)
#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS)
#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS)
#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS)
#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS)
#define VERBOSE (control.flags & FLAG_VERBOSE)
#define VERBOSITY (control.flags & FLAG_VERBOSITY)
#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX)
#define STDIN (control.flags & FLAG_STDIN)
#define STDOUT (control.flags & FLAG_STDOUT)
#define INFO (control.flags & FLAG_INFO)
#define UNLIMITED (control.flags & FLAG_UNLIMITED)
#define HASH_CHECK (control.flags & FLAG_HASH)
#define HAS_MD5 (control.flags & FLAG_MD5)
#define CHECK_FILE (control.flags & FLAG_CHECK)
#define KEEP_BROKEN (control.flags & FLAG_KEEP_BROKEN)
#define LZO_TEST (control.flags & FLAG_THRESHOLD)
#define print_output(format, args...) do {\
fprintf(control.msgout, format, ##args); \
fflush(control.msgout); \
} while (0)
#define print_progress(format, args...) do {\
if (SHOW_PROGRESS) \
print_output(format, ##args); \
} while (0)
#define print_verbose(format, args...) do {\
if (VERBOSE) \
print_output(format, ##args); \
} while (0)
#define print_maxverbose(format, args...) do {\
if (MAX_VERBOSE) \
print_output(format, ##args); \
} while (0)
struct rzip_control control; struct rzip_control control;
@ -66,652 +114,6 @@ static void usage(void)
} }
static void write_magic(int fd_in, int fd_out)
{
struct stat st;
char magic[24];
int i;
memset(magic, 0, sizeof(magic));
strcpy(magic, "LRZI");
magic[4] = LRZIP_MAJOR_VERSION;
magic[5] = LRZIP_MINOR_VERSION;
if (unlikely(fstat(fd_in, &st)))
fatal("bad magic file descriptor!?\n");
memcpy(&magic[6], &control.st_size, 8);
/* save LZMA compression flags */
if (LZMA_COMPRESS) {
for (i = 0; i < 5; i++)
magic[i + 16] = (char)control.lzma_properties[i];
}
/* This is a flag that the archive contains an md5 sum at the end
* which can be used as an integrity check instead of crc check.
* crc is still stored for compatibility with 0.5 versions.
*/
magic[21] = 1;
if (unlikely(lseek(fd_out, 0, SEEK_SET)))
fatal("Failed to seek to BOF to write Magic Header\n");
if (unlikely(write(fd_out, magic, sizeof(magic)) != sizeof(magic)))
fatal("Failed to write magic header\n");
}
static void read_magic(int fd_in, i64 *expected_size)
{
char magic[24];
uint32_t v;
int md5, i;
if (unlikely(read(fd_in, magic, sizeof(magic)) != sizeof(magic)))
fatal("Failed to read magic header\n");
*expected_size = 0;
if (unlikely(strncmp(magic, "LRZI", 4)))
failure("Not an lrzip file\n");
memcpy(&control.major_version, &magic[4], 1);
memcpy(&control.minor_version, &magic[5], 1);
/* Support the convoluted way we described size in versions < 0.40 */
if (control.major_version == 0 && control.minor_version < 4) {
memcpy(&v, &magic[6], 4);
*expected_size = ntohl(v);
memcpy(&v, &magic[10], 4);
*expected_size |= ((i64)ntohl(v)) << 32;
} else
memcpy(expected_size, &magic[6], 8);
/* restore LZMA compression flags only if stored */
if ((int) magic[16]) {
for (i = 0; i < 5; i++)
control.lzma_properties[i] = magic[i + 16];
}
/* Whether this archive contains md5 data at the end or not */
md5 = magic[21];
if (md5 == 1)
control.flags |= FLAG_MD5;
print_verbose("Detected lrzip version %d.%d file.\n", control.major_version, control.minor_version);
if (control.major_version > LRZIP_MAJOR_VERSION ||
(control.major_version == LRZIP_MAJOR_VERSION && control.minor_version > LRZIP_MINOR_VERSION))
print_output("Attempting to work with file produced by newer lrzip version %d.%d file.\n", control.major_version, control.minor_version);
}
/* preserve ownership and permissions where possible */
static void preserve_perms(int fd_in, int fd_out)
{
struct stat st;
if (unlikely(fstat(fd_in, &st)))
fatal("Failed to fstat input file\n");
if (unlikely(fchmod(fd_out, (st.st_mode & 0777))))
print_err("Warning, unable to set permissions on %s\n", control.outfile);
/* chown fail is not fatal */
if (unlikely(fchown(fd_out, st.st_uid, st.st_gid)))
print_err("Warning, unable to set owner on %s\n", control.outfile);
}
/* Open a temporary outputfile to emulate stdout */
static int open_tmpoutfile(void)
{
int fd_out;
if (STDOUT && !TEST_ONLY)
print_verbose("Outputting to stdout.\n");
if (control.tmpdir) {
control.outfile = realloc(NULL, strlen(control.tmpdir) + 16);
if (unlikely(!control.outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control.outfile, control.tmpdir);
strcat(control.outfile, "lrzipout.XXXXXX");
} else {
control.outfile = realloc(NULL, 16);
if (unlikely(!control.outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control.outfile, "lrzipout.XXXXXX");
}
fd_out = mkstemp(control.outfile);
if (unlikely(fd_out == -1))
fatal("Failed to create out tmpfile: %s\n", control.outfile);
return fd_out;
}
/* Dump temporary outputfile to perform stdout */
void dump_tmpoutfile(int fd_out)
{
FILE *tmpoutfp;
int tmpchar;
/* flush anything not yet in the temporary file */
fsync(fd_out);
tmpoutfp = fdopen(fd_out, "r");
if (unlikely(tmpoutfp == NULL))
fatal("Failed to fdopen out tmpfile\n");
rewind(tmpoutfp);
if (!TEST_ONLY) {
print_verbose("Dumping temporary file to stdout.\n");
while ((tmpchar = fgetc(tmpoutfp)) != EOF)
putchar(tmpchar);
fflush(stdout);
rewind(tmpoutfp);
}
if (unlikely(ftruncate(fd_out, 0)))
fatal("Failed to ftruncate fd_out in dump_tmpoutfile\n");
}
/* Open a temporary inputfile to perform stdin decompression */
static int open_tmpinfile(void)
{
int fd_in;
if (control.tmpdir) {
control.infile = malloc(strlen(control.tmpdir) + 15);
if (unlikely(!control.infile))
fatal("Failed to allocate infile name\n");
strcpy(control.infile, control.tmpdir);
strcat(control.infile, "lrzipin.XXXXXX");
} else {
control.infile = malloc(15);
if (unlikely(!control.infile))
fatal("Failed to allocate infile name\n");
strcpy(control.infile, "lrzipin.XXXXXX");
}
fd_in = mkstemp(control.infile);
if (unlikely(fd_in == -1))
fatal("Failed to create in tmpfile: %s\n", control.infile);
/* Unlink temporary file immediately to minimise chance of files left
* lying around in cases of failure. */
if (unlikely(unlink(control.infile)))
fatal("Failed to unlink tmpfile: %s\n", control.infile);
return fd_in;
}
/* Read data from stdin into temporary inputfile */
static void read_tmpinfile(int fd_in)
{
FILE *tmpinfp;
int tmpchar;
if (control.flags & FLAG_SHOW_PROGRESS)
fprintf(control.msgout, "Copying from stdin.\n");
tmpinfp = fdopen(fd_in, "w+");
if (unlikely(tmpinfp == NULL))
fatal("Failed to fdopen in tmpfile\n");
while ((tmpchar = getchar()) != EOF)
fputc(tmpchar, tmpinfp);
fflush(tmpinfp);
rewind(tmpinfp);
}
/*
decompress one file from the command line
*/
static void decompress_file(void)
{
char *tmp, *tmpoutfile, *infilecopy = NULL;
int fd_in, fd_out = -1, fd_hist = -1;
i64 expected_size, free_space;
struct statvfs fbuf;
if (!STDIN) {
if ((tmp = strrchr(control.infile, '.')) && strcmp(tmp,control.suffix)) {
/* make sure infile has an extension. If not, add it
* because manipulations may be made to input filename, set local ptr
*/
infilecopy = malloc(strlen(control.infile) + strlen(control.suffix) + 1);
if (unlikely(infilecopy == NULL))
fatal("Failed to allocate memory for infile suffix\n");
else {
strcpy(infilecopy, control.infile);
strcat(infilecopy, control.suffix);
}
} else
infilecopy = strdup(control.infile);
/* regardless, infilecopy has the input filename */
}
if (!STDOUT && !TEST_ONLY) {
/* if output name already set, use it */
if (control.outname) {
control.outfile = strdup(control.outname);
} else {
/* default output name from infilecopy
* test if outdir specified. If so, strip path from filename of
* infilecopy, then remove suffix.
*/
if (control.outdir && (tmp = strrchr(infilecopy, '/')))
tmpoutfile = strdup(tmp + 1);
else
tmpoutfile = strdup(infilecopy);
/* remove suffix to make outfile name */
if ((tmp = strrchr(tmpoutfile, '.')) && !strcmp(tmp, control.suffix))
*tmp='\0';
control.outfile = malloc((control.outdir == NULL? 0: strlen(control.outdir)) + strlen(tmpoutfile) + 1);
if (unlikely(!control.outfile))
fatal("Failed to allocate outfile name\n");
if (control.outdir) { /* prepend control.outdir */
strcpy(control.outfile, control.outdir);
strcat(control.outfile, tmpoutfile);
} else
strcpy(control.outfile, tmpoutfile);
free(tmpoutfile);
}
if (!STDOUT)
print_progress("Output filename is: %s...Decompressing...\n", control.outfile);
}
if (STDIN) {
fd_in = open_tmpinfile();
read_tmpinfile(fd_in);
} else {
fd_in = open(infilecopy, O_RDONLY);
if (unlikely(fd_in == -1)) {
fatal("Failed to open %s\n",infilecopy);
}
}
if (!(TEST_ONLY | STDOUT)) {
if (FORCE_REPLACE)
fd_out = open(control.outfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
else
fd_out = open(control.outfile, O_WRONLY | O_CREAT | O_EXCL, 0666);
if (unlikely(fd_out == -1)) {
/* We must ensure we don't delete a file that already
* exists just because we tried to create a new one */
control.flags |= FLAG_KEEP_BROKEN;
fatal("Failed to create %s\n", control.outfile);
}
preserve_perms(fd_in, fd_out);
} else
fd_out = open_tmpoutfile();
control.fd_out = fd_out;
read_magic(fd_in, &expected_size);
if (!STDOUT) {
/* Check if there's enough free space on the device chosen to fit the
* decompressed file. */
if (unlikely(fstatvfs(fd_out, &fbuf)))
fatal("Failed to fstatvfs in decompress_file\n");
free_space = fbuf.f_bsize * fbuf.f_bavail;
if (free_space < expected_size) {
if (FORCE_REPLACE)
print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n");
else
failure("Inadequate free space to decompress file, use -f to override.\n");
}
}
fd_hist = open(control.outfile, O_RDONLY);
if (unlikely(fd_hist == -1))
fatal("Failed to open history file %s\n", control.outfile);
/* Unlink temporary file as soon as possible */
if (unlikely((STDOUT || TEST_ONLY) && unlink(control.outfile)))
fatal("Failed to unlink tmpfile: %s\n", control.outfile);
if (NO_MD5)
print_verbose("Not performing MD5 hash check\n");
if (HAS_MD5)
print_verbose("MD5 ");
else
print_verbose("CRC32 ");
print_verbose("being used for integrity testing.\n");
print_progress("Decompressing...");
runzip_fd(fd_in, fd_out, fd_hist, expected_size);
if (STDOUT)
dump_tmpoutfile(fd_out);
/* if we get here, no fatal errors during decompression */
print_progress("\r");
if (!(STDOUT | TEST_ONLY))
print_output("Output filename is: %s: ", control.outfile);
print_progress("[OK] - %lld bytes \n", expected_size);
if (unlikely(close(fd_hist) || close(fd_out)))
fatal("Failed to close files\n");
close(fd_in);
if (!KEEP_FILES) {
if (unlikely(unlink(control.infile)))
fatal("Failed to unlink %s\n", infilecopy);
}
free(control.outfile);
free(infilecopy);
}
static void get_header_info(int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head)
{
if (unlikely(read(fd_in, ctype, 1) != 1))
fatal("Failed to read in get_header_info\n");
if (control.major_version == 0 && control.minor_version < 4) {
u32 c_len32, u_len32, last_head32;
if (unlikely(read(fd_in, &c_len32, 4) != 4))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, &u_len32, 4) != 4))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, &last_head32, 4) != 4))
fatal("Failed to read in get_header_info");
*c_len = c_len32;
*u_len = u_len32;
*last_head = last_head32;
} else {
if (unlikely(read(fd_in, c_len, 8) != 8))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, u_len, 8) != 8))
fatal("Failed to read in get_header_info");
if (unlikely(read(fd_in, last_head, 8) != 8))
fatal("Failed to read_i64 in get_header_info");
}
}
static void get_fileinfo(void)
{
i64 expected_size, infile_size;
int seekspot, fd_in;
long double cratio;
uchar ctype = 0;
struct stat st;
char *tmp, *infilecopy = NULL;
if (!STDIN) {
if ((tmp = strrchr(control.infile, '.')) && strcmp(tmp,control.suffix)) {
infilecopy = malloc(strlen(control.infile) + strlen(control.suffix) + 1);
if (unlikely(infilecopy == NULL))
fatal("Failed to allocate memory for infile suffix\n");
else {
strcpy(infilecopy, control.infile);
strcat(infilecopy, control.suffix);
}
} else
infilecopy = strdup(control.infile);
}
if (STDIN)
fd_in = 0;
else {
fd_in = open(infilecopy, O_RDONLY);
if (unlikely(fd_in == -1))
fatal("Failed to open %s\n", infilecopy);
}
/* Get file size */
if (unlikely(fstat(fd_in, &st)))
fatal("bad magic file descriptor!?\n");
memcpy(&infile_size, &st.st_size, 8);
/* Get decompressed size */
read_magic(fd_in, &expected_size);
/* Version < 0.4 had different file format */
if (control.major_version == 0 && control.minor_version < 4)
seekspot = 50;
else if (control.major_version == 0 && control.minor_version == 4)
seekspot = 74;
else
seekspot = 75;
if (unlikely(lseek(fd_in, seekspot, SEEK_SET) == -1))
fatal("Failed to lseek in get_fileinfo\n");
/* Read the compression type of the first block. It's possible that
not all blocks are compressed so this may not be accurate.
*/
if (unlikely(read(fd_in, &ctype, 1) != 1))
fatal("Failed to read in get_fileinfo\n");
cratio = (long double)expected_size / (long double)infile_size;
print_output("%s:\nlrzip version: %d.%d file\n", infilecopy, control.major_version, control.minor_version);
print_output("Compression: ");
if (ctype == CTYPE_NONE)
print_output("rzip alone\n");
else if (ctype == CTYPE_BZIP2)
print_output("rzip + bzip2\n");
else if (ctype == CTYPE_LZO)
print_output("rzip + lzo\n");
else if (ctype == CTYPE_LZMA)
print_output("rzip + lzma\n");
else if (ctype == CTYPE_GZIP)
print_output("rzip + gzip\n");
else if (ctype == CTYPE_ZPAQ)
print_output("rzip + zpaq\n");
else
print_output("Dunno wtf\n");
print_output("Decompressed file size: %llu\n", expected_size);
print_output("Compressed file size: %llu\n", infile_size);
print_output("Compression ratio: %.3Lf\n", cratio);
if (HAS_MD5) {
char md5_stored[MD5_DIGEST_SIZE];
int i;
print_output("MD5 used for integrity testing\n");
if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1)
fatal("Failed to seek to md5 data in runzip_fd\n");
if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
fatal("Failed to read md5 data in runzip_fd\n");
print_output("MD5: ");
for (i = 0; i < MD5_DIGEST_SIZE; i++)
print_output("%02x", md5_stored[i] & 0xFF);
print_output("\n");
} else
print_output("CRC32 used for integrity testing\n");
if (VERBOSE || MAX_VERBOSE) {
i64 u_len, c_len, last_head, utotal = 0, ctotal = 0, ofs = 25,
stream_head[2];
int header_length = 25, stream = 0, chunk = 0;
if (control.major_version == 0 && control.minor_version < 4)
header_length = 13;
next_chunk:
stream = 0;
stream_head[0] = 0;
stream_head[1] = stream_head[0] + header_length;
print_output("Rzip chunk %d:\n", ++chunk);
while (stream < NUM_STREAMS) {
int block = 1;
if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET)) == -1)
fatal("Failed to seek to header data in get_fileinfo\n");
get_header_info(fd_in, &ctype, &c_len, &u_len, &last_head);
print_output("Stream: %d\n", stream);
print_maxverbose("Offset: %lld\n", ofs);
print_output("Block\tComp\tPercent\tSize\n");
do {
i64 head_off;
if (unlikely(last_head + ofs > infile_size))
failure("Offset greater than archive size, likely corrupted/truncated archive.\n");
if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1)
fatal("Failed to seek to header data in get_fileinfo\n");
get_header_info(fd_in, &ctype, &c_len, &u_len, &last_head);
if (unlikely(last_head < 0 || c_len < 0 || u_len < 0))
failure("Entry negative, likely corrupted archive.\n");
print_output("%d\t", block);
if (ctype == CTYPE_NONE)
print_output("none");
else if (ctype == CTYPE_BZIP2)
print_output("bzip2");
else if (ctype == CTYPE_LZO)
print_output("lzo");
else if (ctype == CTYPE_LZMA)
print_output("lzma");
else if (ctype == CTYPE_GZIP)
print_output("gzip");
else if (ctype == CTYPE_ZPAQ)
print_output("zpaq");
else
print_output("Dunno wtf");
utotal += u_len;
ctotal += c_len;
print_output("\t%.1f%%\t%lld / %lld", (double)c_len / (double)(u_len / 100), c_len, u_len);
print_maxverbose("\tOffset: %lld\tHead: %lld", head_off, last_head);
print_output("\n");
block++;
} while (last_head);
++stream;
}
ofs = lseek(fd_in, 0, SEEK_CUR) + c_len;
/* Chunk byte entry */
if (control.major_version == 0 && control.minor_version > 4)
ofs++;
if (ofs < infile_size - MD5_DIGEST_SIZE)
goto next_chunk;
if (unlikely(ofs > infile_size))
failure("Offset greater than archive size, likely corrupted/truncated archive.\n");
print_output("Rzip compression: %.1f%% %lld / %lld\n",
(double)utotal / (double)(expected_size / 100),
utotal, expected_size);
print_output("Back end compression: %.1f%% %lld / %lld\n",
(double)ctotal / (double)(utotal / 100),
ctotal, utotal);
print_output("Overall compression: %.1f%% %lld / %lld\n",
(double)ctotal / (double)(expected_size / 100),
ctotal, expected_size);
}
if (unlikely(close(fd_in)))
fatal("Failed to close fd_in in get_fileinfo\n");
free(control.outfile);
free(infilecopy);
}
/*
compress one file from the command line
*/
static void compress_file(void)
{
const char *tmp, *tmpinfile; /* we're just using this as a proxy for control.infile.
* Spares a compiler warning
*/
int fd_in, fd_out;
char header[24];
memset(header, 0, sizeof(header));
if (!STDIN) {
/* is extension at end of infile? */
if ((tmp = strrchr(control.infile, '.')) && !strcmp(tmp, control.suffix)) {
print_err("%s: already has %s suffix. Skipping...\n", control.infile, control.suffix);
return;
}
fd_in = open(control.infile, O_RDONLY);
if (unlikely(fd_in == -1))
fatal("Failed to open %s\n", control.infile);
} else
fd_in = 0;
if (!STDOUT) {
if (control.outname) {
/* check if outname has control.suffix */
if (*(control.suffix) == '\0') /* suffix is empty string */
control.outfile = strdup(control.outname);
else if ((tmp=strrchr(control.outname, '.')) && strcmp(tmp, control.suffix)) {
control.outfile = malloc(strlen(control.outname) + strlen(control.suffix) + 1);
if (unlikely(!control.outfile))
fatal("Failed to allocate outfile name\n");
strcpy(control.outfile, control.outname);
strcat(control.outfile, control.suffix);
print_output("Suffix added to %s.\nFull pathname is: %s\n", control.outname, control.outfile);
} else /* no, already has suffix */
control.outfile = strdup(control.outname);
} else {
/* default output name from control.infile
* test if outdir specified. If so, strip path from filename of
* control.infile
*/
if (control.outdir && (tmp = strrchr(control.infile, '/')))
tmpinfile = tmp + 1;
else
tmpinfile = control.infile;
control.outfile = malloc((control.outdir == NULL? 0: strlen(control.outdir)) + strlen(tmpinfile) + strlen(control.suffix) + 1);
if (unlikely(!control.outfile))
fatal("Failed to allocate outfile name\n");
if (control.outdir) { /* prepend control.outdir */
strcpy(control.outfile, control.outdir);
strcat(control.outfile, tmpinfile);
} else
strcpy(control.outfile, tmpinfile);
strcat(control.outfile, control.suffix);
print_progress("Output filename is: %s\n", control.outfile);
}
if (FORCE_REPLACE)
fd_out = open(control.outfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
else
fd_out = open(control.outfile, O_WRONLY | O_CREAT | O_EXCL, 0666);
if (unlikely(fd_out == -1)) {
/* We must ensure we don't delete a file that already
* exists just because we tried to create a new one */
control.flags |= FLAG_KEEP_BROKEN;
fatal("Failed to create %s\n", control.outfile);
}
} else
fd_out = open_tmpoutfile();
control.fd_out = fd_out;
if (unlikely(STDOUT && unlink(control.outfile)))
fatal("Failed to unlink tmpfile: %s\n", control.outfile);
preserve_perms(fd_in, fd_out);
/* write zeroes to 24 bytes at beginning of file */
if (unlikely(write(fd_out, header, sizeof(header)) != sizeof(header)))
fatal("Cannot write file header\n");
rzip_fd(fd_in, fd_out);
/* write magic at end b/c lzma does not tell us properties until it is done */
write_magic(fd_in, fd_out);
if (STDOUT)
dump_tmpoutfile(fd_out);
if (unlikely(close(fd_in) || close(fd_out)))
fatal("Failed to close files\n");
if (!KEEP_FILES) {
if (unlikely(unlink(control.infile)))
fatal("Failed to unlink %s\n", control.infile);
}
free(control.outfile);
}
static void show_summary(void) static void show_summary(void)
{ {
/* OK, if verbosity set, print summary of options selected */ /* OK, if verbosity set, print summary of options selected */
@ -790,6 +192,7 @@ int main(int argc, char *argv[])
memset(&control, 0, sizeof(control)); memset(&control, 0, sizeof(control));
control.msgout = stderr; control.msgout = stderr;
register_outputfile(control.msgout);
control.flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; control.flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD;
control.suffix = ".lrz"; control.suffix = ".lrz";
control.outdir = NULL; control.outdir = NULL;
@ -1013,16 +416,20 @@ int main(int argc, char *argv[])
if (control.outname && (strcmp(control.outname, "-") == 0)) { if (control.outname && (strcmp(control.outname, "-") == 0)) {
control.flags |= FLAG_STDOUT; control.flags |= FLAG_STDOUT;
control.msgout = stderr; control.msgout = stderr;
register_outputfile(control.msgout);
} }
/* If we're using stdin and no output filename, use stdout */ /* If we're using stdin and no output filename, use stdout */
if (STDIN && !control.outname) { if (STDIN && !control.outname) {
control.flags |= FLAG_STDOUT; control.flags |= FLAG_STDOUT;
control.msgout = stderr; control.msgout = stderr;
register_outputfile(control.msgout);
} }
if (!STDOUT) if (!STDOUT) {
control.msgout = stdout; control.msgout = stdout;
register_outputfile(control.msgout);
}
/* Implement signal handler only once flags are set */ /* Implement signal handler only once flags are set */
handler.sa_handler = &sighandler; handler.sa_handler = &sighandler;
sigaction(SIGTERM, &handler, 0); sigaction(SIGTERM, &handler, 0);
@ -1056,11 +463,11 @@ int main(int argc, char *argv[])
gettimeofday(&start_time, NULL); gettimeofday(&start_time, NULL);
if (control.flags & (FLAG_DECOMPRESS | FLAG_TEST_ONLY)) if (control.flags & (FLAG_DECOMPRESS | FLAG_TEST_ONLY))
decompress_file(); decompress_file(&control);
else if (INFO) else if (INFO)
get_fileinfo(); get_fileinfo(&control);
else else
compress_file(); compress_file(&control);
/* compute total time */ /* compute total time */
gettimeofday(&end_time, NULL); gettimeofday(&end_time, NULL);

58
runzip.c
View file

@ -19,26 +19,26 @@
#include "rzip.h" #include "rzip.h"
static inline uchar read_u8(void *ss, int stream) static inline uchar read_u8(rzip_control *control, void *ss, int stream)
{ {
uchar b; uchar b;
if (unlikely(read_stream(ss, stream, &b, 1) != 1)) if (unlikely(read_stream(control, ss, stream, &b, 1) != 1))
fatal("Stream read u8 failed\n"); fatal("Stream read u8 failed\n");
return b; return b;
} }
static inline u32 read_u32(void *ss, int stream) static inline u32 read_u32(rzip_control *control, void *ss, int stream)
{ {
u32 ret; u32 ret;
if (unlikely(read_stream(ss, stream, (uchar *)&ret, 4) != 4)) if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4))
fatal("Stream read u32 failed\n"); fatal("Stream read u32 failed\n");
return ret; return ret;
} }
/* Read a variable length of chars dependant on how big the chunk was */ /* Read a variable length of chars dependant on how big the chunk was */
static inline i64 read_vchars(void *ss, int stream, int length) static inline i64 read_vchars(rzip_control *control, void *ss, int stream, int length)
{ {
int bytes; int bytes;
i64 s = 0; i64 s = 0;
@ -46,24 +46,24 @@ static inline i64 read_vchars(void *ss, int stream, int length)
for (bytes = 0; bytes < length; bytes++) { for (bytes = 0; bytes < length; bytes++) {
int bits = bytes * 8; int bits = bytes * 8;
uchar sb = read_u8(ss, stream); uchar sb = read_u8(control, ss, stream);
s |= (i64)sb << bits; s |= (i64)sb << bits;
} }
return s; return s;
} }
static i64 read_header(void *ss, uchar *head) static i64 read_header(rzip_control *control, void *ss, uchar *head)
{ {
int chunk_bytes = 2; int chunk_bytes = 2;
/* All chunks were unnecessarily encoded 8 bytes wide version 0.4x */ /* All chunks were unnecessarily encoded 8 bytes wide version 0.4x */
if (control.major_version == 0 && control.minor_version == 4) if (control->major_version == 0 && control->minor_version == 4)
chunk_bytes = 8; chunk_bytes = 8;
*head = read_u8(ss, 0); *head = read_u8(control, ss, 0);
return read_vchars(ss, 0, chunk_bytes); return read_vchars(control, ss, 0, chunk_bytes);
} }
static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum) static i64 unzip_literal(rzip_control *control, void *ss, i64 len, int fd_out, uint32 *cksum)
{ {
i64 stream_read; i64 stream_read;
uchar *buf; uchar *buf;
@ -75,7 +75,7 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum)
if (unlikely(!buf)) if (unlikely(!buf))
fatal("Failed to malloc literal buffer of size %lld\n", len); fatal("Failed to malloc literal buffer of size %lld\n", len);
stream_read = read_stream(ss, 1, buf, len); stream_read = read_stream(control, ss, 1, buf, len);
if (unlikely(stream_read == -1 )) if (unlikely(stream_read == -1 ))
fatal("Failed to read_stream in unzip_literal\n"); fatal("Failed to read_stream in unzip_literal\n");
@ -85,13 +85,13 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum)
if (!HAS_MD5) if (!HAS_MD5)
*cksum = CrcUpdate(*cksum, buf, stream_read); *cksum = CrcUpdate(*cksum, buf, stream_read);
if (!NO_MD5) if (!NO_MD5)
md5_process_bytes(buf, stream_read, &control.ctx); md5_process_bytes(buf, stream_read, &control->ctx);
free(buf); free(buf);
return stream_read; return stream_read;
} }
static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum, int chunk_bytes) static i64 unzip_match(rzip_control *control, void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum, int chunk_bytes)
{ {
i64 offset, n, total, cur_pos; i64 offset, n, total, cur_pos;
uchar *buf, *off_buf; uchar *buf, *off_buf;
@ -105,7 +105,7 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
fatal("Seek failed on out file in unzip_match.\n"); fatal("Seek failed on out file in unzip_match.\n");
/* Note the offset is in a different format v0.40+ */ /* Note the offset is in a different format v0.40+ */
offset = read_vchars(ss, 0, chunk_bytes); offset = read_vchars(control, ss, 0, chunk_bytes);
if (unlikely(lseek(fd_hist, cur_pos - offset, SEEK_SET) == -1)) if (unlikely(lseek(fd_hist, cur_pos - offset, SEEK_SET) == -1))
fatal("Seek failed by %d from %d on history file in unzip_match\n", fatal("Seek failed by %d from %d on history file in unzip_match\n",
offset, cur_pos); offset, cur_pos);
@ -127,7 +127,7 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
if (!HAS_MD5) if (!HAS_MD5)
*cksum = CrcUpdate(*cksum, off_buf, n); *cksum = CrcUpdate(*cksum, off_buf, n);
if (!NO_MD5) if (!NO_MD5)
md5_process_bytes(off_buf, n, &control.ctx); md5_process_bytes(off_buf, n, &control->ctx);
len -= n; len -= n;
off_buf += n; off_buf += n;
@ -142,7 +142,7 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
/* decompress a section of an open file. Call fatal() on error /* decompress a section of an open file. Call fatal() on error
return the number of bytes that have been retrieved return the number of bytes that have been retrieved
*/ */
static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally) static i64 runzip_chunk(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally)
{ {
uint32 good_cksum, cksum = 0; uint32 good_cksum, cksum = 0;
i64 len, ofs, total = 0; i64 len, ofs, total = 0;
@ -172,9 +172,9 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
/* Determine the chunk_byte width size. Versions < 0.4 used 4 /* Determine the chunk_byte width size. Versions < 0.4 used 4
* bytes for all offsets, version 0.4 used 8 bytes. Versions 0.5+ use * bytes for all offsets, version 0.4 used 8 bytes. Versions 0.5+ use
* a variable number of bytes depending on chunk size.*/ * a variable number of bytes depending on chunk size.*/
if (control.major_version == 0 && control.minor_version < 4) if (control->major_version == 0 && control->minor_version < 4)
chunk_bytes = 4; chunk_bytes = 4;
else if (control.major_version == 0 && control.minor_version == 4) else if (control->major_version == 0 && control->minor_version == 4)
chunk_bytes = 8; chunk_bytes = 8;
else { else {
/* Read in the stored chunk byte width from the file */ /* Read in the stored chunk byte width from the file */
@ -192,18 +192,18 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
if (fstat(fd_in, &st) || st.st_size - ofs == 0) if (fstat(fd_in, &st) || st.st_size - ofs == 0)
return 0; return 0;
ss = open_stream_in(fd_in, NUM_STREAMS); ss = open_stream_in(control, fd_in, NUM_STREAMS);
if (unlikely(!ss)) if (unlikely(!ss))
fatal("Failed to open_stream_in in runzip_chunk\n"); fatal("Failed to open_stream_in in runzip_chunk\n");
while ((len = read_header(ss, &head)) || head) { while ((len = read_header(control, ss, &head)) || head) {
switch (head) { switch (head) {
case 0: case 0:
total += unzip_literal(ss, len, fd_out, &cksum); total += unzip_literal(control, ss, len, fd_out, &cksum);
break; break;
default: default:
total += unzip_match(ss, len, fd_out, fd_hist, &cksum, chunk_bytes); total += unzip_match(control, ss, len, fd_out, fd_hist, &cksum, chunk_bytes);
break; break;
} }
p = 100 * ((double)(tally + total) / (double)expected_size); p = 100 * ((double)(tally + total) / (double)expected_size);
@ -216,7 +216,7 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
} }
if (!HAS_MD5) { if (!HAS_MD5) {
good_cksum = read_u32(ss, 0); good_cksum = read_u32(control, ss, 0);
if (unlikely(good_cksum != cksum)) if (unlikely(good_cksum != cksum))
failure("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum); failure("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum);
print_maxverbose("Checksum for block: 0x%08x\n", cksum); print_maxverbose("Checksum for block: 0x%08x\n", cksum);
@ -231,7 +231,7 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
/* Decompress an open file. Call fatal() on error /* Decompress an open file. Call fatal() on error
return the number of bytes that have been retrieved return the number of bytes that have been retrieved
*/ */
i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size) i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size)
{ {
char md5_resblock[MD5_DIGEST_SIZE]; char md5_resblock[MD5_DIGEST_SIZE];
char md5_stored[MD5_DIGEST_SIZE]; char md5_stored[MD5_DIGEST_SIZE];
@ -239,13 +239,13 @@ i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size)
i64 total = 0; i64 total = 0;
if (!NO_MD5) if (!NO_MD5)
md5_init_ctx (&control.ctx); md5_init_ctx (&control->ctx);
gettimeofday(&start,NULL); gettimeofday(&start,NULL);
while (total < expected_size) { while (total < expected_size) {
total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total); total += runzip_chunk(control, fd_in, fd_out, fd_hist, expected_size, total);
if (STDOUT) if (STDOUT)
dump_tmpoutfile(fd_out); dump_tmpoutfile(control, fd_out);
} }
gettimeofday(&end,NULL); gettimeofday(&end,NULL);
@ -255,7 +255,7 @@ i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size)
if (!NO_MD5) { if (!NO_MD5) {
int i,j; int i,j;
md5_finish_ctx (&control.ctx, md5_resblock); md5_finish_ctx (&control->ctx, md5_resblock);
if (HAS_MD5) { if (HAS_MD5) {
if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1) if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1)
fatal("Failed to seek to md5 data in runzip_fd\n"); fatal("Failed to seek to md5 data in runzip_fd\n");

194
rzip.c
View file

@ -100,7 +100,7 @@ struct sliding_buffer {
int fd; /* The fd of the mmap */ int fd; /* The fd of the mmap */
} sb; /* Sliding buffer */ } sb; /* Sliding buffer */
static void remap_low_sb(void) static void remap_low_sb(rzip_control *control)
{ {
i64 new_offset; i64 new_offset;
@ -117,14 +117,14 @@ static void remap_low_sb(void)
fatal("Failed to re mmap in remap_low_sb\n"); fatal("Failed to re mmap in remap_low_sb\n");
} }
static inline void remap_high_sb(i64 p) static inline void remap_high_sb(rzip_control *control, i64 p)
{ {
if (unlikely(munmap(sb.buf_high, sb.size_high))) if (unlikely(munmap(sb.buf_high, sb.size_high)))
fatal("Failed to munmap in remap_high_sb\n"); fatal("Failed to munmap in remap_high_sb\n");
sb.size_high = sb.high_length; /* In case we shrunk it when we hit the end of the file */ sb.size_high = sb.high_length; /* In case we shrunk it when we hit the end of the file */
sb.offset_high = p; sb.offset_high = p;
/* Make sure offset is rounded to page size of total offset */ /* Make sure offset is rounded to page size of total offset */
sb.offset_high -= (sb.offset_high + sb.orig_offset) % control.page_size; sb.offset_high -= (sb.offset_high + sb.orig_offset) % control->page_size;
if (unlikely(sb.offset_high + sb.size_high > sb.orig_size)) if (unlikely(sb.offset_high + sb.size_high > sb.orig_size))
sb.size_high = sb.orig_size - sb.offset_high; sb.size_high = sb.orig_size - sb.offset_high;
sb.buf_high = (uchar *)mmap(sb.buf_high, sb.size_high, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_high); sb.buf_high = (uchar *)mmap(sb.buf_high, sb.size_high, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_high);
@ -138,36 +138,36 @@ static inline void remap_high_sb(i64 p)
* it, and a 64k mmap block that slides up and down as is required for any * it, and a 64k mmap block that slides up and down as is required for any
* offsets outside the range of the lower one. This is much slower than mmap * offsets outside the range of the lower one. This is much slower than mmap
* but makes it possible to have unlimited sized compression windows. */ * but makes it possible to have unlimited sized compression windows. */
static uchar *get_sb(i64 p) static uchar *get_sb(rzip_control *control, i64 p)
{ {
i64 low_end = sb.offset_low + sb.size_low; i64 low_end = sb.offset_low + sb.size_low;
if (unlikely(sb.offset_search > low_end)) if (unlikely(sb.offset_search > low_end))
remap_low_sb(); remap_low_sb(control);
if (p >= sb.offset_low && p < low_end) if (p >= sb.offset_low && p < low_end)
return (sb.buf_low + p - sb.offset_low); return (sb.buf_low + p - sb.offset_low);
if (p >= sb.offset_high && p < (sb.offset_high + sb.size_high)) if (p >= sb.offset_high && p < (sb.offset_high + sb.size_high))
return (sb.buf_high + (p - sb.offset_high)); return (sb.buf_high + (p - sb.offset_high));
/* p is not within the low or high buffer range */ /* p is not within the low or high buffer range */
remap_high_sb(p); remap_high_sb(control, p);
return (sb.buf_high + (p - sb.offset_high)); return (sb.buf_high + (p - sb.offset_high));
} }
/* All put_u8/u32/vchars go to stream 0 */ /* All put_u8/u32/vchars go to stream 0 */
static inline void put_u8(void *ss, uchar b) static inline void put_u8(rzip_control *control, void *ss, uchar b)
{ {
if (unlikely(write_stream(ss, 0, &b, 1))) if (unlikely(write_stream(control, ss, 0, &b, 1)))
fatal("Failed to put_u8\n"); fatal("Failed to put_u8\n");
} }
static inline void put_u32(void *ss, uint32_t s) static inline void put_u32(rzip_control *control, void *ss, uint32_t s)
{ {
if (unlikely(write_stream(ss, 0, (uchar *)&s, 4))) if (unlikely(write_stream(control, ss, 0, (uchar *)&s, 4)))
fatal("Failed to put_u32\n"); fatal("Failed to put_u32\n");
} }
/* Put a variable length of bytes dependant on how big the chunk is */ /* Put a variable length of bytes dependant on how big the chunk is */
static inline void put_vchars(void *ss, i64 s, int length) static inline void put_vchars(rzip_control *control, void *ss, i64 s, int length)
{ {
int bytes; int bytes;
@ -175,17 +175,17 @@ static inline void put_vchars(void *ss, i64 s, int length)
int bits = bytes * 8; int bits = bytes * 8;
uchar sb = (s >> bits) & (i64)0XFF; uchar sb = (s >> bits) & (i64)0XFF;
put_u8(ss, sb); put_u8(control, ss, sb);
} }
} }
static void put_header(void *ss, uchar head, i64 len) static void put_header(rzip_control *control, void *ss, uchar head, i64 len)
{ {
put_u8(ss, head); put_u8(control, ss, head);
put_vchars(ss, len, 2); put_vchars(control, ss, len, 2);
} }
static void put_match(struct rzip_state *st, i64 p, i64 offset, i64 len) static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 offset, i64 len)
{ {
do { do {
i64 ofs; i64 ofs;
@ -194,8 +194,8 @@ static void put_match(struct rzip_state *st, i64 p, i64 offset, i64 len)
n = 0xFFFF; n = 0xFFFF;
ofs = (p - offset); ofs = (p - offset);
put_header(st->ss, 1, n); put_header(control, st->ss, 1, n);
put_vchars(st->ss, ofs, st->chunk_bytes); put_vchars(control, st->ss, ofs, st->chunk_bytes);
st->stats.matches++; st->stats.matches++;
st->stats.match_bytes += n; st->stats.match_bytes += n;
@ -206,7 +206,7 @@ static void put_match(struct rzip_state *st, i64 p, i64 offset, i64 len)
} }
/* write some data to a stream mmap encoded. Return -1 on failure */ /* write some data to a stream mmap encoded. Return -1 on failure */
int write_sbstream(void *ss, int stream, i64 p, i64 len) int write_sbstream(rzip_control *control, void *ss, int stream, i64 p, i64 len)
{ {
struct stream_info *sinfo = ss; struct stream_info *sinfo = ss;
@ -217,19 +217,19 @@ int write_sbstream(void *ss, int stream, i64 p, i64 len)
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
memcpy(sinfo->s[stream].buf + sinfo->s[stream].buflen + i, memcpy(sinfo->s[stream].buf + sinfo->s[stream].buflen + i,
get_sb(p + i), 1); get_sb(control, p + i), 1);
} }
sinfo->s[stream].buflen += n; sinfo->s[stream].buflen += n;
p += n; p += n;
len -= n; len -= n;
if (sinfo->s[stream].buflen == sinfo->bufsize) if (sinfo->s[stream].buflen == sinfo->bufsize)
flush_buffer(sinfo, stream); flush_buffer(control, sinfo, stream);
} }
return 0; return 0;
} }
static void put_literal(struct rzip_state *st, i64 last, i64 p) static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, i64 p)
{ {
do { do {
i64 len = p - last; i64 len = p - last;
@ -239,9 +239,9 @@ static void put_literal(struct rzip_state *st, i64 last, i64 p)
st->stats.literals++; st->stats.literals++;
st->stats.literal_bytes += len; st->stats.literal_bytes += len;
put_header(st->ss, 0, len); put_header(control, st->ss, 0, len);
if (unlikely(len && write_sbstream(st->ss, 1, last, len))) if (unlikely(len && write_sbstream(control, st->ss, 1, last, len)))
fatal("Failed to write_stream in put_literal\n"); fatal("Failed to write_stream in put_literal\n");
last += len; last += len;
} while (p > last); } while (p > last);
@ -337,7 +337,7 @@ static void insert_hash(struct rzip_state *st, tag t, i64 offset)
/* Eliminate one hash entry with minimum number of lower bits set. /* Eliminate one hash entry with minimum number of lower bits set.
Returns tag requirement for any new entries. */ Returns tag requirement for any new entries. */
static tag clean_one_from_hash(struct rzip_state *st) static tag clean_one_from_hash(rzip_control *control, struct rzip_state *st)
{ {
tag better_than_min; tag better_than_min;
@ -364,24 +364,24 @@ again:
goto again; goto again;
} }
static inline tag next_tag(struct rzip_state *st, i64 p, tag t) static inline tag next_tag(rzip_control *control, struct rzip_state *st, i64 p, tag t)
{ {
t ^= st->hash_index[*get_sb(p - 1)]; t ^= st->hash_index[*get_sb(control, p - 1)];
t ^= st->hash_index[*get_sb(p + MINIMUM_MATCH - 1)]; t ^= st->hash_index[*get_sb(control, p + MINIMUM_MATCH - 1)];
return t; return t;
} }
static inline tag full_tag(struct rzip_state *st, i64 p) static inline tag full_tag(rzip_control *control, struct rzip_state *st, i64 p)
{ {
tag ret = 0; tag ret = 0;
int i; int i;
for (i = 0; i < MINIMUM_MATCH; i++) for (i = 0; i < MINIMUM_MATCH; i++)
ret ^= st->hash_index[*get_sb(p + i)]; ret ^= st->hash_index[*get_sb(control, p + i)];
return ret; return ret;
} }
static inline i64 match_len(struct rzip_state *st, i64 p0, i64 op, i64 end, static inline i64 match_len(rzip_control *control, struct rzip_state *st, i64 p0, i64 op, i64 end,
i64 *rev) i64 *rev)
{ {
i64 p = p0; i64 p = p0;
@ -390,7 +390,7 @@ static inline i64 match_len(struct rzip_state *st, i64 p0, i64 op, i64 end,
if (op >= p0) if (op >= p0)
return 0; return 0;
while ((*get_sb(p) == *get_sb(op)) && (p < end)) { while ((*get_sb(control, p) == *get_sb(control, op)) && (p < end)) {
p++; p++;
op++; op++;
} }
@ -403,7 +403,7 @@ static inline i64 match_len(struct rzip_state *st, i64 p0, i64 op, i64 end,
if (end < st->last_match) if (end < st->last_match)
end = st->last_match; end = st->last_match;
while (p > end && op > 0 && *get_sb(op - 1) == *get_sb(p - 1)) { while (p > end && op > 0 && *get_sb(control, op - 1) == *get_sb(control, p - 1)) {
op--; op--;
p--; p--;
} }
@ -417,7 +417,7 @@ static inline i64 match_len(struct rzip_state *st, i64 p0, i64 op, i64 end,
return len; return len;
} }
static i64 find_best_match(struct rzip_state *st, tag t, i64 p, i64 end, static i64 find_best_match(rzip_control *control, struct rzip_state *st, tag t, i64 p, i64 end,
i64 *offset, i64 *reverse) i64 *offset, i64 *reverse)
{ {
i64 length = 0; i64 length = 0;
@ -434,7 +434,7 @@ static i64 find_best_match(struct rzip_state *st, tag t, i64 p, i64 end,
i64 mlen; i64 mlen;
if (t == st->hash_table[h].t) { if (t == st->hash_table[h].t) {
mlen = match_len(st, p, st->hash_table[h].offset, end, mlen = match_len(control, st, p, st->hash_table[h].offset, end,
&rev); &rev);
if (mlen) if (mlen)
@ -457,7 +457,7 @@ static i64 find_best_match(struct rzip_state *st, tag t, i64 p, i64 end,
return length; return length;
} }
static void show_distrib(struct rzip_state *st) static void show_distrib(rzip_control *control, struct rzip_state *st)
{ {
i64 primary = 0; i64 primary = 0;
i64 total = 0; i64 total = 0;
@ -478,7 +478,7 @@ static void show_distrib(struct rzip_state *st)
primary*100.0/total); primary*100.0/total);
} }
static void hash_search(struct rzip_state *st, double pct_base, double pct_multiple) static void hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple)
{ {
i64 cksum_limit = 0, p, end; i64 cksum_limit = 0, p, end;
tag t = 0; tag t = 0;
@ -520,7 +520,7 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
current.p = p; current.p = p;
current.ofs = 0; current.ofs = 0;
t = full_tag(st, p); t = full_tag(control, st, p);
while (p < end) { while (p < end) {
int lastpct = 0, last_chunkpct = 0; int lastpct = 0, last_chunkpct = 0;
@ -528,14 +528,14 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
p++; p++;
sb.offset_search = p; sb.offset_search = p;
t = next_tag(st, p, t); t = next_tag(control, st, p, t);
/* Don't look for a match if there are no tags with /* Don't look for a match if there are no tags with
this number of bits in the hash table. */ this number of bits in the hash table. */
if ((t & st->minimum_tag_mask) != st->minimum_tag_mask) if ((t & st->minimum_tag_mask) != st->minimum_tag_mask)
continue; continue;
mlen = find_best_match(st, t, p, end, &offset, &reverse); mlen = find_best_match(control, st, t, p, end, &offset, &reverse);
/* Only insert occasionally into hash. */ /* Only insert occasionally into hash. */
if ((t & tag_mask) == tag_mask) { if ((t & tag_mask) == tag_mask) {
@ -543,7 +543,7 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
st->hash_count++; st->hash_count++;
insert_hash(st, t, p); insert_hash(st, t, p);
if (st->hash_count > st->hash_limit) if (st->hash_count > st->hash_limit)
tag_mask = clean_one_from_hash(st); tag_mask = clean_one_from_hash(control, st);
} }
if (mlen > current.len) { if (mlen > current.len) {
@ -555,12 +555,12 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
if ((current.len >= GREAT_MATCH || p >= current.p + MINIMUM_MATCH) if ((current.len >= GREAT_MATCH || p >= current.p + MINIMUM_MATCH)
&& current.len >= MINIMUM_MATCH) { && current.len >= MINIMUM_MATCH) {
if (st->last_match < current.p) if (st->last_match < current.p)
put_literal(st, st->last_match, current.p); put_literal(control, st, st->last_match, current.p);
put_match(st, current.p, current.ofs, current.len); put_match(control, st, current.p, current.ofs, current.len);
st->last_match = current.p + current.len; st->last_match = current.p + current.len;
current.p = p = st->last_match; current.p = p = st->last_match;
current.len = 0; current.len = 0;
t = full_tag(st, p); t = full_tag(control, st, p);
} }
if (unlikely(p % 128 == 0)) { if (unlikely(p % 128 == 0)) {
@ -579,25 +579,25 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
} }
if (p > (i64)cksum_limit) { if (p > (i64)cksum_limit) {
i64 i, n = MIN(st->chunk_size - p, control.page_size); i64 i, n = MIN(st->chunk_size - p, control->page_size);
uchar *ckbuf = malloc(n); uchar *ckbuf = malloc(n);
if (unlikely(!ckbuf)) if (unlikely(!ckbuf))
fatal("Failed to malloc ckbuf in hash_search\n"); fatal("Failed to malloc ckbuf in hash_search\n");
for (i = 0; i < n; i++) for (i = 0; i < n; i++)
memcpy(ckbuf + i, get_sb(cksum_limit + i), 1); memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1);
st->cksum = CrcUpdate(st->cksum, ckbuf, n); st->cksum = CrcUpdate(st->cksum, ckbuf, n);
md5_process_bytes(ckbuf, n, &control.ctx); md5_process_bytes(ckbuf, n, &control->ctx);
cksum_limit += n; cksum_limit += n;
free(ckbuf); free(ckbuf);
} }
} }
if (MAX_VERBOSE) if (MAX_VERBOSE)
show_distrib(st); show_distrib(control, st);
if (st->last_match < st->chunk_size) if (st->last_match < st->chunk_size)
put_literal(st, st->last_match, st->chunk_size); put_literal(control, st, st->last_match, st->chunk_size);
if (st->chunk_size > cksum_limit) { if (st->chunk_size > cksum_limit) {
i64 i, n = st->chunk_size - cksum_limit; i64 i, n = st->chunk_size - cksum_limit;
@ -606,15 +606,15 @@ static void hash_search(struct rzip_state *st, double pct_base, double pct_multi
if (unlikely(!ckbuf)) if (unlikely(!ckbuf))
fatal("Failed to malloc ckbuf in hash_search\n"); fatal("Failed to malloc ckbuf in hash_search\n");
for (i = 0; i < n; i++) for (i = 0; i < n; i++)
memcpy(ckbuf + i, get_sb(cksum_limit + i), 1); memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1);
st->cksum = CrcUpdate(st->cksum, ckbuf, n); st->cksum = CrcUpdate(st->cksum, ckbuf, n);
md5_process_bytes(ckbuf, n, &control.ctx); md5_process_bytes(ckbuf, n, &control->ctx);
cksum_limit += n; cksum_limit += n;
free(ckbuf); free(ckbuf);
} }
put_literal(st, 0, 0); put_literal(control, st, 0, 0);
put_u32(st->ss, st->cksum); put_u32(control, st->ss, st->cksum);
} }
@ -639,7 +639,7 @@ static inline void *fake_mremap(void *old_address, size_t old_size, size_t new_s
* anonymous ram and reading stdin into it. It means the maximum ram * anonymous ram and reading stdin into it. It means the maximum ram
* we can use will be less but we will already have determined this in * we can use will be less but we will already have determined this in
* rzip_chunk */ * rzip_chunk */
static void mmap_stdin(uchar *buf, struct rzip_state *st) static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st)
{ {
i64 len = st->chunk_size; i64 len = st->chunk_size;
uchar *offset_buf = buf; uchar *offset_buf = buf;
@ -669,17 +669,17 @@ static void mmap_stdin(uchar *buf, struct rzip_state *st)
offset_buf += ret; offset_buf += ret;
len -= ret; len -= ret;
} }
control.st_size += total; control->st_size += total;
} }
static void init_sliding_mmap(struct rzip_state *st, int fd_in, i64 offset) static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in, i64 offset)
{ {
/* Initialise the high buffer */ /* Initialise the high buffer */
if (!STDIN) { if (!STDIN) {
sb.high_length = 65536; sb.high_length = 65536;
/* Round up to the next biggest page size */ /* Round up to the next biggest page size */
if (sb.high_length % control.page_size) if (sb.high_length % control->page_size)
sb.high_length += control.page_size - (sb.high_length % control.page_size); sb.high_length += control->page_size - (sb.high_length % control->page_size);
sb.buf_high = (uchar *)mmap(NULL, sb.high_length, PROT_READ, MAP_SHARED, fd_in, offset); sb.buf_high = (uchar *)mmap(NULL, sb.high_length, PROT_READ, MAP_SHARED, fd_in, offset);
if (unlikely(sb.buf_high == MAP_FAILED)) if (unlikely(sb.buf_high == MAP_FAILED))
fatal("Unable to mmap buf_high in init_sliding_mmap\n"); fatal("Unable to mmap buf_high in init_sliding_mmap\n");
@ -695,17 +695,17 @@ static void init_sliding_mmap(struct rzip_state *st, int fd_in, i64 offset)
/* compress a chunk of an open file. Assumes that the file is able to /* compress a chunk of an open file. Assumes that the file is able to
be mmap'd and is seekable */ be mmap'd and is seekable */
static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset, static void rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out, i64 offset,
double pct_base, double pct_multiple) double pct_base, double pct_multiple)
{ {
init_sliding_mmap(st, fd_in, offset); init_sliding_mmap(control, st, fd_in, offset);
st->ss = open_stream_out(fd_out, NUM_STREAMS, st->chunk_size, st->chunk_bytes); st->ss = open_stream_out(control, fd_out, NUM_STREAMS, st->chunk_size, st->chunk_bytes);
if (unlikely(!st->ss)) if (unlikely(!st->ss))
fatal("Failed to open streams in rzip_chunk\n"); fatal("Failed to open streams in rzip_chunk\n");
print_verbose("Beginning rzip pre-processing phase\n"); print_verbose("Beginning rzip pre-processing phase\n");
hash_search(st, pct_base, pct_multiple); hash_search(control, st, pct_base, pct_multiple);
/* unmap buffer before closing and reallocating streams */ /* unmap buffer before closing and reallocating streams */
if (unlikely(munmap(sb.buf_low, sb.size_low))) if (unlikely(munmap(sb.buf_low, sb.size_low)))
@ -715,7 +715,7 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
fatal("Failed to munmap in rzip_chunk\n"); fatal("Failed to munmap in rzip_chunk\n");
} }
if (unlikely(close_stream_out(st->ss))) if (unlikely(close_stream_out(control, st->ss)))
fatal("Failed to flush/close streams in rzip_chunk\n"); fatal("Failed to flush/close streams in rzip_chunk\n");
} }
@ -723,7 +723,7 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
const i64 two_gig = (1ull << 31) - 4096; const i64 two_gig = (1ull << 31) - 4096;
/* compress a whole file chunks at a time */ /* compress a whole file chunks at a time */
void rzip_fd(int fd_in, int fd_out) void rzip_fd(rzip_control *control, int fd_in, int fd_out)
{ {
/* add timers for ETA estimates /* add timers for ETA estimates
* Base it off the file size and number of iterations required * Base it off the file size and number of iterations required
@ -741,7 +741,7 @@ void rzip_fd(int fd_in, int fd_out)
double chunkmbs; double chunkmbs;
i64 free_space; i64 free_space;
md5_init_ctx (&control.ctx); md5_init_ctx (&control->ctx);
st = calloc(sizeof(*st), 1); st = calloc(sizeof(*st), 1);
if (unlikely(!st)) if (unlikely(!st))
@ -756,10 +756,10 @@ void rzip_fd(int fd_in, int fd_out)
fatal("Failed to stat fd_in in rzip_fd\n"); fatal("Failed to stat fd_in in rzip_fd\n");
if (!STDIN) { if (!STDIN) {
len = control.st_size = s.st_size; len = control->st_size = s.st_size;
print_verbose("File size: %lld\n", len); print_verbose("File size: %lld\n", len);
} else } else
control.st_size = 0; control->st_size = 0;
/* Check if there's enough free space on the device chosen to fit the /* Check if there's enough free space on the device chosen to fit the
* compressed file, based on the compressed file being as large as the * compressed file, based on the compressed file being as large as the
@ -767,7 +767,7 @@ void rzip_fd(int fd_in, int fd_out)
if (unlikely(fstatvfs(fd_out, &fbuf))) if (unlikely(fstatvfs(fd_out, &fbuf)))
fatal("Failed to fstatvfs in compress_file\n"); fatal("Failed to fstatvfs in compress_file\n");
free_space = fbuf.f_bsize * fbuf.f_bavail; free_space = fbuf.f_bsize * fbuf.f_bavail;
if (free_space < control.st_size) { if (free_space < control->st_size) {
if (FORCE_REPLACE) if (FORCE_REPLACE)
print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n");
else else
@ -778,35 +778,35 @@ void rzip_fd(int fd_in, int fd_out)
* allocate 1/3 of it to the main buffer and use a sliding mmap * allocate 1/3 of it to the main buffer and use a sliding mmap
* buffer to work on 2/3 ram size, leaving enough ram for the * buffer to work on 2/3 ram size, leaving enough ram for the
* compression backends */ * compression backends */
control.max_mmap = control.maxram; control->max_mmap = control->maxram;
/* On 32 bits we can have a big window with sliding mmap, but can /* On 32 bits we can have a big window with sliding mmap, but can
* not enable much per mmap/malloc */ * not enable much per mmap/malloc */
if (BITS32) if (BITS32)
control.max_mmap = MIN(control.max_mmap, two_gig); control->max_mmap = MIN(control->max_mmap, two_gig);
round_to_page(&control.max_mmap); round_to_page(&control->max_mmap);
/* Set maximum chunk size to 2/3 of ram if not unlimited or specified /* Set maximum chunk size to 2/3 of ram if not unlimited or specified
* by a control window. When it's smaller than the file size, round it * by a control window. When it's smaller than the file size, round it
* to page size for efficiency. */ * to page size for efficiency. */
if (UNLIMITED) if (UNLIMITED)
control.max_chunk = control.st_size; control->max_chunk = control->st_size;
else if (control.window) else if (control->window)
control.max_chunk = control.window * CHUNK_MULTIPLE; control->max_chunk = control->window * CHUNK_MULTIPLE;
else else
control.max_chunk = control.ramsize / 3 * 2; control->max_chunk = control->ramsize / 3 * 2;
control.max_mmap = MIN(control.max_mmap, control.max_chunk); control->max_mmap = MIN(control->max_mmap, control->max_chunk);
if (control.max_chunk < control.st_size) if (control->max_chunk < control->st_size)
round_to_page(&control.max_chunk); round_to_page(&control->max_chunk);
if (!STDIN) if (!STDIN)
st->chunk_size = MIN(control.max_chunk, len); st->chunk_size = MIN(control->max_chunk, len);
else else
st->chunk_size = control.max_mmap; st->chunk_size = control->max_mmap;
if (st->chunk_size < len) if (st->chunk_size < len)
round_to_page(&st->chunk_size); round_to_page(&st->chunk_size);
st->level = &levels[control.compression_level]; st->level = &levels[control->compression_level];
st->fd_in = fd_in; st->fd_in = fd_in;
st->fd_out = fd_out; st->fd_out = fd_out;
st->stdin_eof = 0; st->stdin_eof = 0;
@ -819,15 +819,15 @@ void rzip_fd(int fd_in, int fd_out)
last.tv_sec = last.tv_usec = 0; last.tv_sec = last.tv_usec = 0;
gettimeofday(&start, NULL); gettimeofday(&start, NULL);
prepare_streamout_threads(); prepare_streamout_threads(control);
while (len > 0 || (STDIN && !st->stdin_eof)) { while (len > 0 || (STDIN && !st->stdin_eof)) {
double pct_base, pct_multiple; double pct_base, pct_multiple;
i64 offset = s.st_size - len; i64 offset = s.st_size - len;
int bits = 8; int bits = 8;
st->chunk_size = control.max_chunk; st->chunk_size = control->max_chunk;
st->mmap_size = control.max_mmap; st->mmap_size = control->max_mmap;
if (!STDIN) { if (!STDIN) {
st->chunk_size = MIN(st->chunk_size, len); st->chunk_size = MIN(st->chunk_size, len);
st->mmap_size = MIN(st->mmap_size, len); st->mmap_size = MIN(st->mmap_size, len);
@ -840,7 +840,7 @@ retry:
/* Better to shrink the window to the largest size that works than fail */ /* Better to shrink the window to the largest size that works than fail */
if (sb.buf_low == MAP_FAILED) { if (sb.buf_low == MAP_FAILED) {
if (unlikely(errno != ENOMEM)) if (unlikely(errno != ENOMEM))
fatal("Failed to mmap %s\n", control.infile); fatal("Failed to mmap %s\n", control->infile);
st->mmap_size = st->mmap_size / 10 * 9; st->mmap_size = st->mmap_size / 10 * 9;
round_to_page(&st->mmap_size); round_to_page(&st->mmap_size);
if (unlikely(!st->mmap_size)) if (unlikely(!st->mmap_size))
@ -848,13 +848,13 @@ retry:
goto retry; goto retry;
} }
st->chunk_size = st->mmap_size; st->chunk_size = st->mmap_size;
mmap_stdin(sb.buf_low, st); mmap_stdin(control, sb.buf_low, st);
} else { } else {
/* NOTE The buf is saved here for !STDIN mode */ /* NOTE The buf is saved here for !STDIN mode */
sb.buf_low = (uchar *)mmap(sb.buf_low, st->mmap_size, PROT_READ, MAP_SHARED, fd_in, offset); sb.buf_low = (uchar *)mmap(sb.buf_low, st->mmap_size, PROT_READ, MAP_SHARED, fd_in, offset);
if (sb.buf_low == MAP_FAILED) { if (sb.buf_low == MAP_FAILED) {
if (unlikely(errno != ENOMEM)) if (unlikely(errno != ENOMEM))
fatal("Failed to mmap %s\n", control.infile); fatal("Failed to mmap %s\n", control->infile);
st->mmap_size = st->mmap_size / 10 * 9; st->mmap_size = st->mmap_size / 10 * 9;
round_to_page(&st->mmap_size); round_to_page(&st->mmap_size);
if (unlikely(!st->mmap_size)) if (unlikely(!st->mmap_size))
@ -866,7 +866,7 @@ retry:
} }
print_maxverbose("Succeeded in testing %lld sized mmap for rzip pre-processing\n", st->mmap_size); print_maxverbose("Succeeded in testing %lld sized mmap for rzip pre-processing\n", st->mmap_size);
if (st->chunk_size > control.ramsize) if (st->chunk_size > control->ramsize)
print_verbose("Compression window is larger than ram, will proceed with unlimited mode possibly much slower\n"); print_verbose("Compression window is larger than ram, will proceed with unlimited mode possibly much slower\n");
if (!passes && !STDIN) { if (!passes && !STDIN) {
@ -893,10 +893,10 @@ retry:
print_maxverbose("Byte width: %d\n", st->chunk_bytes); print_maxverbose("Byte width: %d\n", st->chunk_bytes);
if (STDIN) if (STDIN)
pct_base = (100.0 * -len) / control.st_size; pct_base = (100.0 * -len) / control->st_size;
else else
pct_base = (100.0 * (control.st_size - len)) / control.st_size; pct_base = (100.0 * (control->st_size - len)) / control->st_size;
pct_multiple = ((double)st->chunk_size) / control.st_size; pct_multiple = ((double)st->chunk_size) / control->st_size;
pass++; pass++;
if (st->stdin_eof) if (st->stdin_eof)
passes = pass; passes = pass;
@ -929,28 +929,28 @@ retry:
last.tv_sec = current.tv_sec; last.tv_sec = current.tv_sec;
last.tv_usec = current.tv_usec; last.tv_usec = current.tv_usec;
rzip_chunk(st, fd_in, fd_out, offset, pct_base, pct_multiple); rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple);
/* st->chunk_size may be shrunk in rzip_chunk */ /* st->chunk_size may be shrunk in rzip_chunk */
last_chunk = st->chunk_size; last_chunk = st->chunk_size;
len -= st->chunk_size; len -= st->chunk_size;
} }
close_streamout_threads(); close_streamout_threads(control);
md5_finish_ctx (&control.ctx, md5_resblock); md5_finish_ctx (&control->ctx, md5_resblock);
if (HASH_CHECK || MAX_VERBOSE) { if (HASH_CHECK || MAX_VERBOSE) {
print_output("MD5: "); print_output("MD5: ");
for (j = 0; j < MD5_DIGEST_SIZE; j++) for (j = 0; j < MD5_DIGEST_SIZE; j++)
print_output("%02x", md5_resblock[j] & 0xFF); print_output("%02x", md5_resblock[j] & 0xFF);
print_output("\n"); print_output("\n");
} }
if (unlikely(write(control.fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) if (unlikely(write(control->fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
fatal("Failed to write md5 in rzip_fd\n"); fatal("Failed to write md5 in rzip_fd\n");
gettimeofday(&current, NULL); gettimeofday(&current, NULL);
if (STDIN) if (STDIN)
s.st_size = control.st_size; s.st_size = control->st_size;
chunkmbs = (s.st_size / 1024 / 1024) / ((double)(current.tv_sec-start.tv_sec)? : 1); chunkmbs = (s.st_size / 1024 / 1024) / ((double)(current.tv_sec-start.tv_sec)? : 1);
fstat(fd_out, &s2); fstat(fd_out, &s2);
@ -966,7 +966,7 @@ retry:
(1.0 + st->stats.match_bytes) / st->stats.literal_bytes); (1.0 + st->stats.match_bytes) / st->stats.literal_bytes);
if (!STDIN) if (!STDIN)
print_progress("%s - ", control.infile); print_progress("%s - ", control->infile);
print_progress("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n", print_progress("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n",
1.0 * s.st_size / s2.st_size, chunkmbs); 1.0 * s.st_size / s2.st_size, chunkmbs);

158
rzip.h
View file

@ -17,14 +17,10 @@
along with this program. If not, see <http://www.gnu.org/licenses/>. along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define LRZIP_MAJOR_VERSION 0 #ifndef RZIP_H
#define LRZIP_MINOR_VERSION 5 #define RZIP_H
#define LRZIP_MINOR_SUBVERSION 71 #include "lrzip.h" /* includes config.h */
#include "liblrzip.h"
#define NUM_STREAMS 2
#define STREAM_BUFSIZE (1024 * 1024 * 10)
#include "config.h"
#include "md5.h" #include "md5.h"
#include <sys/types.h> #include <sys/types.h>
@ -75,76 +71,6 @@
/* needed for CRC routines */ /* needed for CRC routines */
#include "lzma/C/7zCrc.h" #include "lzma/C/7zCrc.h"
#ifndef uchar
#define uchar unsigned char
#endif
#ifndef int32
#if (SIZEOF_INT == 4)
#define int32 int
#elif (SIZEOF_LONG == 4)
#define int32 long
#elif (SIZEOF_SHORT == 4)
#define int32 short
#endif
#endif
#ifndef int16
#if (SIZEOF_INT == 2)
#define int16 int
#elif (SIZEOF_SHORT == 2)
#define int16 short
#endif
#endif
#ifndef uint32
#define uint32 unsigned int32
#endif
#ifndef uint16
#define uint16 unsigned int16
#endif
#ifndef MIN
#define MIN(a, b) ((a) < (b)? (a): (b))
#endif
#ifndef MAX
#define MAX(a, b) ((a) > (b)? (a): (b))
#endif
#if !HAVE_STRERROR
extern char *sys_errlist[];
#define strerror(i) sys_errlist[i]
#endif
#ifndef HAVE_ERRNO_H
extern int errno;
#endif
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
typedef long long int i64;
typedef uint16_t u16;
typedef uint32_t u32;
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN)
#define PROCESSORS (1)
#else
#define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
#endif
#ifdef _SC_PAGE_SIZE
#define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
#else
#define PAGE_SIZE (4096)
#endif
void fatal(const char *format, ...); void fatal(const char *format, ...);
void failure(const char *format, ...); void failure(const char *format, ...);
@ -222,33 +148,6 @@ static inline i64 get_ram(void)
#define FLAG_KEEP_BROKEN (1 << 19) #define FLAG_KEEP_BROKEN (1 << 19)
#define FLAG_THRESHOLD (1 << 20) #define FLAG_THRESHOLD (1 << 20)
#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS)
#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA))
#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS)
#define KEEP_FILES (control.flags & FLAG_KEEP_FILES)
#define TEST_ONLY (control.flags & FLAG_TEST_ONLY)
#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE)
#define DECOMPRESS (control.flags & FLAG_DECOMPRESS)
#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS)
#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS)
#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS)
#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS)
#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS)
#define VERBOSE (control.flags & FLAG_VERBOSE)
#define VERBOSITY (control.flags & FLAG_VERBOSITY)
#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX)
#define STDIN (control.flags & FLAG_STDIN)
#define STDOUT (control.flags & FLAG_STDOUT)
#define INFO (control.flags & FLAG_INFO)
#define UNLIMITED (control.flags & FLAG_UNLIMITED)
#define HASH_CHECK (control.flags & FLAG_HASH)
#define HAS_MD5 (control.flags & FLAG_MD5)
#define CHECK_FILE (control.flags & FLAG_CHECK)
#define KEEP_BROKEN (control.flags & FLAG_KEEP_BROKEN)
#define LZO_TEST (control.flags & FLAG_THRESHOLD)
#define NO_MD5 (!(HASH_CHECK) && !(HAS_MD5)) #define NO_MD5 (!(HASH_CHECK) && !(HAS_MD5))
#define BITS32 (sizeof(long) == 4) #define BITS32 (sizeof(long) == 4)
@ -285,8 +184,9 @@ struct rzip_control {
long page_size; long page_size;
int fd_out; int fd_out;
struct md5_ctx ctx; struct md5_ctx ctx;
void *data; // random data pointer associated for use in callbacks
i64 md5_read; // How far into the file the md5 has done so far i64 md5_read; // How far into the file the md5 has done so far
} control; };
struct stream { struct stream {
i64 last_head; i64 last_head;
@ -316,52 +216,36 @@ struct stream_info {
}; };
void sighandler(); void sighandler();
i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size); i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size);
void rzip_fd(int fd_in, int fd_out); void rzip_fd(rzip_control *control, int fd_in, int fd_out);
void *open_stream_out(int f, int n, i64 limit, char cbytes); void *open_stream_out(rzip_control *control, int f, int n, i64 limit, char cbytes);
void *open_stream_in(int f, int n); void *open_stream_in(rzip_control *control, int f, int n);
int write_stream(void *ss, int stream, uchar *p, i64 len); int write_stream(rzip_control *control, void *ss, int stream, uchar *p, i64 len);
i64 read_stream(void *ss, int stream, uchar *p, i64 len); i64 read_stream(rzip_control *control, void *ss, int stream, uchar *p, i64 len);
int close_stream_out(void *ss); int close_stream_out(rzip_control *control, void *ss);
int close_stream_in(void *ss); int close_stream_in(void *ss);
void flush_buffer(struct stream_info *sinfo, int stream); void flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream);
void read_config(struct rzip_control *s); void read_config(struct rzip_control *s);
ssize_t write_1g(int fd, void *buf, i64 len); ssize_t write_1g(int fd, void *buf, i64 len);
ssize_t read_1g(int fd, void *buf, i64 len); ssize_t read_1g(int fd, void *buf, i64 len);
void zpipe_compress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread); void zpipe_compress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread);
void zpipe_decompress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread); void zpipe_decompress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread);
const i64 two_gig; const i64 two_gig;
void prepare_streamout_threads(void); void prepare_streamout_threads(rzip_control *control);
void close_streamout_threads(void); void close_streamout_threads(rzip_control *control);
void round_to_page(i64 *size); void round_to_page(i64 *size);
void dump_tmpoutfile(int fd_out);
void register_infile(const char *name, char delete);
void register_outfile(const char *name, char delete);
void register_outputfile(FILE *f);
#define print_err(format, args...) do {\ #define print_err(format, args...) do {\
fprintf(stderr, format, ##args); \ fprintf(stderr, format, ##args); \
} while (0) } while (0)
#define print_output(format, args...) do {\
fprintf(control.msgout, format, ##args); \
fflush(control.msgout); \
} while (0)
#define print_progress(format, args...) do {\
if (SHOW_PROGRESS) \
print_output(format, ##args); \
} while (0)
#define print_verbose(format, args...) do {\
if (VERBOSE) \
print_output(format, ##args); \
} while (0)
#define print_maxverbose(format, args...) do {\
if (MAX_VERBOSE) \
print_output(format, ##args); \
} while (0)
/* Macros for testing parameters */ /* Macros for testing parameters */
#define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) #define isparameter( parmstring, value ) (!strcasecmp( parmstring, value ))
#define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) #define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value ))
#endif

176
stream.c
View file

@ -88,7 +88,7 @@ void join_pthread(pthread_t th, void **thread_return)
/* just to keep things clean, declare function here /* just to keep things clean, declare function here
* but move body to the end since it's a work function * but move body to the end since it's a work function
*/ */
static int lzo_compresses(uchar *s_buf, i64 s_len); static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len);
static inline FILE *fake_fmemopen(void *buf, size_t buflen, char *mode) static inline FILE *fake_fmemopen(void *buf, size_t buflen, char *mode)
{ {
@ -145,13 +145,13 @@ static inline int fake_open_memstream_update_buffer(FILE *fp, uchar **buf, size_
length in c_len length in c_len
*/ */
static int zpaq_compress_buf(struct compress_thread *cthread, long thread) static int zpaq_compress_buf(rzip_control *control, struct compress_thread *cthread, long thread)
{ {
uchar *c_buf = NULL; uchar *c_buf = NULL;
size_t dlen = 0; size_t dlen = 0;
FILE *in, *out; FILE *in, *out;
if (!lzo_compresses(cthread->s_buf, cthread->s_len)) if (!lzo_compresses(control, cthread->s_buf, cthread->s_len))
return 0; return 0;
in = fmemopen(cthread->s_buf, cthread->s_len, "r"); in = fmemopen(cthread->s_buf, cthread->s_len, "r");
@ -166,7 +166,7 @@ static int zpaq_compress_buf(struct compress_thread *cthread, long thread)
return -1; return -1;
} }
zpipe_compress(in, out, control.msgout, cthread->s_len, zpipe_compress(in, out, control->msgout, cthread->s_len,
(int)(SHOW_PROGRESS), thread); (int)(SHOW_PROGRESS), thread);
if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) if (unlikely(memstream_update_buffer(out, &c_buf, &dlen)))
@ -189,13 +189,13 @@ static int zpaq_compress_buf(struct compress_thread *cthread, long thread)
return 0; return 0;
} }
static int bzip2_compress_buf(struct compress_thread *cthread) static int bzip2_compress_buf(rzip_control *control, struct compress_thread *cthread)
{ {
u32 dlen = cthread->s_len; u32 dlen = cthread->s_len;
int bzip2_ret; int bzip2_ret;
uchar *c_buf; uchar *c_buf;
if (!lzo_compresses(cthread->s_buf, cthread->s_len)) if (!lzo_compresses(control, cthread->s_buf, cthread->s_len))
return 0; return 0;
c_buf = malloc(dlen); c_buf = malloc(dlen);
@ -206,7 +206,7 @@ static int bzip2_compress_buf(struct compress_thread *cthread)
bzip2_ret = BZ2_bzBuffToBuffCompress((char *)c_buf, &dlen, bzip2_ret = BZ2_bzBuffToBuffCompress((char *)c_buf, &dlen,
(char *)cthread->s_buf, cthread->s_len, (char *)cthread->s_buf, cthread->s_len,
control.compression_level, 0, control.compression_level * 10); control->compression_level, 0, control->compression_level * 10);
/* if compressed data is bigger then original data leave as /* if compressed data is bigger then original data leave as
* CTYPE_NONE */ * CTYPE_NONE */
@ -238,7 +238,7 @@ static int bzip2_compress_buf(struct compress_thread *cthread)
return 0; return 0;
} }
static int gzip_compress_buf(struct compress_thread *cthread) static int gzip_compress_buf(rzip_control *control, struct compress_thread *cthread)
{ {
unsigned long dlen = cthread->s_len; unsigned long dlen = cthread->s_len;
uchar *c_buf; uchar *c_buf;
@ -251,7 +251,7 @@ static int gzip_compress_buf(struct compress_thread *cthread)
} }
gzip_ret = compress2(c_buf, &dlen, cthread->s_buf, cthread->s_len, gzip_ret = compress2(c_buf, &dlen, cthread->s_buf, cthread->s_len,
control.compression_level); control->compression_level);
/* if compressed data is bigger then original data leave as /* if compressed data is bigger then original data leave as
* CTYPE_NONE */ * CTYPE_NONE */
@ -283,18 +283,18 @@ static int gzip_compress_buf(struct compress_thread *cthread)
return 0; return 0;
} }
static int lzma_compress_buf(struct compress_thread *cthread) static int lzma_compress_buf(rzip_control *control, struct compress_thread *cthread)
{ {
int lzma_level, lzma_ret; int lzma_level, lzma_ret;
size_t prop_size = 5; /* return value for lzma_properties */ size_t prop_size = 5; /* return value for lzma_properties */
uchar *c_buf; uchar *c_buf;
size_t dlen; size_t dlen;
if (!lzo_compresses(cthread->s_buf, cthread->s_len)) if (!lzo_compresses(control, cthread->s_buf, cthread->s_len))
return 0; return 0;
/* only 7 levels with lzma, scale them */ /* only 7 levels with lzma, scale them */
lzma_level = control.compression_level * 7 / 9 ? : 1; lzma_level = control->compression_level * 7 / 9 ? : 1;
print_verbose("Starting lzma back end compression thread...\n"); print_verbose("Starting lzma back end compression thread...\n");
retry: retry:
@ -309,11 +309,11 @@ retry:
* and receive properties in control->lzma_properties */ * and receive properties in control->lzma_properties */
lzma_ret = LzmaCompress(c_buf, &dlen, cthread->s_buf, lzma_ret = LzmaCompress(c_buf, &dlen, cthread->s_buf,
(size_t)cthread->s_len, control.lzma_properties, &prop_size, (size_t)cthread->s_len, control->lzma_properties, &prop_size,
lzma_level, lzma_level,
0, /* dict size. set default, choose by level */ 0, /* dict size. set default, choose by level */
-1, -1, -1, -1, /* lc, lp, pb, fb */ -1, -1, -1, -1, /* lc, lp, pb, fb */
control.threads); control->threads);
if (lzma_ret != SZ_OK) { if (lzma_ret != SZ_OK) {
switch (lzma_ret) { switch (lzma_ret) {
case SZ_ERROR_MEM: case SZ_ERROR_MEM:
@ -343,7 +343,7 @@ retry:
* fall back to bzip2 compression so the block doesn't * fall back to bzip2 compression so the block doesn't
* remain uncompressed */ * remain uncompressed */
print_verbose("Unable to allocate enough RAM for any sized compression window, falling back to bzip2 compression.\n"); print_verbose("Unable to allocate enough RAM for any sized compression window, falling back to bzip2 compression.\n");
return bzip2_compress_buf(cthread); return bzip2_compress_buf(control, cthread);
} else if (lzma_ret == SZ_ERROR_OUTPUT_EOF) } else if (lzma_ret == SZ_ERROR_OUTPUT_EOF)
return 0; return 0;
return -1; return -1;
@ -363,7 +363,7 @@ retry:
return 0; return 0;
} }
static int lzo_compress_buf(struct compress_thread *cthread) static int lzo_compress_buf(rzip_control *control, struct compress_thread *cthread)
{ {
lzo_uint in_len = cthread->s_len; lzo_uint in_len = cthread->s_len;
lzo_uint dlen = in_len + in_len / 16 + 64 + 3; lzo_uint dlen = in_len + in_len / 16 + 64 + 3;
@ -411,7 +411,7 @@ out_free:
try to decompress a buffer. Return 0 on success and -1 on failure. try to decompress a buffer. Return 0 on success and -1 on failure.
*/ */
static int zpaq_decompress_buf(struct uncomp_thread *ucthread, long thread) static int zpaq_decompress_buf(rzip_control *control, struct uncomp_thread *ucthread, long thread)
{ {
uchar *c_buf = NULL; uchar *c_buf = NULL;
size_t dlen = 0; size_t dlen = 0;
@ -428,7 +428,7 @@ static int zpaq_decompress_buf(struct uncomp_thread *ucthread, long thread)
return -1; return -1;
} }
zpipe_decompress(in, out, control.msgout, ucthread->u_len, (int)(SHOW_PROGRESS), thread); zpipe_decompress(in, out, control->msgout, ucthread->u_len, (int)(SHOW_PROGRESS), thread);
if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) if (unlikely(memstream_update_buffer(out, &c_buf, &dlen)))
fatal("Failed to memstream_update_buffer in zpaq_decompress_buf"); fatal("Failed to memstream_update_buffer in zpaq_decompress_buf");
@ -447,7 +447,7 @@ static int zpaq_decompress_buf(struct uncomp_thread *ucthread, long thread)
return 0; return 0;
} }
static int bzip2_decompress_buf(struct uncomp_thread *ucthread) static int bzip2_decompress_buf(rzip_control *control __UNUSED__, struct uncomp_thread *ucthread)
{ {
u32 dlen = ucthread->u_len; u32 dlen = ucthread->u_len;
int ret = 0, bzerr; int ret = 0, bzerr;
@ -481,7 +481,7 @@ out:
return ret; return ret;
} }
static int gzip_decompress_buf(struct uncomp_thread *ucthread) static int gzip_decompress_buf(rzip_control *control __UNUSED__, struct uncomp_thread *ucthread)
{ {
unsigned long dlen = ucthread->u_len; unsigned long dlen = ucthread->u_len;
int ret = 0, gzerr; int ret = 0, gzerr;
@ -515,7 +515,7 @@ out:
return ret; return ret;
} }
static int lzma_decompress_buf(struct uncomp_thread *ucthread) static int lzma_decompress_buf(rzip_control *control, struct uncomp_thread *ucthread)
{ {
size_t dlen = (size_t)ucthread->u_len; size_t dlen = (size_t)ucthread->u_len;
int ret = 0, lzmaerr; int ret = 0, lzmaerr;
@ -529,9 +529,9 @@ static int lzma_decompress_buf(struct uncomp_thread *ucthread)
goto out; goto out;
} }
/* With LZMA SDK 4.63 we pass control.lzma_properties /* With LZMA SDK 4.63 we pass control->lzma_properties
* which is needed for proper uncompress */ * which is needed for proper uncompress */
lzmaerr = LzmaUncompress(ucthread->s_buf, &dlen, c_buf, (SizeT *)&ucthread->c_len, control.lzma_properties, 5); lzmaerr = LzmaUncompress(ucthread->s_buf, &dlen, c_buf, (SizeT *)&ucthread->c_len, control->lzma_properties, 5);
if (unlikely(lzmaerr)) { if (unlikely(lzmaerr)) {
print_err("Failed to decompress buffer - lzmaerr=%d\n", lzmaerr); print_err("Failed to decompress buffer - lzmaerr=%d\n", lzmaerr);
ret = -1; ret = -1;
@ -551,7 +551,7 @@ out:
return ret; return ret;
} }
static int lzo_decompress_buf(struct uncomp_thread *ucthread) static int lzo_decompress_buf(rzip_control *control __UNUSED__, struct uncomp_thread *ucthread)
{ {
lzo_uint dlen = ucthread->u_len; lzo_uint dlen = ucthread->u_len;
int ret = 0, lzerr; int ret = 0, lzerr;
@ -718,7 +718,7 @@ static int seekto(struct stream_info *sinfo, i64 pos)
static pthread_t *threads; static pthread_t *threads;
void prepare_streamout_threads(void) void prepare_streamout_threads(rzip_control *control)
{ {
int i; int i;
@ -726,19 +726,19 @@ void prepare_streamout_threads(void)
* pre-processing stage, it's faster to have one more thread available * pre-processing stage, it's faster to have one more thread available
* to keep all CPUs busy. There is no point splitting up the chunks * to keep all CPUs busy. There is no point splitting up the chunks
* into multiple threads if there will be no compression back end. */ * into multiple threads if there will be no compression back end. */
if (control.threads > 1) if (control->threads > 1)
++control.threads; ++control->threads;
if (NO_COMPRESS) if (NO_COMPRESS)
control.threads = 1; control->threads = 1;
threads = calloc(sizeof(pthread_t), control.threads); threads = calloc(sizeof(pthread_t), control->threads);
if (unlikely(!threads)) if (unlikely(!threads))
fatal("Unable to calloc threads in prepare_streamout_threads\n"); fatal("Unable to calloc threads in prepare_streamout_threads\n");
cthread = calloc(sizeof(struct compress_thread), control.threads); cthread = calloc(sizeof(struct compress_thread), control->threads);
if (unlikely(!cthread)) if (unlikely(!cthread))
fatal("Unable to calloc cthread in prepare_streamout_threads\n"); fatal("Unable to calloc cthread in prepare_streamout_threads\n");
for (i = 0; i < control.threads; i++) for (i = 0; i < control->threads; i++)
init_mutex(&cthread[i].mutex); init_mutex(&cthread[i].mutex);
} }
@ -746,15 +746,15 @@ static long output_thread;
static pthread_mutex_t output_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t output_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t output_cond = PTHREAD_COND_INITIALIZER; static pthread_cond_t output_cond = PTHREAD_COND_INITIALIZER;
void close_streamout_threads(void) void close_streamout_threads(rzip_control *control)
{ {
int i, close_thread = output_thread; int i, close_thread = output_thread;
/* Wait for the threads in the correct order in case they end up /* Wait for the threads in the correct order in case they end up
* serialised */ * serialised */
for (i = 0; i < control.threads; i++) { for (i = 0; i < control->threads; i++) {
lock_mutex(&cthread[close_thread].mutex); lock_mutex(&cthread[close_thread].mutex);
if (++close_thread == control.threads) if (++close_thread == control->threads)
close_thread = 0; close_thread = 0;
} }
free(cthread); free(cthread);
@ -763,7 +763,7 @@ void close_streamout_threads(void)
/* open a set of output streams, compressing with the given /* open a set of output streams, compressing with the given
compression level and algorithm */ compression level and algorithm */
void *open_stream_out(int f, int n, i64 chunk_limit, char cbytes) void *open_stream_out(rzip_control *control, int f, int n, i64 chunk_limit, char cbytes)
{ {
struct stream_info *sinfo; struct stream_info *sinfo;
i64 testsize, limit; i64 testsize, limit;
@ -799,24 +799,24 @@ void *open_stream_out(int f, int n, i64 chunk_limit, char cbytes)
/* Serious limits imposed on 32 bit capabilities */ /* Serious limits imposed on 32 bit capabilities */
if (BITS32) if (BITS32)
limit = MIN(limit, (two_gig / testbufs) - limit = MIN(limit, (two_gig / testbufs) -
(control.overhead * control.threads)); (control->overhead * control->threads));
testsize = (limit * testbufs) + (control.overhead * control.threads); testsize = (limit * testbufs) + (control->overhead * control->threads);
if (testsize > control.maxram) if (testsize > control->maxram)
limit = (control.maxram - (control.overhead * control.threads)) / testbufs; limit = (control->maxram - (control->overhead * control->threads)) / testbufs;
/* If we don't have enough ram for the number of threads, decrease the /* If we don't have enough ram for the number of threads, decrease the
* number of threads till we do, or only have one thread. */ * number of threads till we do, or only have one thread. */
while (limit < STREAM_BUFSIZE && limit < chunk_limit) { while (limit < STREAM_BUFSIZE && limit < chunk_limit) {
if (control.threads > 1) if (control->threads > 1)
--control.threads; --control->threads;
else else
break; break;
limit = (control.maxram - (control.overhead * control.threads)) / testbufs; limit = (control->maxram - (control->overhead * control->threads)) / testbufs;
limit = MIN(limit, chunk_limit); limit = MIN(limit, chunk_limit);
} }
retest_malloc: retest_malloc:
testsize = (limit * testbufs) + (control.overhead * control.threads); testsize = (limit * testbufs) + (control->overhead * control->threads);
testmalloc = malloc(testsize); testmalloc = malloc(testsize);
if (!testmalloc) { if (!testmalloc) {
limit = limit / 10 * 9; limit = limit / 10 * 9;
@ -827,12 +827,12 @@ retest_malloc:
/* Make the bufsize no smaller than STREAM_BUFSIZE. Round up the /* Make the bufsize no smaller than STREAM_BUFSIZE. Round up the
* bufsize to fit X threads into it */ * bufsize to fit X threads into it */
sinfo->bufsize = MIN(limit, MAX((limit + control.threads - 1) / control.threads, sinfo->bufsize = MIN(limit, MAX((limit + control->threads - 1) / control->threads,
STREAM_BUFSIZE)); STREAM_BUFSIZE));
if (control.threads > 1) if (control->threads > 1)
print_maxverbose("Using up to %d threads to compress up to %lld bytes each.\n", print_maxverbose("Using up to %d threads to compress up to %lld bytes each.\n",
control.threads, sinfo->bufsize); control->threads, sinfo->bufsize);
else else
print_maxverbose("Using only 1 thread to compress up to %lld bytes\n", print_maxverbose("Using only 1 thread to compress up to %lld bytes\n",
sinfo->bufsize); sinfo->bufsize);
@ -847,7 +847,7 @@ retest_malloc:
} }
/* prepare a set of n streams for reading on file descriptor f */ /* prepare a set of n streams for reading on file descriptor f */
void *open_stream_in(int f, int n) void *open_stream_in(rzip_control *control, int f, int n)
{ {
struct stream_info *sinfo; struct stream_info *sinfo;
int total_threads, i; int total_threads, i;
@ -859,10 +859,10 @@ void *open_stream_in(int f, int n)
/* We have one thread dedicated to stream 0, and one more thread than /* We have one thread dedicated to stream 0, and one more thread than
* CPUs to keep them busy, unless we're running single-threaded. */ * CPUs to keep them busy, unless we're running single-threaded. */
if (control.threads > 1) if (control->threads > 1)
total_threads = control.threads + 2; total_threads = control->threads + 2;
else else
total_threads = control.threads + 1; total_threads = control->threads + 1;
threads = calloc(sizeof(pthread_t), total_threads); threads = calloc(sizeof(pthread_t), total_threads);
if (unlikely(!threads)) if (unlikely(!threads))
return NULL; return NULL;
@ -897,7 +897,7 @@ again:
goto failed; goto failed;
/* Compatibility crap for versions < 0.40 */ /* Compatibility crap for versions < 0.40 */
if (control.major_version == 0 && control.minor_version < 4) { if (control->major_version == 0 && control->minor_version < 4) {
u32 v132, v232, last_head32; u32 v132, v232, last_head32;
if (unlikely(read_u32(f, &v132))) if (unlikely(read_u32(f, &v132)))
@ -953,14 +953,16 @@ failed:
/* Enter with s_buf allocated,s_buf points to the compressed data after the /* Enter with s_buf allocated,s_buf points to the compressed data after the
* backend compression and is then freed here */ * backend compression and is then freed here */
static void *compthread(void *t) static void *compthread(void *data)
{ {
long i = (long)t; rzip_control *control = data;
long i = (long)control->data;
control->data = NULL;
struct compress_thread *cti = &cthread[i]; struct compress_thread *cti = &cthread[i];
struct stream_info *ctis = cti->sinfo; struct stream_info *ctis = cti->sinfo;
int waited = 0, ret = 0; int waited = 0, ret = 0;
if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val) == -1)) if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1))
print_err("Warning, unable to set nice value on thread\n"); print_err("Warning, unable to set nice value on thread\n");
cti->c_type = CTYPE_NONE; cti->c_type = CTYPE_NONE;
@ -972,15 +974,15 @@ static void *compthread(void *t)
retry: retry:
if (!NO_COMPRESS && cti->c_len) { if (!NO_COMPRESS && cti->c_len) {
if (LZMA_COMPRESS) if (LZMA_COMPRESS)
ret = lzma_compress_buf(cti); ret = lzma_compress_buf(control, cti);
else if (LZO_COMPRESS) else if (LZO_COMPRESS)
ret = lzo_compress_buf(cti); ret = lzo_compress_buf(control, cti);
else if (BZIP2_COMPRESS) else if (BZIP2_COMPRESS)
ret = bzip2_compress_buf(cti); ret = bzip2_compress_buf(control, cti);
else if (ZLIB_COMPRESS) else if (ZLIB_COMPRESS)
ret = gzip_compress_buf(cti); ret = gzip_compress_buf(control, cti);
else if (ZPAQ_COMPRESS) else if (ZPAQ_COMPRESS)
ret = zpaq_compress_buf(cti, i); ret = zpaq_compress_buf(control, cti, i);
else failure("Dunno wtf compression to use!\n"); else failure("Dunno wtf compression to use!\n");
} }
@ -1047,7 +1049,7 @@ retry:
free(cti->s_buf); free(cti->s_buf);
lock_mutex(&output_lock); lock_mutex(&output_lock);
if (++output_thread == control.threads) if (++output_thread == control->threads)
output_thread = 0; output_thread = 0;
cond_broadcast(&output_cond); cond_broadcast(&output_cond);
unlock_mutex(&output_lock); unlock_mutex(&output_lock);
@ -1057,7 +1059,7 @@ retry:
return 0; return 0;
} }
static void clear_buffer(struct stream_info *sinfo, int stream, int newbuf) static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int stream, int newbuf)
{ {
static long i = 0; static long i = 0;
@ -1071,7 +1073,8 @@ static void clear_buffer(struct stream_info *sinfo, int stream, int newbuf)
print_maxverbose("Starting thread %ld to compress %lld bytes from stream %d\n", print_maxverbose("Starting thread %ld to compress %lld bytes from stream %d\n",
i, cthread[i].s_len, stream); i, cthread[i].s_len, stream);
create_pthread(&threads[i], NULL, compthread, (void *)i); control->data = (void*)i;
create_pthread(&threads[i], NULL, compthread, control);
if (newbuf) { if (newbuf) {
/* The stream buffer has been given to the thread, allocate a new one */ /* The stream buffer has been given to the thread, allocate a new one */
@ -1081,42 +1084,44 @@ static void clear_buffer(struct stream_info *sinfo, int stream, int newbuf)
sinfo->s[stream].buflen = 0; sinfo->s[stream].buflen = 0;
} }
if (++i == control.threads) if (++i == control->threads)
i = 0; i = 0;
} }
/* flush out any data in a stream buffer */ /* flush out any data in a stream buffer */
void flush_buffer(struct stream_info *sinfo, int stream) void flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream)
{ {
clear_buffer(sinfo, stream, 1); clear_buffer(control, sinfo, stream, 1);
} }
static void *ucompthread(void *t) static void *ucompthread(void *data)
{ {
long i = (long)t; rzip_control *control = data;
long i = (long)control->data;
control->data = NULL;
struct uncomp_thread *uci = &ucthread[i]; struct uncomp_thread *uci = &ucthread[i];
int waited = 0, ret = 0; int waited = 0, ret = 0;
if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val) == -1)) if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1))
print_err("Warning, unable to set nice value on thread\n"); print_err("Warning, unable to set nice value on thread\n");
retry: retry:
if (uci->c_type != CTYPE_NONE) { if (uci->c_type != CTYPE_NONE) {
switch (uci->c_type) { switch (uci->c_type) {
case CTYPE_LZMA: case CTYPE_LZMA:
ret = lzma_decompress_buf(uci); ret = lzma_decompress_buf(control, uci);
break; break;
case CTYPE_LZO: case CTYPE_LZO:
ret = lzo_decompress_buf(uci); ret = lzo_decompress_buf(control, uci);
break; break;
case CTYPE_BZIP2: case CTYPE_BZIP2:
ret = bzip2_decompress_buf(uci); ret = bzip2_decompress_buf(control, uci);
break; break;
case CTYPE_GZIP: case CTYPE_GZIP:
ret = gzip_decompress_buf(uci); ret = gzip_decompress_buf(control, uci);
break; break;
case CTYPE_ZPAQ: case CTYPE_ZPAQ:
ret = zpaq_decompress_buf(uci, i); ret = zpaq_decompress_buf(control, uci, i);
break; break;
default: default:
failure("Dunno wtf decompression type to use!\n"); failure("Dunno wtf decompression type to use!\n");
@ -1147,7 +1152,7 @@ retry:
} }
/* fill a buffer from a stream - return -1 on failure */ /* fill a buffer from a stream - return -1 on failure */
static int fill_buffer(struct stream_info *sinfo, int stream) static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int stream)
{ {
i64 header_length, u_len, c_len, last_head; i64 header_length, u_len, c_len, last_head;
struct stream *s = &sinfo->s[stream]; struct stream *s = &sinfo->s[stream];
@ -1168,7 +1173,7 @@ fill_another:
return -1; return -1;
/* Compatibility crap for versions < 0.4 */ /* Compatibility crap for versions < 0.4 */
if (control.major_version == 0 && control.minor_version < 4) { if (control->major_version == 0 && control->minor_version < 4) {
u32 c_len32, u_len32, last_head32; u32 c_len32, u_len32, last_head32;
if (unlikely(read_u32(sinfo->fd, &c_len32))) if (unlikely(read_u32(sinfo->fd, &c_len32)))
@ -1193,7 +1198,7 @@ fill_another:
sinfo->total_read += header_length; sinfo->total_read += header_length;
fsync(control.fd_out); fsync(control->fd_out);
s_buf = malloc(u_len); s_buf = malloc(u_len);
if (unlikely(u_len && !s_buf)) if (unlikely(u_len && !s_buf))
@ -1216,7 +1221,8 @@ fill_another:
ucthread[s->uthread_no].busy = 1; ucthread[s->uthread_no].busy = 1;
print_maxverbose("Starting thread %ld to decompress %lld bytes from stream %d\n", print_maxverbose("Starting thread %ld to decompress %lld bytes from stream %d\n",
s->uthread_no, c_len, stream); s->uthread_no, c_len, stream);
create_pthread(&threads[s->uthread_no], NULL, ucompthread, (void *)s->uthread_no); control->data = (void*)s->uthread_no;
create_pthread(&threads[s->uthread_no], NULL, ucompthread, control);
if (++s->uthread_no == s->base_thread + s->total_threads) if (++s->uthread_no == s->base_thread + s->total_threads)
s->uthread_no = s->base_thread; s->uthread_no = s->base_thread;
@ -1228,7 +1234,7 @@ fill_another:
if (!last_head) if (!last_head)
s->eos = 1; s->eos = 1;
else if (s->uthread_no != s->unext_thread && !ucthread[s->uthread_no].busy && else if (s->uthread_no != s->unext_thread && !ucthread[s->uthread_no].busy &&
sinfo->ram_alloced < control.ramsize / 3) sinfo->ram_alloced < control->ramsize / 3)
goto fill_another; goto fill_another;
out: out:
lock_mutex(&output_lock); lock_mutex(&output_lock);
@ -1253,7 +1259,7 @@ out:
} }
/* write some data to a stream. Return -1 on failure */ /* write some data to a stream. Return -1 on failure */
int write_stream(void *ss, int stream, uchar *p, i64 len) int write_stream(rzip_control *control, void *ss, int stream, uchar *p, i64 len)
{ {
struct stream_info *sinfo = ss; struct stream_info *sinfo = ss;
@ -1269,14 +1275,14 @@ int write_stream(void *ss, int stream, uchar *p, i64 len)
/* Flush the buffer every sinfo->bufsize into one thread */ /* Flush the buffer every sinfo->bufsize into one thread */
if (sinfo->s[stream].buflen == sinfo->bufsize) if (sinfo->s[stream].buflen == sinfo->bufsize)
flush_buffer(sinfo, stream); flush_buffer(control, sinfo, stream);
} }
return 0; return 0;
} }
/* read some data from a stream. Return number of bytes read, or -1 /* read some data from a stream. Return number of bytes read, or -1
on failure */ on failure */
i64 read_stream(void *ss, int stream, uchar *p, i64 len) i64 read_stream(rzip_control *control, void *ss, int stream, uchar *p, i64 len)
{ {
struct stream_info *sinfo = ss; struct stream_info *sinfo = ss;
i64 ret = 0; i64 ret = 0;
@ -1295,7 +1301,7 @@ i64 read_stream(void *ss, int stream, uchar *p, i64 len)
} }
if (len && sinfo->s[stream].bufp == sinfo->s[stream].buflen) { if (len && sinfo->s[stream].bufp == sinfo->s[stream].buflen) {
if (unlikely(fill_buffer(sinfo, stream))) if (unlikely(fill_buffer(control, sinfo, stream)))
return -1; return -1;
if (sinfo->s[stream].bufp == sinfo->s[stream].buflen) if (sinfo->s[stream].bufp == sinfo->s[stream].buflen)
break; break;
@ -1306,14 +1312,14 @@ i64 read_stream(void *ss, int stream, uchar *p, i64 len)
} }
/* flush and close down a stream. return -1 on failure */ /* flush and close down a stream. return -1 on failure */
int close_stream_out(void *ss) int close_stream_out(rzip_control *control, void *ss)
{ {
struct stream_info *sinfo = ss; struct stream_info *sinfo = ss;
int i; int i;
for (i = 0; i < sinfo->num_streams; i++) { for (i = 0; i < sinfo->num_streams; i++) {
if (sinfo->s[i].buflen) if (sinfo->s[i].buflen)
clear_buffer(sinfo, i, 0); clear_buffer(control, sinfo, i, 0);
} }
#if 0 #if 0
@ -1351,7 +1357,7 @@ int close_stream_in(void *ss)
to see if there is any compression at all with lzo first. It is unlikely to see if there is any compression at all with lzo first. It is unlikely
that others will be able to compress if lzo is unable to drop a single byte that others will be able to compress if lzo is unable to drop a single byte
so do not compress any block that is incompressible by lzo. */ so do not compress any block that is incompressible by lzo. */
static int lzo_compresses(uchar *s_buf, i64 s_len) static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len)
{ {
lzo_bytep wrkmem = NULL; lzo_bytep wrkmem = NULL;
lzo_uint in_len, test_len = s_len, save_len = s_len; lzo_uint in_len, test_len = s_len, save_len = s_len;

38
util.c
View file

@ -32,20 +32,44 @@
#include "rzip.h" #include "rzip.h"
static const char *infile = NULL;
static char delete_infile = 0;
static const char *outfile = NULL;
static char delete_outfile = 0;
static FILE *outputfile = NULL;
void register_infile(const char *name, char delete)
{
infile = name;
delete_infile = delete;
}
void register_outfile(const char *name, char delete)
{
outfile = name;
delete_outfile = delete;
}
void register_outputfile(FILE *f)
{
outputfile = f;
}
static void unlink_files(void) static void unlink_files(void)
{ {
/* Delete temporary files generated for testing or faking stdio */ /* Delete temporary files generated for testing or faking stdio */
if (TEST_ONLY || STDOUT || !KEEP_BROKEN) if (outfile && delete_outfile)
unlink(control.outfile); unlink(outfile);
if ((DECOMPRESS || TEST_ONLY) && STDIN) if (infile && delete_infile)
unlink(control.infile); unlink(infile);
} }
static void fatal_exit(void) static void fatal_exit(void)
{ {
unlink_files(); unlink_files();
print_output("Fatal error - exiting\n"); fprintf(outputfile, "Fatal error - exiting\n");
fflush(outputfile);
exit(1); exit(1);
} }
@ -85,9 +109,9 @@ void sighandler()
void round_to_page(i64 *size) void round_to_page(i64 *size)
{ {
*size -= *size % control.page_size; *size -= *size % PAGE_SIZE;
if (unlikely(!*size)) if (unlikely(!*size))
*size = control.page_size; *size = PAGE_SIZE;
} }
void read_config( struct rzip_control *control ) void read_config( struct rzip_control *control )