ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'compat'

Browse source
This commit is contained in:
Con Kolivas 2016-06-10 21:05:39 +10:00
parent 2086185ed5 d05334bd86
commit 4f53d36caa
6 changed files with 111 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
README.md
View file

@ -4,7 +4,7 @@ lrzip - Long Range ZIP or LZMA RZIP
A compression utility that excels at compressing large files (usually > 10-50 MB).
Larger files and/or more free RAM means that the utility will be able to more
effectively compress your files (ie: faster / smaller size), especially if the
filesize(s) exceed 100 MB. You can either choose to optimize for speed (fast
filesize(s) exceed 100 MB. You can either choose to optimise for speed (fast
compression / decompression) or size, but not both.
@ -89,6 +89,8 @@ Simple 'n Easy™: `sudo make install`
|`lrzip -z filename`|An archive "filename.lrz" compressed with **ZPAQ** that can give extreme compression, but takes a bit longer than forever to compress and decompress.|
|`lrzip -l filename`|An archive lightly compressed with **LZO**, meaning really, really fast compression and decompression.|
|`lrunzip filename.lrz`|Decompress filename.lrz to filename.|
|`lrz filename`|As per lrzip above but with gzip compatible semantics (i.e. will be quiet and delete original file)
|`lrz -d filename.lrz`|As per lrunzip above but with gzip compatible semantics (i.e. will be quiet and delete original file)
### lrzip internals
@ -281,13 +283,13 @@ its very nature has very little redundancy. This means that there is not
much that can actually be compressed. If your video/audio/picture is in a
high bitrate, there will be more redundancy than a low bitrate one making it
more suitable to compression. None of the compression techniques in lrzip are
optimised for this sort of dat
optimised for this sort of data.
> A: However, the nature of rzip preparation
means that you'll still get better compression than most normal compression
algorithms give you if you have very large files. ISO images of dvds for
example are best compressed directly instead of individual .VOB files. ZPAQ is
the only compression format that can do any significant compression of
multimedi
multimedia.
> A:
> Q: Is this multithreaded?
@ -397,9 +399,14 @@ compression backend (lzma) needs to compress.
> Q: This version is much slower than the old version?
> A: Make sure you have set CFLAGS and CXXFLAGS. An unoptimized build will be
> A: Make sure you have set CFLAGS and CXXFLAGS. An unoptimised build will be
almost 3 times slower.
> Q: Why not update to the latest version of libzpaq?
> A: For reasons that are unclear the later versions of libzpaq create
corrupt archives when included with lrzip
#### LIMITATIONS
Due to mmap limitations the maximum size a window can be set to is currently
2GB on 32bit unless the -U option is specified. Files generated on 64 bit
@ -465,7 +472,7 @@ Persons above are listed in chronological order of first contribution to **lrzip
#### README Authors
Con Kolivas (`ckolivas` on GitHub) <kernel@kolivas.org>
Sat, 11 March 2011: README
Fri, 10 June 2016: README
Also documented by
Peter Hyman <pete@peterhyman.com>

96
main.c
View file

@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell
@ -49,6 +49,7 @@
# include <arpa/inet.h>
#endif
#include <dirent.h>
#include <getopt.h>
#include <libgen.h>
@ -60,12 +61,14 @@
/* needed for CRC routines */
#include "lzma/C/7zCrc.h"
#define MAX_PATH_LEN 4096
static rzip_control base_control, local_control, *control;
static void usage(bool compat)
{
print_output("lrz%s version %s\n", compat ? "" : "ip", PACKAGE_VERSION);
print_output("Copyright (C) Con Kolivas 2006-2015\n");
print_output("Copyright (C) Con Kolivas 2006-2016\n");
print_output("Based on rzip ");
print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n");
print_output("Usage: lrz%s [options] <file...>\n", compat ? "" : "ip");
@ -85,6 +88,7 @@ static void usage(bool compat)
print_output(" -P, --progress show compression progress\n");
} else
print_output(" -q, --quiet don't show compression progress\n");
print_output(" -r, --recursive operate recursively on directories\n");
print_output(" -t, --test test compressed file integrity\n");
print_output(" -v[v%s], --verbose Increase verbosity\n", compat ? "v" : "");
print_output(" -V, --version show version\n");
@ -229,6 +233,7 @@ static struct option long_options[] = {
{"threads", required_argument, 0, 'p'}, /* 20 */
{"progress", no_argument, 0, 'P'},
{"quiet", no_argument, 0, 'q'},
{"recursive", no_argument, 0, 'r'},
{"suffix", required_argument, 0, 'S'},
{"test", no_argument, 0, 't'},
{"threshold", required_argument, 0, 'T'}, /* 25 */
@ -250,9 +255,41 @@ static void set_stdout(struct rzip_control *control)
register_outputfile(control, control->msgout);
}
/* Recursively enter all directories, adding all regular files to the dirlist array */
static void recurse_dirlist(char *indir, char **dirlist, int *entries)
{
char fname[MAX_PATH_LEN];
struct stat istat;
struct dirent *dp;
DIR *dirp;
dirp = opendir(indir);
if (unlikely(!dirp))
failure("Unable to open directory %s\n", indir);
while ((dp = readdir(dirp)) != NULL) {
if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, ".."))
continue;
sprintf(fname, "%s/%s", indir, dp->d_name);
if (unlikely(stat(fname, &istat)))
failure("Unable to stat file %s\n", fname);
if (S_ISDIR(istat.st_mode)) {
recurse_dirlist(fname, dirlist, entries);
continue;
}
if (!S_ISREG(istat.st_mode)) {
print_err("Not regular file %s\n", fname);
continue;
}
print_maxverbose("Added file %s\n", fname);
*dirlist = realloc(*dirlist, MAX_PATH_LEN * (*entries + 1));
strcpy(*dirlist + MAX_PATH_LEN * (*entries)++, fname);
}
closedir(dirp);
}
int main(int argc, char *argv[])
{
bool lrzcat = false, compat = false;
bool lrzcat = false, compat = false, recurse = false;
struct timeval start_time, end_time;
struct sigaction handler;
double seconds,total_time; // for timers
@ -294,7 +331,7 @@ int main(int argc, char *argv[])
else if (!strstr(eptr,"NOCONFIG"))
read_config(control);
while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pPqS:tTUm:vVw:z?123456789", long_options, &i)) != -1) {
while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pPqrS:tTUm:vVw:z?123456789", long_options, &i)) != -1) {
switch (c) {
case 'b':
if (control->flags & FLAG_NOT_LZMA)
@ -400,6 +437,9 @@ int main(int argc, char *argv[])
case 'q':
control->flags &= ~FLAG_SHOW_PROGRESS;
break;
case 'r':
recurse = true;
break;
case 'S':
if (control->outname)
failure("Specified output filename already, can't specify an extension.\n");
@ -462,8 +502,12 @@ int main(int argc, char *argv[])
argc -= optind;
argv += optind;
if (control->outname && argc > 1)
if (control->outname) {
if (argc > 1)
failure("Cannot specify output filename with more than 1 file\n");
if (recurse)
failure("Cannot specify output filename with recursive\n");
}
if (VERBOSE && !SHOW_PROGRESS) {
print_err("Cannot have -v and -q options. -v wins.\n");
@ -497,25 +541,49 @@ int main(int argc, char *argv[])
/* One extra iteration for the case of no parameters means we will default to stdin/out */
for (i = 0; i <= argc; i++) {
char *dirlist = NULL, *infile = NULL;
int direntries = 0, curentry = 0;
if (i < argc)
control->infile = argv[i];
infile = argv[i];
else if (!(i == 0 && STDIN))
break;
if (control->infile) {
if ((strcmp(control->infile, "-") == 0))
if (infile) {
if ((strcmp(infile, "-") == 0))
control->flags |= FLAG_STDIN;
else {
struct stat infile_stat;
bool isdir = false;
struct stat istat;
stat(control->infile, &infile_stat);
if (unlikely(S_ISDIR(infile_stat.st_mode)))
failure("lrzip only works directly on FILES.\n"
"Use lrztar or pipe through tar for compressing directories.\n");
if (unlikely(stat(infile, &istat)))
failure("Failed to stat %s\n", infile);
isdir = S_ISDIR(istat.st_mode);
if (!recurse && (isdir || !S_ISREG(istat.st_mode))) {
failure("lrzip only works directly on regular FILES.\n"
"Use -r recursive, lrztar or pipe through tar for compressing directories.\n");
}
if (recurse && !isdir)
failure("%s not a directory, -r recursive needs a directory\n", infile);
}
}
if (recurse) {
if (unlikely(STDIN || STDOUT))
failure("Cannot use -r recursive with STDIO\n");
recurse_dirlist(infile, &dirlist, &direntries);
}
if (INFO && STDIN)
failure("Will not get file info from STDIN\n");
recursion:
if (recurse) {
if (curentry >= direntries) {
infile = NULL;
continue;
}
infile = dirlist + MAX_PATH_LEN * curentry++;
}
control->infile = infile;
/* If no output filename is specified, and we're using
* stdin, use stdout */
@ -591,6 +659,8 @@ int main(int argc, char *argv[])
seconds = total_time - hours * 3600 - minutes * 60;
if (!INFO)
print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
if (recurse)
goto recursion;
}
return 0;

9
man/lrz.1.pod
View file

@ -1,6 +1,6 @@
# Copyright
#
# Copyright (C) 2015 Con Kolivas
# Copyright (C) 2016 Con Kolivas
#
# License
#
@ -48,8 +48,8 @@ lrz - gzip compatible command line variant of lrzip
=head1 DESCRIPTION
lrz is identical to the lrzip application however its command line options are
made to be as compatible with gzip as possible.
lrz is identical to the lrzip application however its command line options and
behaviour are made to be as compatible with gzip as possible.
=head1 OPTIONS
@ -65,8 +65,9 @@ General options:
-i, --info show compressed file information
-L, --license display software version and license
-P, --progress show compression progress
-r, --recursive operate recursively on directories
-t, --test test compressed file integrity
-v[v], --verbose Increase verbosity
-v[vv], --verbose Increase verbosity
-V, --version show version
Options affecting output:
-f, --force force overwrite of any existing files

10
man/lrzip.1
View file

@ -1,4 +1,4 @@
.TH "lrzip" "1" "April 2015" "" ""
.TH "lrzip" "1" "June 2016" "" ""
.SH "NAME"
lrzip \- a large-file compression program
.SH "SYNOPSIS"
@ -43,6 +43,7 @@ General options:
\-H, \-\-hash display md5 hash integrity information
\-i, \-\-info show compressed file information
\-q, \-\-quiet don't show compression progress
\-r, \-\-recursive operate recursively on directories
\-t, \-\-test test compressed file integrity
\-v[v], \-\-verbose Increase verbosity
\-V, \-\-version show version
@ -140,6 +141,13 @@ bursts with lzma compression which is the default compression. This means
that it will progress very rapidly for short periods and then stop for
long periods.
.IP
.IP "\fB-r\fP"
If this option is specified, lrzip will recursively enter the directories
specified, compressing or decompressing every file individually in the same
directory. Note for better compression it is recommended to instead combine
files in a tar file rather than compress them separately, either manually
or with the lrztar helper.
.IP
.IP "\fB-t\fP"
This tests the compressed file integrity. It does this by decompressing it
to a temporary file and then deleting it.

239
regression_test.sh
View file

@ -1,239 +0,0 @@
#!/bin/bash
#Very basic regression testing does a number of regular compression /
#decompression / test cycles +/- STDIN +/- STDOUT and with the different
#compression backends.
#Run it with
# regression_test.sh filename
#where filename is any random file to test with (big or small depending on
#what's being tested.
infile=$1
end(){
rm -f lrztest lrztest.lrz
}
if [ ! -e $infile ]; then
echo $infile does not exist, exiting
exit 1
fi
if [ -f lrztest ]; then
echo lrztest file exists, exiting
exit 1
fi
if [ -f lrztest.lrz ]; then
echo lrztest.lrz file exists, exiting
exit 1
fi
trap 'echo "ABORTING";end;exit' 1 2 15
echo testing compression from stdin
./lrzip -vvlfo lrztest.lrz < $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing compression from stdin
end
exit 1
fi
rm lrztest.lrz
echo testing compression to stdout
./lrzip -vvlo - $infile > lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing compression to stdout
end
exit 1
fi
rm lrztest.lrz
echo testing compression from stdin to stdout
./lrzip -vvl < $infile > lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing compression from stdin to stdout
end
exit 1
fi
rm lrztest.lrz
echo testing standard compression
./lrzip -vvlfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing standard compression
end
exit 1
fi
echo testing standard decompression
./lrzip -vvdo lrztest lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest ];then
echo FAILED testing standard decompression
end
exit 1
fi
rm lrztest
echo testing standard decompression with file checking
./lrzip -vvdfco lrztest lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest ];then
echo FAILED testing standard decompression with file checking
end
exit 1
fi
rm lrztest
echo testing decompression from stdin
./lrzip -vvfo lrztest -d < lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest ];then
echo FAILED testing decompression from stdin
end
exit 1
fi
rm lrztest
echo testing decompression to stdout
./lrzip -vvdo - lrztest.lrz > lrztest
if [ $? -ne 0 ] || [ ! -f lrztest ];then
echo FAILED testing decompression to stdout
end
exit 1
fi
rm lrztest
echo testing decompression from stdin to stdout
./lrzip -vvd < lrztest.lrz > lrztest
if [ $? -ne 0 ] || [ ! -f lrztest ];then
echo FAILED testing decompression from stdin to stdout
end
exit 1
fi
rm lrztest
echo testing testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing testing
end
exit 1
fi
echo testing testing from stdin
./lrzip -vvt < lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing testing from stdin
end
exit 1
fi
rm lrztest.lrz
echo testing rzip only compression
./lrzip -vvnfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing rzip only compression
end
exit 1
fi
echo testing rzip only testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing rzip only testing
end
exit 1
fi
rm lrztest.lrz
echo testing lzma compression
./lrzip -vvfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing lzma compression
end
exit 1
fi
echo testing lzma testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing lzma testing
end
exit 1
fi
rm lrztest.lrz
echo testing gzip compression
./lrzip -vvgfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing gzip compression
end
exit 1
fi
echo testing gzip testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing gzip testing
end
exit 1
fi
rm lrztest.lrz
echo testing bzip2 compression
./lrzip -vvbfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing bzip2 compression
end
exit 1
fi
echo testing bzip2 testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing bzip2 testing
end
exit 1
fi
rm lrztest.lrz
echo testing zpaq compression
./lrzip -vvzfo lrztest.lrz $infile
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing zpaq compression
end
exit 1
fi
echo testing zpaq testing
./lrzip -vvt lrztest.lrz
if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then
echo FAILED testing zpaq testing
end
exit 1
fi
end
echo ALL TESTS SUCCESSFUL
exit 0

2
rzip.c
View file

@ -744,11 +744,11 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st,
if (cksum_len < control->page_size)
failure("Failed to malloc any ram for checksum ckbuf\n");
}
control->checksum.buf = buf;
/* Compute checksum. If the entire chunk is longer than maxram,
* do it "per-partes" */
cksem_wait(control, &control->cksumsem);
control->checksum.buf = buf;
control->checksum.len = st->chunk_size - cksum_limit;
cksum_chunks = control->checksum.len / cksum_len;
cksum_remains = control->checksum.len % cksum_len;