From 90c8072debce69fffc680c2fee5dcacd14952980 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Fri, 13 Mar 2015 13:02:54 +1100 Subject: [PATCH 01/18] Add compatibility mode with gzip --- Makefile.am | 2 ++ main.c | 74 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/Makefile.am b/Makefile.am index 43b06d9..1407d72 100644 --- a/Makefile.am +++ b/Makefile.am @@ -107,12 +107,14 @@ install-exec-hook: $(LN_S) -f lrzip$(EXEEXT) $(DESTDIR)$(bindir)/lrunzip$(EXEEXT) $(LN_S) -f lrzip$(EXEEXT) $(DESTDIR)$(bindir)/lrzcat$(EXEEXT) $(LN_S) -f lrztar$(EXEEXT) $(DESTDIR)$(bindir)/lrzuntar$(EXEEXT) + $(LN_S) -f lrz$(EXEEXT) $(DESTDIR)$(bindir)/lrz$(EXEEXT) uninstall-local: rm -f $(bindir)/lrunzip rm -f $(bindir)/lrzcat rm -f $(bindir)/lrzuntar + rm -f $(bindir)/lrz .PHONY: doc diff --git a/main.c b/main.c index 40c943a..74589b7 100644 --- a/main.c +++ b/main.c @@ -50,6 +50,7 @@ #endif #include +#include #include "rzip.h" #include "lrzip_core.h" @@ -61,7 +62,7 @@ static rzip_control base_control, local_control, *control; -static void usage(void) +static void usage(bool compat) { print_output("lrzip version %s\n", PACKAGE_VERSION); print_output("Copyright (C) Con Kolivas 2006-2015\n"); @@ -69,20 +70,30 @@ static void usage(void) print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n"); print_output("Usage: lrzip [options] \n"); print_output("General options:\n"); - print_output(" -c, --check check integrity of file written on decompression\n"); + if (compat) { + print_output(" -c, --stdout output to STDOUT\n"); + print_output(" -C, --check check integrity of file written on decompression\n"); + } else + print_output(" -c, -C, --check check integrity of file written on decompression\n"); print_output(" -d, --decompress decompress\n"); print_output(" -e, --encrypt password protected sha512/aes128 encryption on compression\n"); print_output(" -h, -?, --help show help\n"); print_output(" -H, --hash display md5 hash integrity information\n"); print_output(" -i, --info show compressed file information\n"); - print_output(" -q, --quiet don't show compression progress\n"); + if (compat) + print_output(" -P, --progress show compression progress\n"); + else + print_output(" -q, --quiet don't show compression progress\n"); print_output(" -t, --test test compressed file integrity\n"); print_output(" -v[v], --verbose Increase verbosity\n"); print_output(" -V, --version show version\n"); print_output("Options affecting output:\n"); - print_output(" -D, --delete delete existing files\n"); + if (!compat) + print_output(" -D, --delete delete existing files\n"); print_output(" -f, --force force overwrite of any existing files\n"); - print_output(" -k, --keep-broken keep broken or damaged output files\n"); + if (compat) + print_output(" -k, --keep don't delete source files on de/compression\n"); + print_output(" -%s, --keep-broken keep broken or damaged output files\n", compat ? "K" : "k, -K"); print_output(" -o, --outfile filename specify the output file name and/or path\n"); print_output(" -O, --outdir directory specify the output directory when -o is not used\n"); print_output(" -S, --suffix suffix specify compressed suffix (default '.lrz')\n"); @@ -94,7 +105,7 @@ static void usage(void) print_output(" -z, --zpaq zpaq compression (best, extreme compression, extremely slow)\n"); print_output("Low level options:\n"); print_output(" -L, --level level set lzma/bzip2/gzip compression level (1-9, default 7)\n"); - print_output(" -N, --nice-level value Set nice value to value (default 19)\n"); + print_output(" -N, --nice-level value Set nice value to value (default %d)\n", compat ? 0 : 19); print_output(" -p, --threads value Set processor count to override number of threads\n"); print_output(" -m, --maxram size Set maximim available ram in hundreds of MB\n"); print_output(" overrides detected ammount of available ram\n"); @@ -191,6 +202,7 @@ static void show_summary(void) static struct option long_options[] = { {"bzip2", no_argument, 0, 'b'}, {"check", no_argument, 0, 'c'}, + {"check", no_argument, 0, 'C'}, {"decompress", no_argument, 0, 'd'}, {"delete", no_argument, 0, 'D'}, {"encrypt", no_argument, 0, 'e'}, @@ -200,6 +212,7 @@ static struct option long_options[] = { {"hash", no_argument, 0, 'H'}, {"info", no_argument, 0, 'i'}, {"keep-broken", no_argument, 0, 'k'}, + {"keep-broken", no_argument, 0, 'K'}, {"lzo", no_argument, 0, 'l'}, {"level", no_argument, 0, 'L'}, {"maxram", required_argument, 0, 'm'}, @@ -208,6 +221,7 @@ static struct option long_options[] = { {"outfile", required_argument, 0, 'o'}, {"outdir", required_argument, 0, 'O'}, {"threads", required_argument, 0, 'p'}, + {"progress", no_argument, 0, 'P'}, {"quiet", no_argument, 0, 'q'}, {"suffix", required_argument, 0, 'S'}, {"test", no_argument, 0, 't'}, @@ -217,29 +231,40 @@ static struct option long_options[] = { {"version", no_argument, 0, 'V'}, {"window", required_argument, 0, 'w'}, {"zpaq", no_argument, 0, 'z'}, + {"fast", no_argument, 0, '1'}, + {"best", no_argument, 0, '9'}, {0, 0, 0, 0}, }; int main(int argc, char *argv[]) { + bool lrzcat = false, compat = false; struct timeval start_time, end_time; struct sigaction handler; double seconds,total_time; // for timers - bool lrzcat = false; int c, i; int hours,minutes; extern int optind; - char *eptr; /* for environment */ + char *eptr, *av; /* for environment */ control = &base_control; initialise_control(control); - if (strstr(argv[0], "lrunzip")) + av = basename(argv[0]); + if (!strcmp(av, "lrunzip")) control->flags |= FLAG_DECOMPRESS; - else if (strstr(argv[0], "lrzcat")) { + else if (!strcmp(av, "lrzcat")) { control->flags |= FLAG_DECOMPRESS | FLAG_STDOUT; lrzcat = true; + } else if (!strcmp(av, "lrz")) { + /* Called in gzip compatible command line mode */ + control->flags &= ~FLAG_SHOW_PROGRESS; + control->nice_val = 0; + control->flags &= ~FLAG_KEEP_FILES; + compat = true; + long_options[1].name = "stdout"; + long_options[11].name = "keep"; } /* generate crc table */ @@ -255,7 +280,7 @@ int main(int argc, char *argv[]) else if (!strstr(eptr,"NOCONFIG")) read_config(control); - while ((c = getopt_long(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUm:vVw:z?", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pP:qS:tTUm:vVw:z?123456789", long_options, &i)) != -1) { switch (c) { case 'b': if (control->flags & FLAG_NOT_LZMA) @@ -263,6 +288,11 @@ int main(int argc, char *argv[]) control->flags |= FLAG_BZIP2_COMPRESS; break; case 'c': + if (compat) { + control->flags |= FLAG_STDOUT; + break; + } + case 'C': control->flags |= FLAG_CHECK; control->flags |= FLAG_HASH; break; @@ -285,7 +315,7 @@ int main(int argc, char *argv[]) break; case 'h': case '?': - usage(); + usage(compat); return -1; case 'H': control->flags |= FLAG_HASH; @@ -294,6 +324,11 @@ int main(int argc, char *argv[]) control->flags |= FLAG_INFO; break; case 'k': + if (compat) { + control->flags |= FLAG_KEEP_FILES; + break; + } + case 'K': control->flags |= FLAG_KEEP_BROKEN; break; case 'l': @@ -388,6 +423,17 @@ int main(int argc, char *argv[]) failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_ZPAQ_COMPRESS; break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + control->compression_level = c - '0'; + break; } } @@ -482,12 +528,12 @@ int main(int argc, char *argv[]) if (!FORCE_REPLACE) { if (STDIN && isatty(fileno((FILE *)stdin))) { print_err("Will not read stdin from a terminal. Use -f to override.\n"); - usage(); + usage(compat); exit (1); } if (!TEST_ONLY && STDOUT && isatty(fileno((FILE *)stdout))) { print_err("Will not write stdout to a terminal. Use -f to override.\n"); - usage(); + usage(compat); exit (1); } } From efe2ce91ca72cefb5b392e73a54cc702f5d056d2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Mar 2015 13:58:08 +1100 Subject: [PATCH 02/18] Add support for progress, fast and best flags in compat mode --- main.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/main.c b/main.c index 74589b7..9286d67 100644 --- a/main.c +++ b/main.c @@ -80,9 +80,10 @@ static void usage(bool compat) print_output(" -h, -?, --help show help\n"); print_output(" -H, --hash display md5 hash integrity information\n"); print_output(" -i, --info show compressed file information\n"); - if (compat) + if (compat) { + print_output(" -L, --license display software version and license\n"); print_output(" -P, --progress show compression progress\n"); - else + } else print_output(" -q, --quiet don't show compression progress\n"); print_output(" -t, --test test compressed file integrity\n"); print_output(" -v[v], --verbose Increase verbosity\n"); @@ -104,6 +105,11 @@ static void usage(bool compat) print_output(" -n, --no-compress no backend compression - prepare for other compressor\n"); print_output(" -z, --zpaq zpaq compression (best, extreme compression, extremely slow)\n"); print_output("Low level options:\n"); + if (compat) { + print_output(" -1 .. -9 set lzma/bzip2/gzip compression level (1-9, default 7)\n"); + print_output(" --fast alias for -1\n"); + print_output(" --best alias for -9\n"); + } print_output(" -L, --level level set lzma/bzip2/gzip compression level (1-9, default 7)\n"); print_output(" -N, --nice-level value Set nice value to value (default %d)\n", compat ? 0 : 19); print_output(" -p, --threads value Set processor count to override number of threads\n"); @@ -200,37 +206,37 @@ static void show_summary(void) } static struct option long_options[] = { - {"bzip2", no_argument, 0, 'b'}, + {"bzip2", no_argument, 0, 'b'}, /* 0 */ {"check", no_argument, 0, 'c'}, {"check", no_argument, 0, 'C'}, {"decompress", no_argument, 0, 'd'}, {"delete", no_argument, 0, 'D'}, - {"encrypt", no_argument, 0, 'e'}, + {"encrypt", no_argument, 0, 'e'}, /* 5 */ {"force", no_argument, 0, 'f'}, {"gzip", no_argument, 0, 'g'}, {"help", no_argument, 0, 'h'}, {"hash", no_argument, 0, 'H'}, - {"info", no_argument, 0, 'i'}, + {"info", no_argument, 0, 'i'}, /* 10 */ {"keep-broken", no_argument, 0, 'k'}, {"keep-broken", no_argument, 0, 'K'}, {"lzo", no_argument, 0, 'l'}, - {"level", no_argument, 0, 'L'}, - {"maxram", required_argument, 0, 'm'}, + {"level", required_argument, 0, 'L'}, + {"maxram", required_argument, 0, 'm'}, /* 15 */ {"no-compress", no_argument, 0, 'n'}, {"nice-level", required_argument, 0, 'N'}, {"outfile", required_argument, 0, 'o'}, {"outdir", required_argument, 0, 'O'}, - {"threads", required_argument, 0, 'p'}, + {"threads", required_argument, 0, 'p'}, /* 20 */ {"progress", no_argument, 0, 'P'}, {"quiet", no_argument, 0, 'q'}, {"suffix", required_argument, 0, 'S'}, {"test", no_argument, 0, 't'}, - {"threshold", required_argument, 0, 'T'}, + {"threshold", required_argument, 0, 'T'}, /* 25 */ {"unlimited", no_argument, 0, 'U'}, {"verbose", no_argument, 0, 'v'}, {"version", no_argument, 0, 'V'}, {"window", required_argument, 0, 'w'}, - {"zpaq", no_argument, 0, 'z'}, + {"zpaq", no_argument, 0, 'z'}, /* 30 */ {"fast", no_argument, 0, '1'}, {"best", no_argument, 0, '9'}, {0, 0, 0, 0}, @@ -280,7 +286,7 @@ int main(int argc, char *argv[]) else if (!strstr(eptr,"NOCONFIG")) read_config(control); - while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pP:qS:tTUm:vVw:z?123456789", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pPqS:tTUm:vVw:z?123456789", long_options, &i)) != -1) { switch (c) { case 'b': if (control->flags & FLAG_NOT_LZMA) @@ -379,6 +385,9 @@ int main(int argc, char *argv[]) if (control->threads < 1) failure("Must have at least one thread\n"); break; + case 'P': + control->flags |= FLAG_SHOW_PROGRESS; + break; case 'q': control->flags &= ~FLAG_SHOW_PROGRESS; break; From 3cfb993ce38815e4e022baabc6b2f448277550aa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:18:11 +1000 Subject: [PATCH 03/18] Show correct appname when called in compat mode --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index 9286d67..49001b9 100644 --- a/main.c +++ b/main.c @@ -64,7 +64,7 @@ static rzip_control base_control, local_control, *control; static void usage(bool compat) { - print_output("lrzip version %s\n", PACKAGE_VERSION); + print_output("lrz%s version %s\n", compat ? "" : "ip", PACKAGE_VERSION); print_output("Copyright (C) Con Kolivas 2006-2015\n"); print_output("Based on rzip "); print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n"); From 591e106d577728faa8980b8e44baad9319c64d4a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:22:43 +1000 Subject: [PATCH 04/18] Don't show extra message in compat mode decompress --- lrzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lrzip.c b/lrzip.c index 1bf39a3..4a69b6f 100644 --- a/lrzip.c +++ b/lrzip.c @@ -802,7 +802,7 @@ bool decompress_file(rzip_control *control) /* if we get here, no fatal_return(( errors during decompression */ print_progress("\r"); if (!(STDOUT | TEST_ONLY)) - print_output("Output filename is: %s: ", control->outfile); + print_progress("Output filename is: %s: ", control->outfile); if (!expected_size) expected_size = control->st_size; if (!ENCRYPT) From 773df5436cd2f000a85ed4a1a84822b89d81ff3e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:28:01 +1000 Subject: [PATCH 05/18] Fix false warning on compressing from stdin without keep files --- lrzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lrzip.c b/lrzip.c index 4a69b6f..2e9a028 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1217,7 +1217,7 @@ bool compress_file(rzip_control *control) if (TMP_OUTBUF) close_tmpoutbuf(control); - if (!KEEP_FILES) { + if (!KEEP_FILES && !STDIN) { if (unlikely(unlink(control->infile))) fatal_return(("Failed to unlink %s\n", control->infile), false); } From 0005c1cd95cb8c8eeddc8edd37db4e403e45fa39 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:31:23 +1000 Subject: [PATCH 06/18] Fix false warning on decompressing from stdin without keep files --- lrzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lrzip.c b/lrzip.c index 2e9a028..0cfcf27 100644 --- a/lrzip.c +++ b/lrzip.c @@ -820,7 +820,7 @@ bool decompress_file(rzip_control *control) close(fd_in); - if (!KEEP_FILES) { + if (!KEEP_FILES && !STDIN) { if (unlikely(unlink(control->infile))) fatal_return(("Failed to unlink %s\n", infilecopy), false); } From d7ebee964050f19d2d345528099fb1dca2550155 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:47:31 +1000 Subject: [PATCH 07/18] Style police --- lrzip.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lrzip.c b/lrzip.c index 0cfcf27..95e99a9 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1276,13 +1276,13 @@ bool initialise_control(rzip_control *control) } size_t len = strlen(eptr); - control->tmpdir = malloc(len+2); + control->tmpdir = malloc(len + 2); if (control->tmpdir == NULL) fatal_return(("Failed to allocate for tmpdir\n"), false); strcpy(control->tmpdir, eptr); - if (control->tmpdir[len-1] != '/') { + if (control->tmpdir[len - 1] != '/') { control->tmpdir[len] = '/'; /* need a trailing slash */ - control->tmpdir[len+1] = '\0'; + control->tmpdir[len + 1] = '\0'; } return true; } From fbd56ce9c716ea4e3a9970a89c55531da784f01f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 15:59:37 +1000 Subject: [PATCH 08/18] Set STDOUT correctly in compat mode --- main.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/main.c b/main.c index 49001b9..5d0a37c 100644 --- a/main.c +++ b/main.c @@ -242,6 +242,14 @@ static struct option long_options[] = { {0, 0, 0, 0}, }; +static void set_stdout(struct rzip_control *control) +{ + control->flags |= FLAG_STDOUT; + control->outFILE = stdout; + control->msgout = stderr; + register_outputfile(control, control->msgout); +} + int main(int argc, char *argv[]) { bool lrzcat = false, compat = false; @@ -295,7 +303,7 @@ int main(int argc, char *argv[]) break; case 'c': if (compat) { - control->flags |= FLAG_STDOUT; + set_stdout(control); break; } case 'C': @@ -504,15 +512,11 @@ int main(int argc, char *argv[]) if (INFO && STDIN) failure("Will not get file info from STDIN\n"); + /* If no output filename is specified, and we're using + * stdin, use stdout */ if ((control->outname && (strcmp(control->outname, "-") == 0)) || - /* If no output filename is specified, and we're using - * stdin, use stdout */ - (!control->outname && STDIN) || lrzcat ) { - control->flags |= FLAG_STDOUT; - control->outFILE = stdout; - control->msgout = stderr; - register_outputfile(control, control->msgout); - } + (!control->outname && STDIN) || lrzcat) + set_stdout(control); if (lrzcat) { control->msgout = stderr; @@ -540,7 +544,7 @@ int main(int argc, char *argv[]) usage(compat); exit (1); } - if (!TEST_ONLY && STDOUT && isatty(fileno((FILE *)stdout))) { + if (!TEST_ONLY && STDOUT && isatty(fileno((FILE *)stdout)) && !compat) { print_err("Will not write stdout to a terminal. Use -f to override.\n"); usage(compat); exit (1); From 788a70e6f6b22a9d04852881469e2d72053eb118 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 16:20:27 +1000 Subject: [PATCH 09/18] Try /tmp/ if none of the temporary environment directories or the current directory are writeable --- lrzip.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lrzip.c b/lrzip.c index 95e99a9..028a56c 100644 --- a/lrzip.c +++ b/lrzip.c @@ -424,22 +424,38 @@ bool write_fdin(rzip_control *control) /* Open a temporary inputfile to perform stdin decompression */ int open_tmpinfile(rzip_control *control) { - int fd_in; + int fd_in = -1; + /* Use temporary directory if there is one */ if (control->tmpdir) { control->infile = malloc(strlen(control->tmpdir) + 15); if (unlikely(!control->infile)) fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, control->tmpdir); strcat(control->infile, "lrzipin.XXXXXX"); - } else { - control->infile = malloc(15); + fd_in = mkstemp(control->infile); + } + + /* Try the current directory */ + if (fd_in == -1) { + free(control->infile); + control->infile = malloc(16); if (unlikely(!control->infile)) fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, "lrzipin.XXXXXX"); + fd_in = mkstemp(control->infile); + } + + /* Use /tmp if nothing is writeable so far */ + if (fd_in == -1) { + free(control->infile); + control->infile = malloc(20); + if (unlikely(!control->infile)) + fatal_return(("Failed to allocate infile name\n"), -1); + strcpy(control->infile, "/tmp/lrzipin.XXXXXX"); + fd_in = mkstemp(control->infile); } - fd_in = mkstemp(control->infile); if (unlikely(fd_in == -1)) fatal_return(("Failed to create in tmpfile: %s\n", control->infile), -1); register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); From fcb64e6dbbc19dd4403f2f24038faa82f38d6633 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 16:38:20 +1000 Subject: [PATCH 10/18] Do not fail if we are unable to write temporary files, giving a warning only that it might fail if we don't have enough ram --- lrzip.c | 29 ++++++++++++++++------------- stream.c | 3 ++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lrzip.c b/lrzip.c index 028a56c..80ddd81 100644 --- a/lrzip.c +++ b/lrzip.c @@ -311,8 +311,8 @@ int open_tmpoutfile(rzip_control *control) fd_out = mkstemp(control->outfile); if (fd_out == -1) { - print_verbose("WARNING: Failed to create out tmpfile: %s , will fail if cannot perform entirely in ram\n", - control->outfile, DECOMPRESS ? "de" : ""); + print_progress("WARNING: Failed to create out tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n", + control->outfile, DECOMPRESS ? "de" : ""); } else register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); return fd_out; @@ -456,15 +456,18 @@ int open_tmpinfile(rzip_control *control) fd_in = mkstemp(control->infile); } - if (unlikely(fd_in == -1)) - fatal_return(("Failed to create in tmpfile: %s\n", control->infile), -1); - register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); - /* Unlink temporary file immediately to minimise chance of files left - * lying around in cases of failure_return((. */ - if (unlikely(unlink(control->infile))) { - fatal("Failed to unlink tmpfile: %s\n", control->infile); - close(fd_in); - return -1; + if (fd_in == -1) { + print_progress("WARNING: Failed to create in tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n", + control->infile, DECOMPRESS ? "de" : ""); + } else { + register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); + /* Unlink temporary file immediately to minimise chance of files left + * lying around in cases of failure_return((. */ + if (unlikely(unlink(control->infile))) { + fatal("Failed to unlink tmpfile: %s\n", control->infile); + close(fd_in); + return -1; + } } return fd_in; } @@ -490,6 +493,8 @@ bool read_tmpinfile(rzip_control *control, int fd_in) FILE *tmpinfp; int tmpchar; + if (fd_in == -1) + return false; if (control->flags & FLAG_SHOW_PROGRESS) fprintf(control->msgout, "Copying from stdin.\n"); tmpinfp = fdopen(fd_in, "w+"); @@ -719,8 +724,6 @@ bool decompress_file(rzip_control *control) if (STDIN) { fd_in = open_tmpinfile(control); - if (unlikely(fd_in == -1)) - return false; read_tmpinmagic(control); if (ENCRYPT) failure_return(("Cannot decompress encrypted file from STDIN\n"), false); diff --git a/stream.c b/stream.c index c2784ed..da47928 100644 --- a/stream.c +++ b/stream.c @@ -703,7 +703,8 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) /* We're decompressing from STDIN */ if (unlikely(control->in_ofs + len > control->in_maxlen)) { /* We're unable to fit it all into the temp buffer */ - dump_stdin(control); + if (dump_stdin(control)) + failure_return(("Inadequate ram to %compress from STDIN and unable to create in tmpfile"), -1); goto read_fd; } if (control->in_ofs + len > control->in_len) { From d9cbf03a455af8cc811c917c8e15c5c6e3b84717 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 16:44:01 +1000 Subject: [PATCH 11/18] Fix lrz symbolic linkage --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 1407d72..f9907bf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -107,7 +107,7 @@ install-exec-hook: $(LN_S) -f lrzip$(EXEEXT) $(DESTDIR)$(bindir)/lrunzip$(EXEEXT) $(LN_S) -f lrzip$(EXEEXT) $(DESTDIR)$(bindir)/lrzcat$(EXEEXT) $(LN_S) -f lrztar$(EXEEXT) $(DESTDIR)$(bindir)/lrzuntar$(EXEEXT) - $(LN_S) -f lrz$(EXEEXT) $(DESTDIR)$(bindir)/lrz$(EXEEXT) + $(LN_S) -f lrzip$(EXEEXT) $(DESTDIR)$(bindir)/lrz$(EXEEXT) uninstall-local: From f7ae8fddf911afb8170d833e3a5f5ee87b595053 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 17:21:54 +1000 Subject: [PATCH 12/18] Add rudimentary manpage for lrz --- man/Makefile.am | 8 +-- man/lrunzip.1.pod | 3 +- man/lrz.1.pod | 128 +++++++++++++++++++++++++++++++++++++++++++++ man/lrzcat.1.pod | 3 +- man/lrzip.1 | 5 +- man/lrztar.1.pod | 3 +- man/lrzuntar.1.pod | 3 +- 7 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 man/lrz.1.pod diff --git a/man/Makefile.am b/man/Makefile.am index acb4df0..f4df115 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,12 +1,12 @@ -MAINTAINERCLEANFILES = Makefile.in lrunzip.1 lrztar.1 lrzuntar.1 +MAINTAINERCLEANFILES = Makefile.in lrunzip.1 lrztar.1 lrzuntar.1 lrz.1 -man1_MANS = lrzip.1 lrunzip.1 lrzcat.1 lrztar.1 lrzuntar.1 +man1_MANS = lrzip.1 lrunzip.1 lrzcat.1 lrztar.1 lrzuntar.1 lrz.1 man5_MANS = lrzip.conf.5 -BUILT_SOURCES = lrunzip.1 lrzcat.1 lrztar.1 lrzuntar.1 +BUILT_SOURCES = lrunzip.1 lrzcat.1 lrztar.1 lrzuntar.1 lrz.1 CLEANFILES = $(BUILT_SOURCES) -EXTRA_DIST = lrzip.1 lrunzip.1.pod lrzcat.1.pod lrztar.1.pod lrzuntar.1.pod $(man5_MANS) +EXTRA_DIST = lrzip.1 lrunzip.1.pod lrzcat.1.pod lrztar.1.pod lrzuntar.1.pod lrz.1.pod $(man5_MANS) SUFFIXES = .1 .1.pod .1.pod.1: diff --git a/man/lrunzip.1.pod b/man/lrunzip.1.pod index f5869e2..53c220e 100644 --- a/man/lrunzip.1.pod +++ b/man/lrunzip.1.pod @@ -1,6 +1,6 @@ # Copyright # -# Copyright (C) 2010-2011 Con Kolivas +# Copyright (C) 2010-2015 Con Kolivas # Copyright (C) 2009-2009 Jari Aalto # # License @@ -70,6 +70,7 @@ lrzip(1), lrzcat(1), lrztar(1), lrzuntar(1), +lrz(1), bzip2(1), gzip(1), lzop(1), diff --git a/man/lrz.1.pod b/man/lrz.1.pod new file mode 100644 index 0000000..2ac4746 --- /dev/null +++ b/man/lrz.1.pod @@ -0,0 +1,128 @@ +# Copyright +# +# Copyright (C) 2015 Con Kolivas +# +# License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Description +# +# To learn what TOP LEVEL section to use in manual pages, +# see POSIX/Susv standard and "tility Description Defaults" at +# http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap01.html#tag_01_11 +# +# This is manual page in Perl POD format. Read more at +# http://perldoc.perl.org/perlpod.html or run command: +# +# perldoc perlpod | less +# +# To check the syntax: +# +# podchecker *.pod +# +# Create manual page with command: +# +# pod2man PAGE.N.pod > PAGE.N + +=pod + +=head1 NAME + +lrz - gzip compatible command line variant of lrzip + +=head1 SYNOPSIS + + lrz [OPTIONS] + +=head1 DESCRIPTION + +lrz is identical to the lrzip application however its command line options are +made to be as compatible with gzip as possible. + +=head1 OPTIONS + +lrz differs from lrzip by accepting the following commands: + +General options: + -c, --stdout output to STDOUT + -C, --check check integrity of file written on decompression + -d, --decompress decompress + -e, --encrypt password protected sha512/aes128 encryption on compression + -h, -?, --help show help + -H, --hash display md5 hash integrity information + -i, --info show compressed file information + -L, --license display software version and license + -P, --progress show compression progress + -t, --test test compressed file integrity + -v[v], --verbose Increase verbosity + -V, --version show version +Options affecting output: + -f, --force force overwrite of any existing files + -k, --keep don't delete source files on de/compression + -K, --keep-broken keep broken or damaged output files + -o, --outfile filename specify the output file name and/or path + -O, --outdir directory specify the output directory when -o is not used + -S, --suffix suffix specify compressed suffix (default '.lrz') +Options affecting compression: + -b, --bzip2 bzip2 compression + -g, --gzip gzip compression using zlib + -l, --lzo lzo compression (ultra fast) + -n, --no-compress no backend compression - prepare for other compressor + -z, --zpaq zpaq compression (best, extreme compression, extremely slow) +Low level options: + -1 .. -9 set lzma/bzip2/gzip compression level (1-9, default 7) + --fast alias for -1 + --best alias for -9 + -L, --level level set lzma/bzip2/gzip compression level (1-9, default 7) + -N, --nice-level value Set nice value to value (default 0) + -p, --threads value Set processor count to override number of threads + -m, --maxram size Set maximim available ram in hundreds of MB + overrides detected ammount of available ram + -T, --threshold Disable LZO compressibility testing + -U, --unlimited Use unlimited window size beyond ramsize (potentially much slower) + -w, --window size maximum compression window in hundreds of MB + default chosen by heuristic dependent on ram and chosen compression + +See also lrzip(1) + +=head1 ENVIRONMENT + +lrz uses the same environment and configuration files as lrzip(1) + +=head1 FILES + +See lrzip(1) + +=head1 SEE ALSO + +lrzip.conf(5), +lrzip(1), +lrunzip(1), +lrztar(1), +lrzuntar(1), +bzip2(1), +gzip(1), +lzop(1), +rzip(1), +zip(1) + +=head1 AUTHORS + +This manual page was written by Con Kolivas (but +may be used by others). Released under license GNU GPL version 2 or (at +your option) any later version. For more information about license, +visit . + +=cut diff --git a/man/lrzcat.1.pod b/man/lrzcat.1.pod index d523f1a..d41de2a 100644 --- a/man/lrzcat.1.pod +++ b/man/lrzcat.1.pod @@ -1,6 +1,6 @@ # Copyright # -# Copyright (C) 2011 Con Kolivas +# Copyright (C) 2011-2015 Con Kolivas # # License # @@ -69,6 +69,7 @@ lrzip(1), lrunzip(1), lrztar(1), lrzuntar(1), +lrz(1), bzip2(1), gzip(1), lzop(1), diff --git a/man/lrzip.1 b/man/lrzip.1 index 1721ec3..741d461 100644 --- a/man/lrzip.1 +++ b/man/lrzip.1 @@ -1,4 +1,4 @@ -.TH "lrzip" "1" "May 2011" "" "" +.TH "lrzip" "1" "April 2015" "" "" .SH "NAME" lrzip \- a large-file compression program .SH "SYNOPSIS" @@ -17,6 +17,8 @@ lrztar \-d [lrzip options] .br lrzuntar [lrzip options] .br +lrz [lrz options] +.br LRZIP=NOCONFIG [lrzip|lrunzip] [OPTIONS] .PP .SH "DESCRIPTION" @@ -331,6 +333,7 @@ lrunzip(1), lrzcat(1), lrztar(1), lrzuntar(1), +lrz(1), bzip2(1), gzip(1), lzop(1), diff --git a/man/lrztar.1.pod b/man/lrztar.1.pod index 9dcca94..dd43e2c 100644 --- a/man/lrztar.1.pod +++ b/man/lrztar.1.pod @@ -1,6 +1,6 @@ # Copyright # -# Copyright (C) 2010-2011 Con Kolivas +# Copyright (C) 2010-2015 Con Kolivas # Copyright (C) 2009-2010 Jari Aalto # # License @@ -74,6 +74,7 @@ lrzuntar(1), lrzip(1), lrunzip(1), lrzcat(1), +lrz(1), bzip2(1), gzip(1), lzop(1), diff --git a/man/lrzuntar.1.pod b/man/lrzuntar.1.pod index a070910..2f1d638 100644 --- a/man/lrzuntar.1.pod +++ b/man/lrzuntar.1.pod @@ -1,6 +1,6 @@ # Copyright # -# Copyright (C) 2010-2011 Con Kolivas +# Copyright (C) 2010-2015 Con Kolivas # # License # @@ -52,6 +52,7 @@ lrztar(1), lrzip(1), lrunzip(1), lrzcat(1), +lrz(1), bzip2(1), gzip(1), lzop(1), From bed16a4272c495b9cb37d0adc65a9837678b29c6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 16 Apr 2015 17:23:35 +1000 Subject: [PATCH 13/18] Cosmetic help change for compat --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index 5d0a37c..429b9aa 100644 --- a/main.c +++ b/main.c @@ -68,7 +68,7 @@ static void usage(bool compat) print_output("Copyright (C) Con Kolivas 2006-2015\n"); print_output("Based on rzip "); print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n"); - print_output("Usage: lrzip [options] \n"); + print_output("Usage: lrz%s [options] \n", compat ? "" : "ip"); print_output("General options:\n"); if (compat) { print_output(" -c, --stdout output to STDOUT\n"); From e39af7eb0ff2929a1655bd473f44eb2593374b3f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 1 May 2015 12:47:03 +1000 Subject: [PATCH 14/18] Add sanity check to prevent trying to malloc more ram than a system/environment is capable of --- stream.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream.c b/stream.c index da47928..c705407 100644 --- a/stream.c +++ b/stream.c @@ -1625,6 +1625,8 @@ fill_another: sinfo->total_read += padded_len; fsync(control->fd_out); + if (unlikely(u_len > control->maxram)) + fatal_return(("Unable to malloc buffer of size %lld in this environment\n", u_len), -1); s_buf = malloc(MAX(u_len, MIN_SIZE)); if (unlikely(u_len && !s_buf)) fatal_return(("Unable to malloc buffer of size %lld in fill_buffer\n", u_len), -1); From a01b6615f339ce2f6220725bb9a8308cb356f900 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 09:47:13 +1000 Subject: [PATCH 15/18] Big endian fix for Solaris Sparc courtesy of joelfredrikson. --- lrzip_private.h | 4 ++++ rzip.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lrzip_private.h b/lrzip_private.h index 4a79918..fa29f44 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -67,6 +67,10 @@ void *alloca (size_t); # include #endif #ifndef __BYTE_ORDER +# ifndef __BIG_ENDIAN +# define __BIG_ENDIAN 4321 +# define __LITTLE_ENDIAN 1234 +# endif # ifdef WORDS_BIGENDIAN # define __BYTE_ORDER __BIG_ENDIAN # else diff --git a/rzip.c b/rzip.c index 38d7fc7..0320515 100644 --- a/rzip.c +++ b/rzip.c @@ -772,7 +772,7 @@ static inline void init_hash_indexes(struct rzip_state *st) st->hash_index[i] = ((random() << 16) ^ random()); } -#if defined(__APPLE__) || defined(__FreeBSD__) +#if !defined(__linux) # define mremap fake_mremap static inline void *fake_mremap(void *old_address, size_t old_size, size_t new_size, int flags __UNUSED__) From 9a351d1b505002f15b77bf108928f823fa152cd0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 18:02:13 +1000 Subject: [PATCH 16/18] Add support for various combinations in compat mode --- main.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.c b/main.c index 429b9aa..37dc7b1 100644 --- a/main.c +++ b/main.c @@ -303,6 +303,7 @@ int main(int argc, char *argv[]) break; case 'c': if (compat) { + control->flags |= FLAG_KEEP_FILES; set_stdout(control); break; } @@ -409,6 +410,8 @@ int main(int argc, char *argv[]) case 't': if (control->outname) failure("Cannot specify an output file name when just testing.\n"); + if (compat) + control->flags |= FLAG_KEEP_FILES; if (!KEEP_FILES) failure("Doubt that you want to delete a file when just testing.\n"); control->flags |= FLAG_TEST_ONLY; @@ -421,7 +424,9 @@ int main(int argc, char *argv[]) break; case 'v': /* set verbosity flag */ - if (!(control->flags & FLAG_VERBOSITY) && !(control->flags & FLAG_VERBOSITY_MAX)) + if (!(control->flags & FLAG_SHOW_PROGRESS)) + control->flags |= FLAG_SHOW_PROGRESS; + else if (!(control->flags & FLAG_VERBOSITY) && !(control->flags & FLAG_VERBOSITY_MAX)) control->flags |= FLAG_VERBOSITY; else if ((control->flags & FLAG_VERBOSITY)) { control->flags &= ~FLAG_VERBOSITY; From 2021d5e8623844a66bf7a8cab10b6774a82d92f0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 18:14:43 +1000 Subject: [PATCH 17/18] Add one more verbose for compat mode --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index 37dc7b1..b149717 100644 --- a/main.c +++ b/main.c @@ -86,7 +86,7 @@ static void usage(bool compat) } else print_output(" -q, --quiet don't show compression progress\n"); print_output(" -t, --test test compressed file integrity\n"); - print_output(" -v[v], --verbose Increase verbosity\n"); + print_output(" -v[v%s], --verbose Increase verbosity\n", compat ? "v" : ""); print_output(" -V, --version show version\n"); print_output("Options affecting output:\n"); if (!compat) From 32c7fa82a21da2e04f0310b684afe2fabf38215c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 18:18:14 +1000 Subject: [PATCH 18/18] Tidy --- lrzip.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lrzip.c b/lrzip.c index 43144b5..ffdb85d 100644 --- a/lrzip.c +++ b/lrzip.c @@ -788,9 +788,10 @@ bool decompress_file(rzip_control *control) if (unlikely(!open_tmpoutbuf(control))) return false; - if (!STDIN) + if (!STDIN) { if (unlikely(!read_magic(control, fd_in, &expected_size))) return false; + } if (!STDOUT && !TEST_ONLY) { /* Check if there's enough free space on the device chosen to fit the @@ -960,7 +961,8 @@ bool get_fileinfo(rzip_control *control) infile_size = st.st_size; /* Get decompressed size */ - if (unlikely(!read_magic(control, fd_in, &expected_size))) goto error; + if (unlikely(!read_magic(control, fd_in, &expected_size))) + goto error; if (ENCRYPT) { print_output("Encrypted lrzip archive. No further information available\n"); @@ -1220,10 +1222,14 @@ bool compress_file(rzip_control *control) fatal_goto(("Failed to create %s\n", control->outfile), error); } control->fd_out = fd_out; - if (!STDIN) - if (unlikely(!preserve_perms(control, fd_in, fd_out))) goto error; - } else - if (unlikely(!open_tmpoutbuf(control))) goto error; + if (!STDIN) { + if (unlikely(!preserve_perms(control, fd_in, fd_out))) + goto error; + } + } else { + if (unlikely(!open_tmpoutbuf(control))) + goto error; + } /* Write zeroes to header at beginning of file */ if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header))) @@ -1232,8 +1238,10 @@ bool compress_file(rzip_control *control) rzip_fd(control, fd_in, fd_out); /* Wwrite magic at end b/c lzma does not tell us properties until it is done */ - if (!STDOUT) - if (unlikely(!write_magic(control))) goto error; + if (!STDOUT) { + if (unlikely(!write_magic(control))) + goto error; + } if (ENCRYPT) release_hashes(control);