diff --git a/ChangeLog b/ChangeLog index 0e78f51..1d5bd86 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,26 @@ lrzip ChangeLog +MARCH 2011, version 0.571 Con Kolivas +* Only retry mmaping if it's a memory error, otherwise it may give spurious +errors. +* Check for free space before compression/decompression and abort if there +is inadequate free space if the -f option is not passed. +* Fix the wrong check in rzip.c which was rounding down the page size and +making for one extra small chunk at the end. +* Check the correct stdout when refusing to pipe to a terminal. +* Fix windows EOL on lzma.txt. +* Ignore what stdout is going to when testing from stdin. +* More verbose summary after we know whether we have stdin/out to more +accurately reflect the window that will be used. +* Updated gitignore +* Unlink temporary files immediately to avoid files lying around. +* Check free space AFTER reading magic, and not when decompressing to stdout. +* Don't dump output to stdout when just testing a file. +* Dump the temporary file generated on emulating stdout on decompression after +every chunk is decompressed instead of after the whole file is decompressed. + +MARCH 2011, Michael Blumenkrantz +* Updated autotools/conf build system. + FEBRUARY 2011, version 0.570 Con Kolivas * Change the lzo testing to a bool on/off instead of taking a parameter. * Clean up the messy help output. diff --git a/WHATS-NEW b/WHATS-NEW index 534b6d9..4062126 100644 --- a/WHATS-NEW +++ b/WHATS-NEW @@ -1,4 +1,20 @@ -lrzip-0.561 +lrzip-0.571 + +A new build configuration system. +Avoid spurious errors on failing to mmap a file. +Fee space will now be checked to ensure there is enough room for the +compressed or decompressed file and lrzip will abort unless the -f option is +passed to it. +The extra little chunk at the end of every large file should now be fixed. +The file lzma.txt now has unix end-of-lines. +There will be a more accurate summary of what compression window will be used +when lrzip is invoked with STDIN/STDOUT. +STDIN will now be able to show estimated time to completion and percentage +complete once lrzip knows how much file is left. +Temporary files are much less likely to be left lying around. +Less temporary file space will be used when decompressing to stdout. + +lrzip-0.570 Multi-threaded performance has been improved with a significant speed-up on both compression and decompression. New benchmark results have been added to diff --git a/main.c b/main.c index 7065172..b5ed5de 100644 --- a/main.c +++ b/main.c @@ -167,7 +167,7 @@ static int open_tmpoutfile(void) if (STDOUT) print_verbose("Outputting to stdout.\n"); if (control.tmpdir) { - control.outfile = realloc(NULL, strlen(control.tmpdir)+16); + control.outfile = realloc(NULL, strlen(control.tmpdir) + 16); if (unlikely(!control.outfile)) fatal("Failed to allocate outfile name\n"); strcpy(control.outfile, control.tmpdir); @@ -181,17 +181,17 @@ static int open_tmpoutfile(void) fd_out = mkstemp(control.outfile); if (unlikely(fd_out == -1)) - fatal("Failed to create out tmpfile: %s\n", strerror(errno)); + fatal("Failed to create out tmpfile: %s\n", control.outfile); return fd_out; } /* Dump temporary outputfile to perform stdout */ -static void dump_tmpoutfile(int fd_out) +void dump_tmpoutfile(int fd_out) { FILE *tmpoutfp; int tmpchar; - print_progress("Dumping to stdout.\n"); + print_verbose("Dumping temporary file to stdout.\n"); /* flush anything not yet in the temporary file */ fsync(fd_out); tmpoutfp = fdopen(fd_out, "r"); @@ -199,10 +199,15 @@ static void dump_tmpoutfile(int fd_out) fatal("Failed to fdopen out tmpfile: %s\n", strerror(errno)); rewind(tmpoutfp); - while ((tmpchar = fgetc(tmpoutfp)) != EOF) - putchar(tmpchar); + if (!TEST_ONLY) { + while ((tmpchar = fgetc(tmpoutfp)) != EOF) + putchar(tmpchar); + } + fflush(stdout); - fflush(control.msgout); + rewind(tmpoutfp); + if (unlikely(ftruncate(fd_out, 0))) + fatal("Failed to ftruncate fd_out in dump_tmpoutfile\n"); } /* Open a temporary inputfile to perform stdin decompression */ @@ -211,7 +216,7 @@ static int open_tmpinfile(void) int fd_in; if (control.tmpdir) { - control.infile = malloc(strlen(control.tmpdir)+15); + control.infile = malloc(strlen(control.tmpdir) + 15); if (unlikely(!control.infile)) fatal("Failed to allocate infile name\n"); strcpy(control.infile, control.tmpdir); @@ -225,7 +230,11 @@ static int open_tmpinfile(void) fd_in = mkstemp(control.infile); if (unlikely(fd_in == -1)) - fatal("Failed to create in tmpfile: %s\n", strerror(errno)); + fatal("Failed to create in tmpfile: %s\n", control.infile); + /* Unlink temporary file immediately to minimise chance of files left + * lying around in cases of failure. */ + if (unlikely(unlink(control.infile))) + fatal("Failed to unlink tmpfile: %s\n", control.infile); return fd_in; } @@ -338,19 +347,11 @@ static void decompress_file(void) fd_out = open_tmpoutfile(); control.fd_out = fd_out; - if (control.tmp_outfile) - fd_hist = shm_open(control.tmp_outfile, O_RDONLY, 0777); - else - fd_hist = open(control.outfile, O_RDONLY); - if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control.outfile); - control.fd_hist = fd_hist; - read_magic(fd_in, &expected_size); - /* Check if there's enough free space on the device chosen to fit the - * decompressed file. */ if (!STDOUT) { + /* Check if there's enough free space on the device chosen to fit the + * decompressed file. */ if (unlikely(fstatvfs(fd_out, &fbuf))) fatal("Failed to fstatvfs in decompress_file\n"); free_space = fbuf.f_bsize * fbuf.f_bavail; @@ -358,11 +359,19 @@ static void decompress_file(void) if (FORCE_REPLACE) print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n"); else - failure("Inadequate free space to decompress file, use -f to override." - " Free space %lld, expected size %lld\n", free_space, expected_size); + failure("Inadequate free space to decompress file, use -f to override.\n"); } } + fd_hist = open(control.outfile, O_RDONLY); + if (unlikely(fd_hist == -1)) + fatal("Failed to open history file %s\n", control.outfile); + control.fd_hist = fd_hist; + + /* Unlink temporary file as soon as possible */ + if (unlikely((STDOUT || TEST_ONLY) && unlink(control.outfile))) + fatal("Failed to unlink tmpfile: %s\n", control.outfile); + if (NO_MD5) print_verbose("Not performing MD5 hash check\n"); if (HAS_MD5) @@ -387,15 +396,9 @@ static void decompress_file(void) if (unlikely(close(fd_hist) || close(fd_out))) fatal("Failed to close files\n"); - if (TEST_ONLY | STDOUT) { - /* Delete temporary files generated for testing or faking stdout */ - if (unlikely(unlink(control.outfile))) - fatal("Failed to unlink tmpfile: %s\n", strerror(errno)); - } - close(fd_in); - if (!(KEEP_FILES | TEST_ONLY) || STDIN) { + if (!KEEP_FILES) { if (unlikely(unlink(control.infile))) fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno)); } @@ -600,12 +603,8 @@ next_chunk: ctotal, expected_size); } - if (STDIN) { - if (unlikely(unlink(control.infile))) - fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno)); - } else - if (unlikely(close(fd_in))) - fatal("Failed to close fd_in in get_fileinfo\n"); + if (unlikely(close(fd_in))) + fatal("Failed to close fd_in in get_fileinfo\n"); free(control.outfile); free(infilecopy); @@ -688,6 +687,9 @@ static void compress_file(void) fd_out = open_tmpoutfile(); control.fd_out = fd_out; + if (unlikely(STDOUT && unlink(control.outfile))) + fatal("Failed to unlink tmpfile: %s\n", control.outfile); + preserve_perms(fd_in, fd_out); /* write zeroes to 24 bytes at beginning of file */ @@ -705,12 +707,6 @@ static void compress_file(void) if (unlikely(close(fd_in) || close(fd_out))) fatal("Failed to close files\n"); - if (STDOUT) { - /* Delete temporary files generated for testing or faking stdout */ - if (unlikely(unlink(control.outfile))) - fatal("Failed to unlink tmpfile: %s\n", strerror(errno)); - } - if (!KEEP_FILES) { if (unlikely(unlink(control.infile))) fatal("Failed to unlink %s: %s\n", control.infile, strerror(errno)); @@ -1002,6 +998,7 @@ int main(int argc, char *argv[]) /* Decrease usable ram size on 32 bits due to kernel/userspace split */ if (BITS32) control.ramsize = MAX(control.ramsize - 900000000ll, 900000000ll); + control.maxram = control.ramsize / 3; /* Set the main nice value to half that of the backend threads since * the rzip stage is usually the rate limiting step */ diff --git a/runzip.c b/runzip.c index 49f9c59..9922523 100644 --- a/runzip.c +++ b/runzip.c @@ -242,8 +242,11 @@ i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size) md5_init_ctx (&control.ctx); gettimeofday(&start,NULL); - while (total < expected_size) + while (total < expected_size) { total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total); + if (STDOUT) + dump_tmpoutfile(fd_out); + } gettimeofday(&end,NULL); print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n", diff --git a/rzip.c b/rzip.c index 0462c8f..7b43a35 100644 --- a/rzip.c +++ b/rzip.c @@ -766,8 +766,8 @@ void rzip_fd(int fd_in, int fd_out) /* Check if there's enough free space on the device chosen to fit the * compressed file, based on the compressed file being as large as the * uncompressed file. */ - if (unlikely(fstatvfs(fd_in, &fbuf))) - fatal("Failed to fstatvfs in decompress_file\n"); + if (unlikely(fstatvfs(fd_out, &fbuf))) + fatal("Failed to fstatvfs in compress_file\n"); free_space = fbuf.f_bsize * fbuf.f_bavail; if (free_space < control.st_size) { if (FORCE_REPLACE) @@ -780,7 +780,7 @@ void rzip_fd(int fd_in, int fd_out) * allocate 1/3 of it to the main buffer and use a sliding mmap * buffer to work on 2/3 ram size, leaving enough ram for the * compression backends */ - control.max_mmap = control.ramsize / 3; + control.max_mmap = control.maxram; /* On 32 bits we can have a big window with sliding mmap, but can * not enable much per mmap/malloc */ diff --git a/rzip.h b/rzip.h index 175e2aa..f2c5ceb 100644 --- a/rzip.h +++ b/rzip.h @@ -270,6 +270,7 @@ struct rzip_control { const char *suffix; int compression_level; i64 overhead; // compressor overhead + i64 maxram; // the largest chunk of ram to allocate unsigned char lzma_properties[5]; // lzma properties, encoded i64 window; unsigned long flags; @@ -333,6 +334,7 @@ const i64 two_gig; void prepare_streamout_threads(void); void close_streamout_threads(void); void round_to_page(i64 *size); +void dump_tmpoutfile(int fd_out); #define print_err(format, args...) do {\ fprintf(stderr, format, ##args); \ diff --git a/stream.c b/stream.c index ff3b2a5..dbcd001 100644 --- a/stream.c +++ b/stream.c @@ -763,18 +763,18 @@ void close_streamout_threads(void) /* open a set of output streams, compressing with the given compression level and algorithm */ -void *open_stream_out(int f, int n, i64 limit, char cbytes) +void *open_stream_out(int f, int n, i64 chunk_limit, char cbytes) { struct stream_info *sinfo; + i64 testsize, limit; uchar *testmalloc; - i64 testsize; int i, testbufs; sinfo = calloc(sizeof(struct stream_info), 1); if (unlikely(!sinfo)) return NULL; - sinfo->bufsize = limit; + sinfo->bufsize = limit = chunk_limit; sinfo->chunk_bytes = cbytes; sinfo->num_streams = n; @@ -802,8 +802,19 @@ void *open_stream_out(int f, int n, i64 limit, char cbytes) (control.overhead * control.threads)); testsize = (limit * testbufs) + (control.overhead * control.threads); - if (testsize > control.ramsize / 3) - limit = (control.ramsize / 3 - (control.overhead * control.threads)) / testbufs; + if (testsize > control.maxram) + limit = (control.maxram - (control.overhead * control.threads)) / testbufs; + + /* If we don't have enough ram for the number of threads, decrease the + * number of threads till we do, or only have one thread. */ + while (limit < STREAM_BUFSIZE && limit < chunk_limit) { + if (control.threads > 1) + --control.threads; + else + break; + limit = (control.maxram - (control.overhead * control.threads)) / testbufs; + limit = MIN(limit, chunk_limit); + } retest_malloc: testsize = (limit * testbufs) + (control.overhead * control.threads); testmalloc = malloc(testsize);