Merge branch 'temp'

Conflicts:
	main.c

ChangeLog

@@ -1,4 +1,26 @@
 lrzip ChangeLog
+MARCH 2011, version 0.571 Con Kolivas
+* Only retry mmaping if it's a memory error, otherwise it may give spurious
+errors.
+* Check for free space before compression/decompression and abort if there
+is inadequate free space if the -f option is not passed.
+* Fix the wrong check in rzip.c which was rounding down the page size and
+making for one extra small chunk at the end.
+* Check the correct stdout when refusing to pipe to a terminal.
+* Fix windows EOL on lzma.txt.
+* Ignore what stdout is going to when testing from stdin.
+* More verbose summary after we know whether we have stdin/out to more
+accurately reflect the window that will be used.
+* Updated gitignore
+* Unlink temporary files immediately to avoid files lying around.
+* Check free space AFTER reading magic, and not when decompressing to stdout.
+* Don't dump output to stdout when just testing a file.
+* Dump the temporary file generated on emulating stdout on decompression after
+every chunk is decompressed instead of after the whole file is decompressed.
+
+MARCH 2011, Michael Blumenkrantz
+* Updated autotools/conf build system.
+
 FEBRUARY 2011, version 0.570 Con Kolivas
 * Change the lzo testing to a bool on/off instead of taking a parameter.
 * Clean up the messy help output.

WHATS-NEW

@@ -1,4 +1,20 @@
-lrzip-0.561
+lrzip-0.571
+
+A new build configuration system.
+Avoid spurious errors on failing to mmap a file.
+Fee space will now be checked to ensure there is enough room for the
+compressed or decompressed file and lrzip will abort unless the -f option is
+passed to it.
+The extra little chunk at the end of every large file should now be fixed.
+The file lzma.txt now has unix end-of-lines.
+There will be a more accurate summary of what compression window will be used
+when lrzip is invoked with STDIN/STDOUT.
+STDIN will now be able to show estimated time to completion and percentage
+complete once lrzip knows how much file is left.
+Temporary files are much less likely to be left lying around.
+Less temporary file space will be used when decompressing to stdout.
+
+lrzip-0.570
 
 Multi-threaded performance has been improved with a significant speed-up on
 both compression and decompression. New benchmark results have been added to

main.c

@@ -167,7 +167,7 @@ static int open_tmpoutfile(void)
 	if (STDOUT)
 		print_verbose("Outputting to stdout.\n");
 	if (control.tmpdir) {
-		control.outfile = realloc(NULL, strlen(control.tmpdir)+16);
+		control.outfile = realloc(NULL, strlen(control.tmpdir) + 16);
 		if (unlikely(!control.outfile))
 			fatal("Failed to allocate outfile name\n");
 		strcpy(control.outfile, control.tmpdir);
@@ -181,17 +181,17 @@ static int open_tmpoutfile(void)
 
 	fd_out = mkstemp(control.outfile);
 	if (unlikely(fd_out == -1))
-		fatal("Failed to create out tmpfile: %s\n", strerror(errno));
+		fatal("Failed to create out tmpfile: %s\n", control.outfile);
 	return fd_out;
 }
 
 /* Dump temporary outputfile to perform stdout */
-static void dump_tmpoutfile(int fd_out)
+void dump_tmpoutfile(int fd_out)
 {
 	FILE *tmpoutfp;
 	int tmpchar;
 
-	print_progress("Dumping to stdout.\n");
+	print_verbose("Dumping temporary file to stdout.\n");
 	/* flush anything not yet in the temporary file */
 	fsync(fd_out);
 	tmpoutfp = fdopen(fd_out, "r");
@@ -199,10 +199,15 @@ static void dump_tmpoutfile(int fd_out)
 		fatal("Failed to fdopen out tmpfile: %s\n", strerror(errno));
 	rewind(tmpoutfp);
 
-	while ((tmpchar = fgetc(tmpoutfp)) != EOF)
-		putchar(tmpchar);
+	if (!TEST_ONLY) {
+		while ((tmpchar = fgetc(tmpoutfp)) != EOF)
+			putchar(tmpchar);
+	}
+	fflush(stdout);
 
-	fflush(control.msgout);
+	rewind(tmpoutfp);
+	if (unlikely(ftruncate(fd_out, 0)))
+		fatal("Failed to ftruncate fd_out in dump_tmpoutfile\n");
 }
 
 /* Open a temporary inputfile to perform stdin decompression */
@@ -211,7 +216,7 @@ static int open_tmpinfile(void)
 	int fd_in;
 
 	if (control.tmpdir) {
-		control.infile = malloc(strlen(control.tmpdir)+15);
+		control.infile = malloc(strlen(control.tmpdir) + 15);
 		if (unlikely(!control.infile))
 			fatal("Failed to allocate infile name\n");
 		strcpy(control.infile, control.tmpdir);
@@ -225,7 +230,11 @@ static int open_tmpinfile(void)
 
 	fd_in = mkstemp(control.infile);
 	if (unlikely(fd_in == -1))
-		fatal("Failed to create in tmpfile: %s\n", strerror(errno));
+		fatal("Failed to create in tmpfile: %s\n", control.infile);
+	/* Unlink temporary file immediately to minimise chance of files left
+	 * lying around in cases of failure. */
+	if (unlikely(unlink(control.infile)))
+		fatal("Failed to unlink tmpfile: %s\n", control.infile);
 	return fd_in;
 }
 
@@ -338,19 +347,11 @@ static void decompress_file(void)
 		fd_out = open_tmpoutfile();
 	control.fd_out = fd_out;
 
-	if (control.tmp_outfile)
-		fd_hist = shm_open(control.tmp_outfile, O_RDONLY, 0777);
-	else
-		fd_hist = open(control.outfile, O_RDONLY);
-	if (unlikely(fd_hist == -1))
-		fatal("Failed to open history file %s\n", control.outfile);
-	control.fd_hist = fd_hist;
-
 	read_magic(fd_in, &expected_size);
 
-	/* Check if there's enough free space on the device chosen to fit the
-	 * decompressed file. */
 	if (!STDOUT) {
+		/* Check if there's enough free space on the device chosen to fit the
+		* decompressed file. */
 		if (unlikely(fstatvfs(fd_out, &fbuf)))
 			fatal("Failed to fstatvfs in decompress_file\n");
 		free_space = fbuf.f_bsize * fbuf.f_bavail;
@@ -358,11 +359,19 @@ static void decompress_file(void)
 			if (FORCE_REPLACE)
 				print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n");
 			else
-				failure("Inadequate free space to decompress file, use -f to override."
-					" Free space %lld, expected size %lld\n", free_space, expected_size);
+				failure("Inadequate free space to decompress file, use -f to override.\n");
 		}
 	}
 
+	fd_hist = open(control.outfile, O_RDONLY);
+	if (unlikely(fd_hist == -1))
+		fatal("Failed to open history file %s\n", control.outfile);
+	control.fd_hist = fd_hist;
+
+	/* Unlink temporary file as soon as possible */
+	if (unlikely((STDOUT || TEST_ONLY) && unlink(control.outfile)))
+		fatal("Failed to unlink tmpfile: %s\n", control.outfile);
+
 	if (NO_MD5)
 		print_verbose("Not performing MD5 hash check\n");
 	if (HAS_MD5)
@@ -387,15 +396,9 @@ static void decompress_file(void)
 	if (unlikely(close(fd_hist) || close(fd_out)))
 		fatal("Failed to close files\n");
 
-	if (TEST_ONLY | STDOUT) {
-		/* Delete temporary files generated for testing or faking stdout */
-		if (unlikely(unlink(control.outfile)))
-			fatal("Failed to unlink tmpfile: %s\n", strerror(errno));
-	}
-
 	close(fd_in);
 
-	if (!(KEEP_FILES | TEST_ONLY) || STDIN) {
+	if (!KEEP_FILES) {
 		if (unlikely(unlink(control.infile)))
 			fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno));
 	}
@@ -600,12 +603,8 @@ next_chunk:
 			     ctotal, expected_size);
 	}
 
-	if (STDIN) {
-		if (unlikely(unlink(control.infile)))
-			fatal("Failed to unlink %s: %s\n", infilecopy, strerror(errno));
-	} else
-		if (unlikely(close(fd_in)))
-			fatal("Failed to close fd_in in get_fileinfo\n");
+	if (unlikely(close(fd_in)))
+		fatal("Failed to close fd_in in get_fileinfo\n");
 
 	free(control.outfile);
 	free(infilecopy);
@@ -688,6 +687,9 @@ static void compress_file(void)
 		fd_out = open_tmpoutfile();
 	control.fd_out = fd_out;
 
+	if (unlikely(STDOUT && unlink(control.outfile)))
+		fatal("Failed to unlink tmpfile: %s\n", control.outfile);
+
 	preserve_perms(fd_in, fd_out);
 
 	/* write zeroes to 24 bytes at beginning of file */
@@ -705,12 +707,6 @@ static void compress_file(void)
 	if (unlikely(close(fd_in) || close(fd_out)))
 		fatal("Failed to close files\n");
 
-	if (STDOUT) {
-		/* Delete temporary files generated for testing or faking stdout */
-		if (unlikely(unlink(control.outfile)))
-			fatal("Failed to unlink tmpfile: %s\n", strerror(errno));
-	}
-
 	if (!KEEP_FILES) {
 		if (unlikely(unlink(control.infile)))
 			fatal("Failed to unlink %s: %s\n", control.infile, strerror(errno));
@@ -1002,6 +998,7 @@ int main(int argc, char *argv[])
 	/* Decrease usable ram size on 32 bits due to kernel/userspace split */
 	if (BITS32)
 		control.ramsize = MAX(control.ramsize - 900000000ll, 900000000ll);
+	control.maxram = control.ramsize / 3;
 
 	/* Set the main nice value to half that of the backend threads since
 	 * the rzip stage is usually the rate limiting step */

runzip.c

@@ -242,8 +242,11 @@ i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size)
 		md5_init_ctx (&control.ctx);
 	gettimeofday(&start,NULL);
 
-	while (total < expected_size)
+	while (total < expected_size) {
 		total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total);
+		if (STDOUT)
+			dump_tmpoutfile(fd_out);
+	}
 
 	gettimeofday(&end,NULL);
 	print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n",

rzip.c

@@ -766,8 +766,8 @@ void rzip_fd(int fd_in, int fd_out)
 	/* Check if there's enough free space on the device chosen to fit the
 	 * compressed file, based on the compressed file being as large as the
 	 * uncompressed file. */
-	if (unlikely(fstatvfs(fd_in, &fbuf)))
-		fatal("Failed to fstatvfs in decompress_file\n");
+	if (unlikely(fstatvfs(fd_out, &fbuf)))
+		fatal("Failed to fstatvfs in compress_file\n");
 	free_space = fbuf.f_bsize * fbuf.f_bavail;
 	if (free_space < control.st_size) {
 		if (FORCE_REPLACE)
@@ -780,7 +780,7 @@ void rzip_fd(int fd_in, int fd_out)
 	 * allocate 1/3 of it to the main buffer and use a sliding mmap
 	 * buffer to work on 2/3 ram size, leaving enough ram for the
 	 * compression backends */
-	control.max_mmap = control.ramsize / 3;
+	control.max_mmap = control.maxram;
 
 	/* On 32 bits we can have a big window with sliding mmap, but can
 	 * not enable much per mmap/malloc */

rzip.h

@@ -270,6 +270,7 @@ struct rzip_control {
 	const char *suffix;
 	int compression_level;
 	i64 overhead; // compressor overhead
+	i64 maxram; // the largest chunk of ram to allocate
 	unsigned char lzma_properties[5]; // lzma properties, encoded
 	i64 window;
 	unsigned long flags;
@@ -333,6 +334,7 @@ const i64 two_gig;
 void prepare_streamout_threads(void);
 void close_streamout_threads(void);
 void round_to_page(i64 *size);
+void dump_tmpoutfile(int fd_out);
 
 #define print_err(format, args...)	do {\
 	fprintf(stderr, format, ##args);	\

stream.c

@@ -763,18 +763,18 @@ void close_streamout_threads(void)
 
 /* open a set of output streams, compressing with the given
    compression level and algorithm */
-void *open_stream_out(int f, int n, i64 limit, char cbytes)
+void *open_stream_out(int f, int n, i64 chunk_limit, char cbytes)
 {
 	struct stream_info *sinfo;
+	i64 testsize, limit;
 	uchar *testmalloc;
-	i64 testsize;
 	int i, testbufs;
 
 	sinfo = calloc(sizeof(struct stream_info), 1);
 	if (unlikely(!sinfo))
 		return NULL;
 
-	sinfo->bufsize = limit;
+	sinfo->bufsize = limit = chunk_limit;
 
 	sinfo->chunk_bytes = cbytes;
 	sinfo->num_streams = n;
@@ -802,8 +802,19 @@ void *open_stream_out(int f, int n, i64 limit, char cbytes)
 			(control.overhead * control.threads));
 
 	testsize = (limit * testbufs) + (control.overhead * control.threads);
-	if (testsize > control.ramsize / 3)
-		limit = (control.ramsize / 3 - (control.overhead * control.threads)) / testbufs;
+	if (testsize > control.maxram)
+		limit = (control.maxram - (control.overhead * control.threads)) / testbufs;
+
+	/* If we don't have enough ram for the number of threads, decrease the
+	 * number of threads till we do, or only have one thread. */
+	while (limit < STREAM_BUFSIZE && limit < chunk_limit) {
+		if (control.threads > 1)
+			--control.threads;
+		else
+			break;
+		limit = (control.maxram - (control.overhead * control.threads)) / testbufs;
+		limit = MIN(limit, chunk_limit);
+	}
 retest_malloc:
 	testsize = (limit * testbufs) + (control.overhead * control.threads);
 	testmalloc = malloc(testsize);