Fix stdout dumping to console when there is inadequate memory to compress in ram by properly using temporary files.

lrzip.c

@@ -318,6 +318,7 @@ int open_tmpoutfile(rzip_control *control)
 			       control->outfile, DECOMPRESS ? "de" : "");
 	} else
 		register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN);
+	print_maxverbose("Created temporary outfile %s\n", control->outfile);
 	return fd_out;
 }
 
@@ -365,7 +366,7 @@ bool write_fdout(rzip_control *control, void *buf, i64 len)
 	return true;
 }
 
-bool flush_tmpoutbuf(rzip_control *control)
+static bool flush_tmpoutbuf(rzip_control *control)
 {
 	if (!TEST_ONLY) {
 		print_maxverbose("Dumping buffer to physical file.\n");
@@ -383,10 +384,10 @@ bool flush_tmpoutbuf(rzip_control *control)
 }
 
 /* Dump temporary outputfile to perform stdout */
-bool dump_tmpoutfile(rzip_control *control, int fd_out)
+static bool dump_tmpoutfile(rzip_control *control)
 {
+	int tmpchar, fd_out = control->fd_out;
 	FILE *tmpoutfp;
-	int tmpchar;
 
 	if (unlikely(fd_out == -1))
 		fatal_return(("Failed: No temporary outfile created, unable to do in ram\n"), false);
@@ -410,6 +411,15 @@ bool dump_tmpoutfile(rzip_control *control, int fd_out)
 	return true;
 }
 
+bool flush_tmpout(rzip_control *control)
+{
+	if (!STDOUT)
+		return true;
+	if (TMP_OUTBUF)
+		return flush_tmpoutbuf(control);
+	return dump_tmpoutfile(control);
+}
+
 /* Used if we're unable to read STDIN into the temporary buffer, shunts data
  * to temporary file */
 bool write_fdin(rzip_control *control)
@@ -857,11 +867,6 @@ bool decompress_file(rzip_control *control)
 	 * created. */
 	clear_rulist(control);
 
-	if (STDOUT && !TMP_OUTBUF) {
-		if (unlikely(!dump_tmpoutfile(control, fd_out)))
-			return false;
-	}
-
 	/* if we get here, no fatal_return(( errors during decompression */
 	print_progress("\r");
 	if (!(STDOUT | TEST_ONLY))
@@ -1311,6 +1316,12 @@ bool compress_file(rzip_control *control)
 				goto error;
 		}
 	} else {
+		control->fd_out = fd_out = open_tmpoutfile(control);
+		if (likely(fd_out != -1)) {
+			/* Unlink temporary file as soon as possible */
+			if (unlikely(unlink(control->outfile)))
+				fatal_return(("Failed to unlink tmpfile: %s\n", control->outfile), false);
+		}
 		if (unlikely(!open_tmpoutbuf(control)))
 			goto error;
 	}
@@ -1321,7 +1332,7 @@ bool compress_file(rzip_control *control)
 
 	rzip_fd(control, fd_in, fd_out);
 
-	/* Wwrite magic at end b/c lzma does not tell us properties until it is done */
+	/* Write magic at end b/c lzma does not tell us properties until it is done */
 	if (!STDOUT) {
 		if (unlikely(!write_magic(control)))
 			goto error;

lrzip_core.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -27,7 +27,7 @@ bool write_magic(rzip_control *control);
 bool read_magic(rzip_control *control, int fd_in, i64 *expected_size);
 bool preserve_perms(rzip_control *control, int fd_in, int fd_out);
 int open_tmpoutfile(rzip_control *control);
-bool dump_tmpoutfile(rzip_control *control, int fd_out);
+bool flush_tmpout(rzip_control *control);
 int open_tmpinfile(rzip_control *control);
 bool read_tmpinfile(rzip_control *control, int fd_in);
 bool decompress_file(rzip_control *control);
@@ -36,7 +36,6 @@ bool get_fileinfo(rzip_control *control);
 bool compress_file(rzip_control *control);
 bool write_fdout(rzip_control *control, void *buf, i64 len);
 bool write_fdin(rzip_control *control);
-bool flush_tmpoutbuf(rzip_control *control);
 void close_tmpoutbuf(rzip_control *control);
 void clear_tmpinbuf(rzip_control *control);
 bool clear_tmpinfile(rzip_control *control);

runzip.c

@@ -407,17 +407,11 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp
 			}
 		}
 		total += u;
-		if (TMP_OUTBUF) {
+		if (unlikely(!flush_tmpout(control))) {
-			if (unlikely(!flush_tmpoutbuf(control))) {
+			print_err("Failed to flush_tmpout in runzip_fd\n");
-				print_err("Failed to flush_tmpoutbuf in runzip_fd\n");
 				return -1;
-			}
-		} else if (STDOUT) {
-			if (unlikely(!dump_tmpoutfile(control, fd_out))) {
-				print_err("Failed to dump_tmpoutfile in runzip_fd\n");
-				return -1;
-			}
 		}
+
 		if (TMP_INBUF)
 			clear_tmpinbuf(control);
 		else if (STDIN && !DECOMPRESS) {

rzip.c

@@ -1221,11 +1221,9 @@ retry:
 		}
 	}
 
-	if (TMP_OUTBUF) {
+	if (unlikely(!flush_tmpout(control))) {
-		if (unlikely(!flush_tmpoutbuf(control))) {
 			dealloc(st);
-			failure("Failed to flush_tmpoutbuf in rzip_fd\n");
+			failure("Failed to flush_tmpout in rzip_fd\n");
-		}
 	}
 
 	gettimeofday(&current, NULL);

stream.c

@@ -610,20 +610,22 @@ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret)
 	if (unlikely(control->out_ofs + ret > control->out_maxlen)) {
 		/* The data won't fit in a temporary output buffer so we have
 		 * to fall back to temporary files. */
-		print_verbose("Unable to decompress entirely in ram, will use physical files\n");
+		print_verbose("Unable to %scompress entirely in ram, will use physical files\n",
-		if (unlikely(control->fd_out == -1))
+			      DECOMPRESS ? "de" : "");
-			failure("Was unable to decompress entirely in ram and no temporary file creation was possible\n");
+		if (unlikely(control->fd_out == -1)) {
+			failure("Was unable to %scompress entirely in ram and no temporary file creation was possible\n",
+				DECOMPRESS ? "de" : "");
+		}
+		/* Copy tmp_outbuf to tmpoutfile before deallocation */
 		if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) {
 			print_err("Unable to write_fdout tmpoutbuf in put_fdout\n");
 			return -1;
 		}
+		/* Deallocate now unused tmpoutbuf and unset tmp_outbuf flag */
 		close_tmpoutbuf(control);
-		if (unlikely(!write_fdout(control, offset_buf, ret))) {
+		return write(control->fd_out, offset_buf, (size_t)ret);
-			print_err("Unable to write_fdout offset_buf in put_fdout\n");
-			return -1;
-		}
-		return ret;
 	}
+
 	memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret);
 	control->out_ofs += ret;
 	if (likely(control->out_ofs > control->out_len))
@@ -695,8 +697,10 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len)
 		/* We're decompressing from STDIN */
 		if (unlikely(control->in_ofs + len > control->in_maxlen)) {
 			/* We're unable to fit it all into the temp buffer */
-			if (dump_stdin(control))
+			if (dump_stdin(control)) {
-				failure_return(("Inadequate ram to %compress from STDIN and unable to create in tmpfile"), -1);
+				failure_return(("Inadequate ram to %scompress from STDIN and unable to create in tmpfile",
+					DECOMPRESS ? "de" : ""), -1);
+			}
 			goto read_fd;
 		}
 		if (control->in_ofs + len > control->in_len) {
@@ -1366,16 +1370,11 @@ retry:
 	if (!ctis->chunks++) {
 		int j;
 
-		if (TMP_OUTBUF) {
+		if (STDOUT) {
 			lock_mutex(control, &control->control_lock);
 			if (!control->magic_written)
 				write_magic(control);
 			unlock_mutex(control, &control->control_lock);
-
-			if (unlikely(!flush_tmpoutbuf(control))) {
-				print_err("Failed to flush_tmpoutbuf in compthread\n");
-				goto error;
-			}
 		}
 
 		print_maxverbose("Writing initial chunk bytes value %d at %lld\n",