Implement the real workings of writing to a temporary buffer before flushing to stdout.

lrzip.c

@@ -75,10 +75,9 @@ static i64 enc_loops(uchar b1, uchar b2)
 	return (i64)b2 << (i64)b1;
 }
 
-static char *make_magic(rzip_control *control, int fd_in)
+static char *make_magic(rzip_control *control)
 {
 	struct timeval tv;
-	struct stat st;
 	char *magic;
 
 	magic = calloc(MAGIC_LEN, 1);
@@ -88,9 +87,6 @@ static char *make_magic(rzip_control *control, int fd_in)
 	magic[4] = LRZIP_MAJOR_VERSION;
 	magic[5] = LRZIP_MINOR_VERSION;
 
-	if (unlikely(!STDIN && fstat(fd_in, &st)))
-		fatal("bad magic file descriptor!?\n");
-
 	/* File size is stored as zero for streaming STDOUT blocks when the
 	 * file size is unknown. */
 	if (!STDIN || !STDOUT || control->eof)
@@ -122,24 +118,26 @@ static char *make_magic(rzip_control *control, int fd_in)
 	return magic;
 }
 
-void write_stdout_header(rzip_control *control, int fd_in)
+void write_stdout_header(rzip_control *control)
 {
-	char *magic = make_magic(control, fd_in);
+	char *magic = make_magic(control);
 
 	memcpy(control->tmp_outbuf, magic, MAGIC_LEN);
+	control->magic_written = 1;
 
 	free(magic);
 }
 
 static void write_magic(rzip_control *control, int fd_in, int fd_out)
 {
-	char *magic = make_magic(control, fd_in);
+	char *magic = make_magic(control);
 
 	if (unlikely(lseek(fd_out, 0, SEEK_SET)))
 		fatal("Failed to seek to BOF to write Magic Header\n");
 
 	if (unlikely(write(fd_out, magic, MAGIC_LEN) != MAGIC_LEN))
 		fatal("Failed to write magic header\n");
+	control->magic_written = 1;
 
 	free(magic);
 }
@@ -242,8 +240,36 @@ int open_tmpoutfile(rzip_control *control)
 	return fd_out;
 }
 
+extern const one_g;
+
+static void fwrite_stdout(void *buf, i64 len)
+{
+	uchar *offset_buf = buf;
+	ssize_t ret;
+	i64 total;
+
+	total = 0;
+	while (len > 0) {
+		if (len > one_g)
+			ret = one_g;
+		else
+			ret = len;
+		ret = fwrite(offset_buf, 1, ret, stdout);
+		if (unlikely(ret <= 0))
+			fatal("Failed to fwrite in fwrite_stdout\n");
+		len -= ret;
+		offset_buf += ret;
+		total += ret;
+	}
+	fflush(stdout);
+}
+
 void flush_stdout(rzip_control *control)
 {
+	print_verbose("Dumping buffer to stdout.\n");
+	fwrite_stdout(control->tmp_outbuf, control->out_len);
+	control->rel_ofs += control->out_len;
+	control->out_ofs = control->out_len = 0;
 }
 
 /* Dump temporary outputfile to perform stdout */
@@ -714,7 +740,7 @@ static void open_tmpoutbuf(rzip_control *control)
 	control->tmp_outbuf = malloc(control->maxram);
 	if (unlikely(!control->tmp_outbuf))
 		fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n");
-	control->out_ofs = MAGIC_LEN;
+	control->out_ofs = control->out_len = MAGIC_LEN;
 }
 
 /*
@@ -790,14 +816,10 @@ void compress_file(rzip_control *control)
 			control->flags |= FLAG_KEEP_BROKEN;
 			fatal("Failed to create %s\n", control->outfile);
 		}
-	} else
-		fd_out = open_tmpoutfile(control);
-	control->fd_out = fd_out;
-
-	if (unlikely(STDOUT && unlink(control->outfile)))
-		fatal("Failed to unlink tmpfile: %s\n", control->outfile);
-
-	preserve_perms(control, fd_in, fd_out);
+		control->fd_out = fd_out;
+		preserve_perms(control, fd_in, fd_out);
+	} else 
+		open_tmpoutbuf(control);
 
 	/* Write zeroes to header at beginning of file */
 	if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header)))

lrzip.h

@@ -32,6 +32,6 @@ void decompress_file(rzip_control *control);
 void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head);
 void get_fileinfo(rzip_control *control);
 void compress_file(rzip_control *control);
-void write_stdout_header(rzip_control *control, int fd_in);
+void write_stdout_header(rzip_control *control);
 void flush_stdout(rzip_control *control);
 #endif

lrzip_private.h

@@ -162,6 +162,7 @@ struct rzip_control {
 	char *tmp_outbuf; // Temporary file storage for stdout
 	i64 out_ofs; // Output offset when tmp_outbuf in use
 	i64 out_len; // Total length of tmp_outbuf
+	i64 rel_ofs; // Relative offset when stdou has been flushed
 	FILE *msgout; //stream for output messages
 	const char *suffix;
 	int compression_level;
@@ -188,6 +189,7 @@ struct rzip_control {
 	i64 secs;
 	i64 usecs;
 	unsigned char eof;
+	unsigned char magic_written;
 	md5_ctx ctx;
 	i64 md5_read; // How far into the file the md5 has done so far
 };

rzip.c

@@ -789,17 +789,19 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out)
 	} else
 		control->st_size = 0;
 
-	/* Check if there's enough free space on the device chosen to fit the
-	 * compressed file, based on the compressed file being as large as the
-	 * uncompressed file. */
-	if (unlikely(fstatvfs(fd_out, &fbuf)))
-		fatal("Failed to fstatvfs in compress_file\n");
-	free_space = fbuf.f_bsize * fbuf.f_bavail;
-	if (free_space < control->st_size) {
-		if (FORCE_REPLACE)
-			print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n");
-		else
-			failure("Possibly inadequate free space to compress file, use -f to override.\n");
+	if (!STDOUT) {
+		/* Check if there's enough free space on the device chosen to fit the
+		* compressed file, based on the compressed file being as large as the
+		* uncompressed file. */
+		if (unlikely(fstatvfs(fd_out, &fbuf)))
+			fatal("Failed to fstatvfs in compress_file\n");
+		free_space = fbuf.f_bsize * fbuf.f_bavail;
+		if (free_space < control->st_size) {
+			if (FORCE_REPLACE)
+				print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n");
+			else
+				failure("Possibly inadequate free space to compress file, use -f to override.\n");
+		}
 	}
 
 	/* Optimal use of ram involves using no more than 2/3 of it, so we
@@ -958,29 +960,29 @@ retry:
 		last.tv_usec = current.tv_usec;
 
 		rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple);
-		if (STDOUT) {
-			if (len == 0) // No header written yet
-				write_stdout_header(control, fd_in);
-			flush_stdout(control);
-		}
 
 		/* st->chunk_size may be shrunk in rzip_chunk */
 		last_chunk = st->chunk_size;
 		len -= st->chunk_size;
+		if (!len)
+			control->eof = 1;
 	}
 
 	close_streamout_threads(control);
 
-	md5_finish_ctx (&control->ctx, md5_resblock);
+	md5_finish_ctx(&control->ctx, md5_resblock);
 	if (HASH_CHECK || MAX_VERBOSE) {
 		print_output("MD5: ");
 		for (j = 0; j < MD5_DIGEST_SIZE; j++)
 			print_output("%02x", md5_resblock[j] & 0xFF);
 		print_output("\n");
 	}
-	if (unlikely(write(control->fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
+	if (unlikely(write_1g(control, control->fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
 		fatal("Failed to write md5 in rzip_fd\n");
 
+	if (STDOUT)
+		flush_stdout(control);
+
 	gettimeofday(&current, NULL);
 	if (STDIN)
 		s.st_size = control->st_size;

stream.c

@@ -639,7 +639,7 @@ ssize_t put_fdout(rzip_control *control, int fd, void *offset_buf, ssize_t ret)
 {
 	if (!STDOUT || DECOMPRESS)
 		return write(fd, offset_buf, (size_t)ret);
-	memcpy(control->tmp_outbuf, offset_buf, ret);
+	memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret);
 	control->out_ofs += ret;
 	if (likely(control->out_ofs > control->out_len))
 		control->out_len = control->out_ofs;
@@ -762,10 +762,20 @@ static int read_i64(int f, i64 *v)
 }
 
 /* seek to a position within a set of streams - return -1 on failure */
-static int seekto(struct stream_info *sinfo, i64 pos)
+static int seekto(rzip_control *control, struct stream_info *sinfo, i64 pos)
 {
 	i64 spos = pos + sinfo->initial_pos;
 
+	if (!DECOMPRESS && STDOUT) {
+		spos -= control->rel_ofs;
+		control->out_ofs = spos;
+		if (unlikely(spos > control->out_len || spos < 0)) {
+			print_err("Trying to seek to %lld outside tmp outbuf in seekto\n", spos);
+			return -1;
+		}
+		return 0;
+	}
+
 	if (unlikely(lseek(sinfo->fd, spos, SEEK_SET) != spos)) {
 		print_err("Failed to seek to %lld in stream\n", pos);
 		return -1;
@@ -773,6 +783,18 @@ static int seekto(struct stream_info *sinfo, i64 pos)
 	return 0;
 }
 
+static i64 get_seek(rzip_control *control, int fd)
+{
+	i64 ret;
+
+	if (!DECOMPRESS && STDOUT)
+		return control->rel_ofs + control->out_ofs;
+	ret = lseek(fd, 0, SEEK_CUR);
+	if (unlikely(ret == -1))
+		fatal("Failed to lseek in get_seek\n");
+	return ret;
+}
+
 void prepare_streamout_threads(rzip_control *control)
 {
 	int i;
@@ -1082,6 +1104,12 @@ retry:
 	if (!ctis->chunks++) {
 		int j;
 
+		if (STDOUT) {
+			if (!control->magic_written)
+				write_stdout_header(control);
+			flush_stdout(control);
+		}
+
 		/* Write chunk bytes of this block */
 		write_u8(control, ctis->fd, ctis->chunk_bytes);
 
@@ -1091,7 +1119,7 @@ retry:
 		write_i64(control, ctis->fd, ctis->size);
 
 		/* First chunk of this stream, write headers */
-		ctis->initial_pos = lseek(ctis->fd, 0, SEEK_CUR);
+		ctis->initial_pos = get_seek(control, ctis->fd);
 
 		for (j = 0; j < ctis->num_streams; j++) {
 			ctis->s[j].last_head = ctis->cur_pos + 17;
@@ -1103,14 +1131,14 @@ retry:
 		}
 	}
 
-	if (unlikely(seekto(ctis, ctis->s[cti->streamno].last_head)))
+	if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head)))
 		fatal("Failed to seekto in compthread %d\n", i);
 
 	if (unlikely(write_i64(control, ctis->fd, ctis->cur_pos)))
 		fatal("Failed to write_i64 in compthread %d\n", i);
 
 	ctis->s[cti->streamno].last_head = ctis->cur_pos + 17;
-	if (unlikely(seekto(ctis, ctis->cur_pos)))
+	if (unlikely(seekto(control, ctis, ctis->cur_pos)))
 		fatal("Failed to seekto cur_pos in compthread %d\n", i);
 
 	print_maxverbose("Thread %ld writing %lld compressed bytes from stream %d\n", i, cti->c_len, cti->streamno);
@@ -1256,7 +1284,7 @@ fill_another:
 	if (unlikely(ucthread[s->uthread_no].busy))
 		failure("Trying to start a busy thread, this shouldn't happen!\n");
 
-	if (unlikely(seekto(sinfo, s->last_head)))
+	if (unlikely(seekto(control, sinfo, s->last_head)))
 		return -1;
 
 	if (unlikely(read_u8(sinfo->fd, &c_type)))