diff --git a/lrzip.c b/lrzip.c index a07b986..8088090 100644 --- a/lrzip.c +++ b/lrzip.c @@ -75,10 +75,9 @@ static i64 enc_loops(uchar b1, uchar b2) return (i64)b2 << (i64)b1; } -static char *make_magic(rzip_control *control, int fd_in) +static char *make_magic(rzip_control *control) { struct timeval tv; - struct stat st; char *magic; magic = calloc(MAGIC_LEN, 1); @@ -88,9 +87,6 @@ static char *make_magic(rzip_control *control, int fd_in) magic[4] = LRZIP_MAJOR_VERSION; magic[5] = LRZIP_MINOR_VERSION; - if (unlikely(!STDIN && fstat(fd_in, &st))) - fatal("bad magic file descriptor!?\n"); - /* File size is stored as zero for streaming STDOUT blocks when the * file size is unknown. */ if (!STDIN || !STDOUT || control->eof) @@ -122,24 +118,26 @@ static char *make_magic(rzip_control *control, int fd_in) return magic; } -void write_stdout_header(rzip_control *control, int fd_in) +void write_stdout_header(rzip_control *control) { - char *magic = make_magic(control, fd_in); + char *magic = make_magic(control); memcpy(control->tmp_outbuf, magic, MAGIC_LEN); + control->magic_written = 1; free(magic); } static void write_magic(rzip_control *control, int fd_in, int fd_out) { - char *magic = make_magic(control, fd_in); + char *magic = make_magic(control); if (unlikely(lseek(fd_out, 0, SEEK_SET))) fatal("Failed to seek to BOF to write Magic Header\n"); if (unlikely(write(fd_out, magic, MAGIC_LEN) != MAGIC_LEN)) fatal("Failed to write magic header\n"); + control->magic_written = 1; free(magic); } @@ -242,8 +240,36 @@ int open_tmpoutfile(rzip_control *control) return fd_out; } +extern const one_g; + +static void fwrite_stdout(void *buf, i64 len) +{ + uchar *offset_buf = buf; + ssize_t ret; + i64 total; + + total = 0; + while (len > 0) { + if (len > one_g) + ret = one_g; + else + ret = len; + ret = fwrite(offset_buf, 1, ret, stdout); + if (unlikely(ret <= 0)) + fatal("Failed to fwrite in fwrite_stdout\n"); + len -= ret; + offset_buf += ret; + total += ret; + } + fflush(stdout); +} + void flush_stdout(rzip_control *control) { + print_verbose("Dumping buffer to stdout.\n"); + fwrite_stdout(control->tmp_outbuf, control->out_len); + control->rel_ofs += control->out_len; + control->out_ofs = control->out_len = 0; } /* Dump temporary outputfile to perform stdout */ @@ -714,7 +740,7 @@ static void open_tmpoutbuf(rzip_control *control) control->tmp_outbuf = malloc(control->maxram); if (unlikely(!control->tmp_outbuf)) fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); - control->out_ofs = MAGIC_LEN; + control->out_ofs = control->out_len = MAGIC_LEN; } /* @@ -790,14 +816,10 @@ void compress_file(rzip_control *control) control->flags |= FLAG_KEEP_BROKEN; fatal("Failed to create %s\n", control->outfile); } - } else - fd_out = open_tmpoutfile(control); - control->fd_out = fd_out; - - if (unlikely(STDOUT && unlink(control->outfile))) - fatal("Failed to unlink tmpfile: %s\n", control->outfile); - - preserve_perms(control, fd_in, fd_out); + control->fd_out = fd_out; + preserve_perms(control, fd_in, fd_out); + } else + open_tmpoutbuf(control); /* Write zeroes to header at beginning of file */ if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header))) diff --git a/lrzip.h b/lrzip.h index 3721034..17395f1 100644 --- a/lrzip.h +++ b/lrzip.h @@ -32,6 +32,6 @@ void decompress_file(rzip_control *control); void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head); void get_fileinfo(rzip_control *control); void compress_file(rzip_control *control); -void write_stdout_header(rzip_control *control, int fd_in); +void write_stdout_header(rzip_control *control); void flush_stdout(rzip_control *control); #endif diff --git a/lrzip_private.h b/lrzip_private.h index 8f98f02..6228385 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -162,6 +162,7 @@ struct rzip_control { char *tmp_outbuf; // Temporary file storage for stdout i64 out_ofs; // Output offset when tmp_outbuf in use i64 out_len; // Total length of tmp_outbuf + i64 rel_ofs; // Relative offset when stdou has been flushed FILE *msgout; //stream for output messages const char *suffix; int compression_level; @@ -188,6 +189,7 @@ struct rzip_control { i64 secs; i64 usecs; unsigned char eof; + unsigned char magic_written; md5_ctx ctx; i64 md5_read; // How far into the file the md5 has done so far }; diff --git a/rzip.c b/rzip.c index cd5e430..e9d6b6b 100644 --- a/rzip.c +++ b/rzip.c @@ -789,17 +789,19 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) } else control->st_size = 0; - /* Check if there's enough free space on the device chosen to fit the - * compressed file, based on the compressed file being as large as the - * uncompressed file. */ - if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal("Failed to fstatvfs in compress_file\n"); - free_space = fbuf.f_bsize * fbuf.f_bavail; - if (free_space < control->st_size) { - if (FORCE_REPLACE) - print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); - else - failure("Possibly inadequate free space to compress file, use -f to override.\n"); + if (!STDOUT) { + /* Check if there's enough free space on the device chosen to fit the + * compressed file, based on the compressed file being as large as the + * uncompressed file. */ + if (unlikely(fstatvfs(fd_out, &fbuf))) + fatal("Failed to fstatvfs in compress_file\n"); + free_space = fbuf.f_bsize * fbuf.f_bavail; + if (free_space < control->st_size) { + if (FORCE_REPLACE) + print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); + else + failure("Possibly inadequate free space to compress file, use -f to override.\n"); + } } /* Optimal use of ram involves using no more than 2/3 of it, so we @@ -958,29 +960,29 @@ retry: last.tv_usec = current.tv_usec; rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple); - if (STDOUT) { - if (len == 0) // No header written yet - write_stdout_header(control, fd_in); - flush_stdout(control); - } /* st->chunk_size may be shrunk in rzip_chunk */ last_chunk = st->chunk_size; len -= st->chunk_size; + if (!len) + control->eof = 1; } close_streamout_threads(control); - md5_finish_ctx (&control->ctx, md5_resblock); + md5_finish_ctx(&control->ctx, md5_resblock); if (HASH_CHECK || MAX_VERBOSE) { print_output("MD5: "); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_resblock[j] & 0xFF); print_output("\n"); } - if (unlikely(write(control->fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) + if (unlikely(write_1g(control, control->fd_out, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) fatal("Failed to write md5 in rzip_fd\n"); + if (STDOUT) + flush_stdout(control); + gettimeofday(¤t, NULL); if (STDIN) s.st_size = control->st_size; diff --git a/stream.c b/stream.c index 0f64fe7..974fcc9 100644 --- a/stream.c +++ b/stream.c @@ -639,7 +639,7 @@ ssize_t put_fdout(rzip_control *control, int fd, void *offset_buf, ssize_t ret) { if (!STDOUT || DECOMPRESS) return write(fd, offset_buf, (size_t)ret); - memcpy(control->tmp_outbuf, offset_buf, ret); + memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret); control->out_ofs += ret; if (likely(control->out_ofs > control->out_len)) control->out_len = control->out_ofs; @@ -762,10 +762,20 @@ static int read_i64(int f, i64 *v) } /* seek to a position within a set of streams - return -1 on failure */ -static int seekto(struct stream_info *sinfo, i64 pos) +static int seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) { i64 spos = pos + sinfo->initial_pos; + if (!DECOMPRESS && STDOUT) { + spos -= control->rel_ofs; + control->out_ofs = spos; + if (unlikely(spos > control->out_len || spos < 0)) { + print_err("Trying to seek to %lld outside tmp outbuf in seekto\n", spos); + return -1; + } + return 0; + } + if (unlikely(lseek(sinfo->fd, spos, SEEK_SET) != spos)) { print_err("Failed to seek to %lld in stream\n", pos); return -1; @@ -773,6 +783,18 @@ static int seekto(struct stream_info *sinfo, i64 pos) return 0; } +static i64 get_seek(rzip_control *control, int fd) +{ + i64 ret; + + if (!DECOMPRESS && STDOUT) + return control->rel_ofs + control->out_ofs; + ret = lseek(fd, 0, SEEK_CUR); + if (unlikely(ret == -1)) + fatal("Failed to lseek in get_seek\n"); + return ret; +} + void prepare_streamout_threads(rzip_control *control) { int i; @@ -1082,6 +1104,12 @@ retry: if (!ctis->chunks++) { int j; + if (STDOUT) { + if (!control->magic_written) + write_stdout_header(control); + flush_stdout(control); + } + /* Write chunk bytes of this block */ write_u8(control, ctis->fd, ctis->chunk_bytes); @@ -1091,7 +1119,7 @@ retry: write_i64(control, ctis->fd, ctis->size); /* First chunk of this stream, write headers */ - ctis->initial_pos = lseek(ctis->fd, 0, SEEK_CUR); + ctis->initial_pos = get_seek(control, ctis->fd); for (j = 0; j < ctis->num_streams; j++) { ctis->s[j].last_head = ctis->cur_pos + 17; @@ -1103,14 +1131,14 @@ retry: } } - if (unlikely(seekto(ctis, ctis->s[cti->streamno].last_head))) + if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) fatal("Failed to seekto in compthread %d\n", i); if (unlikely(write_i64(control, ctis->fd, ctis->cur_pos))) fatal("Failed to write_i64 in compthread %d\n", i); ctis->s[cti->streamno].last_head = ctis->cur_pos + 17; - if (unlikely(seekto(ctis, ctis->cur_pos))) + if (unlikely(seekto(control, ctis, ctis->cur_pos))) fatal("Failed to seekto cur_pos in compthread %d\n", i); print_maxverbose("Thread %ld writing %lld compressed bytes from stream %d\n", i, cti->c_len, cti->streamno); @@ -1256,7 +1284,7 @@ fill_another: if (unlikely(ucthread[s->uthread_no].busy)) failure("Trying to start a busy thread, this shouldn't happen!\n"); - if (unlikely(seekto(sinfo, s->last_head))) + if (unlikely(seekto(control, sinfo, s->last_head))) return -1; if (unlikely(read_u8(sinfo->fd, &c_type)))