From 4a6fa7602fda8a34b18be9f7a9d6e086d238eb9b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 14 Mar 2011 10:07:51 +1100 Subject: [PATCH] Begin decompressing to stdout via temporary buffer by creating a read/write wrapper. --- lrzip.c | 56 ++++++++++++++++++++++++++------------------- lrzip_private.h | 2 ++ runzip.c | 61 +++++++++++++++++++++++++++++++++++++++++++------ stream.c | 7 +++--- 4 files changed, 93 insertions(+), 33 deletions(-) diff --git a/lrzip.c b/lrzip.c index c70e894..63049d4 100644 --- a/lrzip.c +++ b/lrzip.c @@ -343,6 +343,19 @@ void read_tmpinfile(rzip_control *control, int fd_in) rewind(tmpinfp); } +/* To perform STDOUT, we allocate a proportion of ram that is then used as + * a pseudo-temporary file */ +static void open_tmpoutbuf(rzip_control *control) +{ + control->flags |= FLAG_TMP_OUTBUF; + control->out_maxlen = control->maxram + control->page_size; + control->tmp_outbuf = malloc(control->out_maxlen); + if (unlikely(!control->tmp_outbuf)) + fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); + if (!DECOMPRESS && !TEST_ONLY) + control->out_ofs = control->out_len = MAGIC_LEN; +} + /* decompress one file from the command line */ @@ -427,9 +440,21 @@ void decompress_file(rzip_control *control) } preserve_perms(control, fd_in, fd_out); - } else + } else { + /* When using a temporary output buffer we still generate + * temporary output files in case we use them should we run + * out of space. */ + open_tmpoutbuf(control); fd_out = open_tmpoutfile(control); - control->fd_out = fd_out; + if (unlikely(fd_out == -1)) + fatal("Failed to create %s\n", control->outfile); + fd_hist = open(control->outfile, O_RDONLY); + if (unlikely(fd_hist == -1)) + fatal("Failed to open history file %s\n", control->outfile); + /* Unlink temporary file as soon as possible */ + if (unlikely(unlink(control->outfile))) + fatal("Failed to unlink tmpfile: %s\n", control->outfile); + } read_magic(control, fd_in, &expected_size); @@ -445,15 +470,12 @@ void decompress_file(rzip_control *control) else failure("Inadequate free space to decompress file, use -f to override.\n"); } + fd_hist = open(control->outfile, O_RDONLY); + if (unlikely(fd_hist == -1)) + fatal("Failed to open history file %s\n", control->outfile); } - - fd_hist = open(control->outfile, O_RDONLY); - if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control->outfile); - - /* Unlink temporary file as soon as possible */ - if (unlikely((STDOUT || TEST_ONLY) && unlink(control->outfile))) - fatal("Failed to unlink tmpfile: %s\n", control->outfile); + control->fd_out = fd_out; + control->fd_hist = fd_hist; if (NO_MD5) print_verbose("Not performing MD5 hash check\n"); @@ -467,7 +489,7 @@ void decompress_file(rzip_control *control) runzip_fd(control, fd_in, fd_out, fd_hist, expected_size); - if (STDOUT) + if (STDOUT && !TMP_OUTBUF) dump_tmpoutfile(control, fd_out); /* if we get here, no fatal errors during decompression */ @@ -729,18 +751,6 @@ next_chunk: free(infilecopy); } -/* To perform STDOUT, we allocate a proportion of ram that is then used as - * a pseudo-temporary file */ -static void open_tmpoutbuf(rzip_control *control) -{ - control->flags |= FLAG_TMP_OUTBUF; - control->out_maxlen = control->maxram + control->page_size; - control->tmp_outbuf = malloc(control->out_maxlen); - if (unlikely(!control->tmp_outbuf)) - fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); - control->out_ofs = control->out_len = MAGIC_LEN; -} - /* compress one file from the command line */ diff --git a/lrzip_private.h b/lrzip_private.h index eec9ff7..68cd3ea 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -162,6 +162,7 @@ struct rzip_control { char *tmpdir; // when stdin, stdout, or test used char *tmp_outbuf; // Temporary file storage for stdout i64 out_ofs; // Output offset when tmp_outbuf in use + i64 hist_ofs; // History offset i64 out_len; // Total length of tmp_outbuf i64 out_maxlen; // The largest the tmp_outbuf can be used i64 rel_ofs; // Relative tmp_outbuf offset when stdout has been flushed @@ -183,6 +184,7 @@ struct rzip_control { i64 st_size; long page_size; int fd_out; + int fd_hist; char encrypt; i64 encloops; uchar loop_byte1; diff --git a/runzip.c b/runzip.c index 2c88313..92d5594 100644 --- a/runzip.c +++ b/runzip.c @@ -73,6 +73,37 @@ static inline i64 read_vchars(rzip_control *control, void *ss, int stream, int l return s; } +static i64 seekcur_fdout(rzip_control *control) +{ + if (!TMP_OUTBUF) + return lseek(control->fd_out, 0, SEEK_CUR); + return (control->rel_ofs + control->out_ofs); +} + +static i64 seekto_fdout(rzip_control *control, i64 pos) +{ + if (!TMP_OUTBUF) + return lseek(control->fd_out, pos, SEEK_SET); + control->out_ofs = pos - control->rel_ofs; + if (unlikely(control->out_ofs < 0 || control->out_ofs > control->out_maxlen)) { + print_err("Trying to seek outside tmpoutbuf in seekto_fdout\n"); + return -1; + } + return pos; +} + +static i64 seekto_fdhist(rzip_control *control, i64 pos) +{ + if (!TMP_OUTBUF) + return lseek(control->fd_hist, pos, SEEK_SET); + control->hist_ofs = pos - control->rel_ofs; + if (unlikely(control->hist_ofs < 0 || control->hist_ofs > control->out_maxlen)) { + print_err("Trying to seek outside tmpoutbuf in seekto_fdhist\n"); + return -1; + } + return pos; +} + static i64 read_header(rzip_control *control, void *ss, uchar *head) { int chunk_bytes = 2; @@ -112,6 +143,18 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, int fd_out, u return stream_read; } +static i64 read_fdhist(struct rzip_control *control, void *buf, i64 len) +{ + if (!TMP_OUTBUF) + return read_1g(control->fd_hist, buf, len); + if (unlikely(len + control->hist_ofs > control->out_maxlen)) { + print_err("Trying to read beyond end of tmpoutbuf in read_fdhist\n"); + return -1; + } + memcpy(buf, control->tmp_outbuf + control->hist_ofs, len); + return len; +} + static i64 unzip_match(rzip_control *control, void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum, int chunk_bytes) { i64 offset, n, total, cur_pos; @@ -121,13 +164,13 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, int fd_out, int failure("len %lld is negative in unzip_match!\n",len); total = 0; - cur_pos = lseek(fd_out, 0, SEEK_CUR); + cur_pos = seekcur_fdout(control); if (unlikely(cur_pos == -1)) fatal("Seek failed on out file in unzip_match.\n"); /* Note the offset is in a different format v0.40+ */ offset = read_vchars(control, ss, 0, chunk_bytes); - if (unlikely(lseek(fd_hist, cur_pos - offset, SEEK_SET) == -1)) + if (unlikely(seekto_fdhist(control, cur_pos - offset) == -1)) fatal("Seek failed by %d from %d on history file in unzip_match\n", offset, cur_pos); @@ -139,7 +182,7 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, int fd_out, int while (len) { n = MIN(len, offset); - if (unlikely(read_1g(fd_hist, off_buf, (size_t)n) != (ssize_t)n)) + if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) fatal("Failed to read %d bytes in unzip_match\n", n); if (unlikely(write_1g(control, fd_out, off_buf, (size_t)n) != (ssize_t)n)) @@ -267,8 +310,12 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp do { total += runzip_chunk(control, fd_in, fd_out, fd_hist, expected_size, total); - if (STDOUT) - dump_tmpoutfile(control, fd_out); + if (STDOUT) { + if (TMP_OUTBUF) + flush_stdout(control); + else + dump_tmpoutfile(control, fd_out); + } } while (total < expected_size || (!expected_size && !control->eof)); gettimeofday(&end,NULL); @@ -308,8 +355,8 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp int i, j; memcpy(md5_stored, md5_resblock, MD5_DIGEST_SIZE); - if (unlikely(lseek(fd_hist, 0, SEEK_SET) == -1)) - fatal("Failed to lseek fd_hist in runzip_fd\n"); + if (unlikely(seekto_fdhist(control, 0) == -1)) + fatal("Failed to seekto_fdhist in runzip_fd\n"); if (unlikely((md5_fstream = fdopen(fd_hist, "r")) == NULL)) fatal("Failed to fdopen fd_hist in runzip_fd\n"); if (unlikely(md5_stream(md5_fstream, md5_resblock))) diff --git a/stream.c b/stream.c index e22e909..f788b33 100644 --- a/stream.c +++ b/stream.c @@ -772,9 +772,10 @@ static int seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) if (TMP_OUTBUF) { spos -= control->rel_ofs; control->out_ofs = spos; - if (unlikely(spos > control->out_len || spos < 0)) { - print_err("Trying to seek to %lld outside tmp outbuf in seekto\n", spos); - return -1; + if (unlikely((!(DECOMPRESS || TEST_ONLY) && (spos > control->out_len)) || + spos > control->out_maxlen || spos < 0)) { + print_err("Trying to seek to %lld outside tmp outbuf in seekto\n", spos); + return -1; } return 0; }