diff --git a/README.md b/README.md index ed6bfff..00b59ea 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Simple 'n Easy™: `sudo make install` ### lrzip internals lrzip uses an extended version of [rzip](http://rzip.samba.org/) which does a first pass long distance -redundancy reduction. lrzip's modifications allow it to scale to accomodate various memory sizes. +redundancy reduction. lrzip's modifications allow it to scale to accommodate various memory sizes. Then, one of the following scenarios occurs: diff --git a/configure.ac b/configure.ac index 7bfe011..b6d7159 100644 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,7 @@ AC_CONFIG_SRCDIR([configure.ac]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) -AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign]) +AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_USE_SYSTEM_EXTENSIONS diff --git a/lrzip.c b/lrzip.c index 80ddd81..43144b5 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998-2003 Andrew Tridgell @@ -513,13 +513,25 @@ bool read_tmpinfile(rzip_control *control, int fd_in) * a pseudo-temporary file */ static bool open_tmpoutbuf(rzip_control *control) { + i64 maxlen = control->maxram; + void *buf; + + while (42) { + round_to_page(&maxlen); + buf = malloc(maxlen); + if (buf) { + print_maxverbose("Malloced %"PRId64" for tmp_outbuf\n", maxlen); + break; + } + maxlen = maxlen / 3 * 2; + if (maxlen < 100000000) + fatal_return(("Unable to even malloc 100MB for tmp_outbuf\n"), false); + } control->flags |= FLAG_TMP_OUTBUF; - control->out_maxlen = control->maxram; /* Allocate slightly more so we can cope when the buffer overflows and * fall back to a real temporary file */ - control->tmp_outbuf = malloc(control->maxram + control->page_size); - if (unlikely(!control->tmp_outbuf)) - fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false); + control->out_maxlen = maxlen - control->page_size; + control->tmp_outbuf = buf; if (!DECOMPRESS && !TEST_ONLY) control->out_ofs = control->out_len = MAGIC_LEN;\ return true; diff --git a/lrzip_private.h b/lrzip_private.h index fa29f44..9ac9f47 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998-2003 Andrew Tridgell diff --git a/lrztar b/lrztar index e773fe2..fafc108 100755 --- a/lrztar +++ b/lrztar @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (C) George Makrydakis 2009-2011,2013 -# Copyright (C) Con Kolivas 2011-2012 +# Copyright (C) Con Kolivas 2011-2012,2016 # A bash wrapper for Con Kolivas' excellent lrzip utility. For the time # being, lrzip does not like pipes, so we had to do this. It is kind of @@ -47,14 +47,14 @@ Notice: v_w=0 v_S=0 v_D=0 v_p=0 v_q=0 v_L=0 \ v_n=0 v_l=0 v_b=0 v_g=0 v_z=0 v_U=0 \ v_T=0 v_N=0 v_v=0 v_f=0 v_d=0 v_h=0 \ - v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 x= i="$(pwd)" + v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 v_m=0 x= i="$(pwd)" which tar &> /dev/null \ || { printf "lrztar: no tar in your path\n"; return 1; } which lrzip &> /dev/null \ || { printf "lrztar: no lrzip in your path\n"; return 1; } which lrzcat &> /dev/null \ || { printf "lrztar: no lrzcat in your path\n"; return 1; } - while getopts w:O:S:DqL:nlbgzUTN:p:vfo:d:tVhHck x; do + while getopts w:O:S:DqL:nlbgzUm:TN:p:vfo:d:tVhHck x; do [[ $x == [tV] ]] && { printf "lrztar: invalid option for lrztar: %s\n" "$x"; return 1; diff --git a/rzip.c b/rzip.c index 0320515..56a9291 100644 --- a/rzip.c +++ b/rzip.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 1998 Andrew Tridgell Modified to use flat hash, memory limit and variable hash culling @@ -50,6 +50,7 @@ #ifdef HAVE_ARPA_INET_H # include #endif +#include #include "md5.h" #include "stream.h" @@ -729,23 +730,35 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st, put_literal(control, st, st->last_match, st->chunk_size); if (st->chunk_size > cksum_limit) { + i64 cksum_len = control->maxram; + void *buf; + + while (42) { + round_to_page(&cksum_len); + buf = malloc(cksum_len); + if (buf) { + print_maxverbose("Malloced %"PRId64" for checksum ckbuf\n", cksum_len); + break; + } + cksum_len = cksum_len / 3 * 2; + if (cksum_len < control->page_size) + failure("Failed to malloc any ram for checksum ckbuf\n"); + } + control->checksum.buf = buf; + /* Compute checksum. If the entire chunk is longer than maxram, * do it "per-partes" */ cksem_wait(control, &control->cksumsem); control->checksum.len = st->chunk_size - cksum_limit; - cksum_chunks = control->checksum.len / control->maxram; - cksum_remains = control->checksum.len % control->maxram; - - control->checksum.buf = malloc(control->maxram); - if (unlikely(!control->checksum.buf)) - failure("Failed to malloc ckbuf in hash_search2\n"); + cksum_chunks = control->checksum.len / cksum_len; + cksum_remains = control->checksum.len % cksum_len; for (i = 0; i < cksum_chunks; i++) { - control->do_mcpy(control, control->checksum.buf, cksum_limit, control->maxram); - cksum_limit += control->maxram; - st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->maxram); + control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_len); + cksum_limit += cksum_len; + st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_len); if (!NO_MD5) - md5_process_bytes(control->checksum.buf, control->maxram, &control->ctx); + md5_process_bytes(control->checksum.buf, cksum_len, &control->ctx); } /* Process end of the checksum buffer */ control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_remains); diff --git a/stream.c b/stream.c index c705407..96efd39 100644 --- a/stream.c +++ b/stream.c @@ -1,6 +1,6 @@ /* Copyright (C) 2011 Serge Belyshev - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998 Andrew Tridgell @@ -1122,7 +1122,8 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) } } sinfo->initial_pos = get_readseek(control, f); - if (unlikely(sinfo->initial_pos == -1)) goto failed; + if (unlikely(sinfo->initial_pos == -1)) + goto failed; for (i = 0; i < n; i++) { uchar c, enc_head[25 + SALT_LEN]; @@ -1172,8 +1173,11 @@ again: } sinfo->total_read += header_length; - if (ENCRYPT) - if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; + if (ENCRYPT) { + if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) + goto failed; + sinfo->total_read += SALT_LEN; + } v1 = le64toh(v1); v2 = le64toh(v2); @@ -1228,16 +1232,17 @@ static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, if (unlikely(!head)) fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false); buf = head + SALT_LEN; - if (unlikely(!get_rand(control, head, SALT_LEN))) goto error; + if (unlikely(!get_rand(control, head, SALT_LEN))) + goto error; if (unlikely(seekto(control, sinfo, ofs - SALT_LEN))) failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, head, SALT_LEN))) failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error); - if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error; + if (unlikely(!lrz_encrypt(control, buf, 25, head))) + goto error; if (unlikely(seekto(control, sinfo, ofs))) failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error); @@ -1307,7 +1312,8 @@ retry: cti->s_buf = realloc(cti->s_buf, MIN_SIZE); if (unlikely(!cti->s_buf)) fatal_goto(("Failed to realloc s_buf in compthread\n"), error); - if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error; + if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) + goto error; } /* If compression fails for whatever reason multithreaded, then wait @@ -1363,7 +1369,8 @@ retry: /* First chunk of this stream, write headers */ ctis->initial_pos = get_seek(control, ctis->fd); - if (unlikely(ctis->initial_pos == -1)) goto error; + if (unlikely(ctis->initial_pos == -1)) + goto error; print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos); for (j = 0; j < ctis->num_streams; j++) { @@ -1419,10 +1426,12 @@ retry: ctis->cur_pos += 1 + (write_len * 3); if (ENCRYPT) { - if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error; + if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) + goto error; if (unlikely(write_buf(control, cti->salt, SALT_LEN))) fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error); - if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; + if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) + goto error; ctis->cur_pos += SALT_LEN; } @@ -1572,8 +1581,11 @@ fill_another: if (unlikely(read_seekto(control, sinfo, s->last_head))) return -1; - if (unlikely(ENCRYPT && read_buf(control, sinfo->fd, enc_head, SALT_LEN))) - return -1; + if (ENCRYPT) { + if (unlikely(read_buf(control, sinfo->fd, enc_head, SALT_LEN))) + return -1; + sinfo->total_read += SALT_LEN; + } if (unlikely(read_u8(control, sinfo->fd, &c_type))) return -1; @@ -1608,7 +1620,6 @@ fill_another: return -1; header_length = 1 + (read_len * 3); } - print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head); sinfo->total_read += header_length; if (ENCRYPT) { @@ -1616,10 +1627,12 @@ fill_another: return -1; if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN))) return -1; + sinfo->total_read += SALT_LEN; } c_len = le64toh(c_len); u_len = le64toh(u_len); last_head = le64toh(last_head); + print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head); padded_len = MAX(c_len, MIN_SIZE); sinfo->total_read += padded_len; @@ -1635,9 +1648,10 @@ fill_another: if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len))) return -1; - if (ENCRYPT) + if (ENCRYPT) { if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt))) return -1; + } ucthread[s->uthread_no].s_buf = s_buf; ucthread[s->uthread_no].c_len = c_len; diff --git a/util.c b/util.c index 442a884..d79125b 100644 --- a/util.c +++ b/util.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Serge Belyshev Copyright (C) 2008, 2011 Peter Hyman Copyright (C) 1998 Andrew Tridgell