From 0639d10de58ae1f9fe39232e72e1f6f98a6313b8 Mon Sep 17 00:00:00 2001 From: orthographic-pedant Date: Wed, 30 Sep 2015 15:23:03 -0400 Subject: [PATCH 01/10] Fixed typographical error, changed accomodate to accommodate in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed6bfff..00b59ea 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Simple 'n Easy™: `sudo make install` ### lrzip internals lrzip uses an extended version of [rzip](http://rzip.samba.org/) which does a first pass long distance -redundancy reduction. lrzip's modifications allow it to scale to accomodate various memory sizes. +redundancy reduction. lrzip's modifications allow it to scale to accommodate various memory sizes. Then, one of the following scenarios occurs: From b8c1e9ca63136e0c1ef4e4cd0b5b78f20b68323b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 09:47:13 +1000 Subject: [PATCH 02/10] Big endian fix for Solaris Sparc courtesy of joelfredrikson. --- lrzip_private.h | 4 ++++ rzip.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lrzip_private.h b/lrzip_private.h index 4a79918..fa29f44 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -67,6 +67,10 @@ void *alloca (size_t); # include #endif #ifndef __BYTE_ORDER +# ifndef __BIG_ENDIAN +# define __BIG_ENDIAN 4321 +# define __LITTLE_ENDIAN 1234 +# endif # ifdef WORDS_BIGENDIAN # define __BYTE_ORDER __BIG_ENDIAN # else diff --git a/rzip.c b/rzip.c index 38d7fc7..0320515 100644 --- a/rzip.c +++ b/rzip.c @@ -772,7 +772,7 @@ static inline void init_hash_indexes(struct rzip_state *st) st->hash_index[i] = ((random() << 16) ^ random()); } -#if defined(__APPLE__) || defined(__FreeBSD__) +#if !defined(__linux) # define mremap fake_mremap static inline void *fake_mremap(void *old_address, size_t old_size, size_t new_size, int flags __UNUSED__) From 7808f3dcbca8e3e5301ce0418cd69b294ee97be3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 09:56:17 +1000 Subject: [PATCH 03/10] Add support for -m option in lrztar --- lrztar | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lrztar b/lrztar index e773fe2..dddb9a3 100755 --- a/lrztar +++ b/lrztar @@ -47,14 +47,14 @@ Notice: v_w=0 v_S=0 v_D=0 v_p=0 v_q=0 v_L=0 \ v_n=0 v_l=0 v_b=0 v_g=0 v_z=0 v_U=0 \ v_T=0 v_N=0 v_v=0 v_f=0 v_d=0 v_h=0 \ - v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 x= i="$(pwd)" + v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 v_m=0 x= i="$(pwd)" which tar &> /dev/null \ || { printf "lrztar: no tar in your path\n"; return 1; } which lrzip &> /dev/null \ || { printf "lrztar: no lrzip in your path\n"; return 1; } which lrzcat &> /dev/null \ || { printf "lrztar: no lrzcat in your path\n"; return 1; } - while getopts w:O:S:DqL:nlbgzUTN:p:vfo:d:tVhHck x; do + while getopts w:O:S:DqL:nlbgzUm:TN:p:vfo:d:tVhHck x; do [[ $x == [tV] ]] && { printf "lrztar: invalid option for lrztar: %s\n" "$x"; return 1; From 21496ded07d9b0c05bcb5fbfd901ea145672b3c0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 10:18:26 +1000 Subject: [PATCH 04/10] Enable subdir objects for future automake compatibility --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7bfe011..b6d7159 100644 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,7 @@ AC_CONFIG_SRCDIR([configure.ac]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) -AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign]) +AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_USE_SYSTEM_EXTENSIONS From c7a111bd3279b3e57f001bb15e8353bf738caee3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 10:51:55 +1000 Subject: [PATCH 05/10] Base temporary output buffer on maximum mallocable, not maxram --- lrzip.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lrzip.c b/lrzip.c index 1bf39a3..b72b9ee 100644 --- a/lrzip.c +++ b/lrzip.c @@ -492,13 +492,25 @@ bool read_tmpinfile(rzip_control *control, int fd_in) * a pseudo-temporary file */ static bool open_tmpoutbuf(rzip_control *control) { + i64 maxlen = control->maxram; + void *buf; + + while (42) { + round_to_page(&maxlen); + buf = malloc(maxlen); + if (buf) { + print_maxverbose("Malloced %"PRId64" for tmp_outbuf\n", maxlen); + break; + } + maxlen = maxlen / 3 * 2; + if (maxlen < 100000000) + fatal_return(("Unable to even malloc 100MB for tmp_outbuf\n"), false); + } control->flags |= FLAG_TMP_OUTBUF; - control->out_maxlen = control->maxram; /* Allocate slightly more so we can cope when the buffer overflows and * fall back to a real temporary file */ - control->tmp_outbuf = malloc(control->maxram + control->page_size); - if (unlikely(!control->tmp_outbuf)) - fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false); + control->out_maxlen = maxlen - control->page_size; + control->tmp_outbuf = buf; if (!DECOMPRESS && !TEST_ONLY) control->out_ofs = control->out_len = MAGIC_LEN;\ return true; From 70bd5e9d3add335c67ea9535e6ea41af61edab3e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 11:04:40 +1000 Subject: [PATCH 06/10] Allow less than maxram to be malloced for checksum to fix Failed to malloc ckbuf in hash_search2 --- rzip.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/rzip.c b/rzip.c index 0320515..6bdff90 100644 --- a/rzip.c +++ b/rzip.c @@ -50,6 +50,7 @@ #ifdef HAVE_ARPA_INET_H # include #endif +#include #include "md5.h" #include "stream.h" @@ -729,23 +730,35 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st, put_literal(control, st, st->last_match, st->chunk_size); if (st->chunk_size > cksum_limit) { + i64 cksum_len = control->maxram; + void *buf; + + while (42) { + round_to_page(&cksum_len); + buf = malloc(cksum_len); + if (buf) { + print_maxverbose("Malloced %"PRId64" for checksum ckbuf\n", cksum_len); + break; + } + cksum_len = cksum_len / 3 * 2; + if (cksum_len < control->page_size) + failure("Failed to malloc any ram for checksum ckbuf\n"); + } + control->checksum.buf = buf; + /* Compute checksum. If the entire chunk is longer than maxram, * do it "per-partes" */ cksem_wait(control, &control->cksumsem); control->checksum.len = st->chunk_size - cksum_limit; - cksum_chunks = control->checksum.len / control->maxram; - cksum_remains = control->checksum.len % control->maxram; - - control->checksum.buf = malloc(control->maxram); - if (unlikely(!control->checksum.buf)) - failure("Failed to malloc ckbuf in hash_search2\n"); + cksum_chunks = control->checksum.len / cksum_len; + cksum_remains = control->checksum.len % cksum_len; for (i = 0; i < cksum_chunks; i++) { - control->do_mcpy(control, control->checksum.buf, cksum_limit, control->maxram); - cksum_limit += control->maxram; - st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->maxram); + control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_len); + cksum_limit += cksum_len; + st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_len); if (!NO_MD5) - md5_process_bytes(control->checksum.buf, control->maxram, &control->ctx); + md5_process_bytes(control->checksum.buf, cksum_len, &control->ctx); } /* Process end of the checksum buffer */ control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_remains); From 56a84987dd1bdaf0b9cd98c53c50685c82159240 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 11:16:41 +1000 Subject: [PATCH 07/10] Update copyright dates --- lrzip.c | 2 +- lrzip_private.h | 2 +- lrztar | 2 +- rzip.c | 2 +- util.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lrzip.c b/lrzip.c index b72b9ee..732d2ec 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998-2003 Andrew Tridgell diff --git a/lrzip_private.h b/lrzip_private.h index fa29f44..9ac9f47 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998-2003 Andrew Tridgell diff --git a/lrztar b/lrztar index dddb9a3..fafc108 100755 --- a/lrztar +++ b/lrztar @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (C) George Makrydakis 2009-2011,2013 -# Copyright (C) Con Kolivas 2011-2012 +# Copyright (C) Con Kolivas 2011-2012,2016 # A bash wrapper for Con Kolivas' excellent lrzip utility. For the time # being, lrzip does not like pipes, so we had to do this. It is kind of diff --git a/rzip.c b/rzip.c index 6bdff90..56a9291 100644 --- a/rzip.c +++ b/rzip.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 1998 Andrew Tridgell Modified to use flat hash, memory limit and variable hash culling diff --git a/util.c b/util.c index 442a884..d79125b 100644 --- a/util.c +++ b/util.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Serge Belyshev Copyright (C) 2008, 2011 Peter Hyman Copyright (C) 1998 Andrew Tridgell From 007bf5de93155c71bd77b3ce4cd164dee86eea68 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 11:48:04 +1000 Subject: [PATCH 08/10] Show correct lengths during testing on big endian and compressed archives --- stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream.c b/stream.c index 2af80a4..5e449b2 100644 --- a/stream.c +++ b/stream.c @@ -1,6 +1,6 @@ /* Copyright (C) 2011 Serge Belyshev - Copyright (C) 2006-2015 Con Kolivas + Copyright (C) 2006-2016 Con Kolivas Copyright (C) 2011 Peter Hyman Copyright (C) 1998 Andrew Tridgell @@ -1607,7 +1607,6 @@ fill_another: return -1; header_length = 1 + (read_len * 3); } - print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head); sinfo->total_read += header_length; if (ENCRYPT) { @@ -1619,6 +1618,7 @@ fill_another: c_len = le64toh(c_len); u_len = le64toh(u_len); last_head = le64toh(last_head); + print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head); padded_len = MAX(c_len, MIN_SIZE); sinfo->total_read += padded_len; From 5627479c6009387931cdb18b56120c5e773b48ff Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 13:56:54 +1000 Subject: [PATCH 09/10] Tidy gotos --- stream.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/stream.c b/stream.c index 5e449b2..4ebaeb4 100644 --- a/stream.c +++ b/stream.c @@ -1121,7 +1121,8 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) } } sinfo->initial_pos = get_readseek(control, f); - if (unlikely(sinfo->initial_pos == -1)) goto failed; + if (unlikely(sinfo->initial_pos == -1)) + goto failed; for (i = 0; i < n; i++) { uchar c, enc_head[25 + SALT_LEN]; @@ -1171,8 +1172,10 @@ again: } sinfo->total_read += header_length; - if (ENCRYPT) - if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; + if (ENCRYPT) { + if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) + goto failed; + } v1 = le64toh(v1); v2 = le64toh(v2); @@ -1227,16 +1230,17 @@ static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, if (unlikely(!head)) fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false); buf = head + SALT_LEN; - if (unlikely(!get_rand(control, head, SALT_LEN))) goto error; + if (unlikely(!get_rand(control, head, SALT_LEN))) + goto error; if (unlikely(seekto(control, sinfo, ofs - SALT_LEN))) failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, head, SALT_LEN))) failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error); - if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error; + if (unlikely(!lrz_encrypt(control, buf, 25, head))) + goto error; if (unlikely(seekto(control, sinfo, ofs))) failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error); @@ -1306,7 +1310,8 @@ retry: cti->s_buf = realloc(cti->s_buf, MIN_SIZE); if (unlikely(!cti->s_buf)) fatal_goto(("Failed to realloc s_buf in compthread\n"), error); - if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error; + if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) + goto error; } /* If compression fails for whatever reason multithreaded, then wait @@ -1362,7 +1367,8 @@ retry: /* First chunk of this stream, write headers */ ctis->initial_pos = get_seek(control, ctis->fd); - if (unlikely(ctis->initial_pos == -1)) goto error; + if (unlikely(ctis->initial_pos == -1)) + goto error; print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos); for (j = 0; j < ctis->num_streams; j++) { @@ -1418,10 +1424,12 @@ retry: ctis->cur_pos += 1 + (write_len * 3); if (ENCRYPT) { - if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error; + if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) + goto error; if (unlikely(write_buf(control, cti->salt, SALT_LEN))) fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error); - if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; + if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) + goto error; ctis->cur_pos += SALT_LEN; } From 7e92f4cb47e720cd3853e5371a22b5ce8b2e1fbe Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 9 Jun 2016 15:00:43 +1000 Subject: [PATCH 10/10] Fix decompression of multiple chunk encrypted archives --- stream.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/stream.c b/stream.c index 4ebaeb4..11e1420 100644 --- a/stream.c +++ b/stream.c @@ -1175,6 +1175,7 @@ again: if (ENCRYPT) { if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; + sinfo->total_read += SALT_LEN; } v1 = le64toh(v1); @@ -1579,8 +1580,11 @@ fill_another: if (unlikely(read_seekto(control, sinfo, s->last_head))) return -1; - if (unlikely(ENCRYPT && read_buf(control, sinfo->fd, enc_head, SALT_LEN))) - return -1; + if (ENCRYPT) { + if (unlikely(read_buf(control, sinfo->fd, enc_head, SALT_LEN))) + return -1; + sinfo->total_read += SALT_LEN; + } if (unlikely(read_u8(control, sinfo->fd, &c_type))) return -1; @@ -1622,6 +1626,7 @@ fill_another: return -1; if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN))) return -1; + sinfo->total_read += SALT_LEN; } c_len = le64toh(c_len); u_len = le64toh(u_len); @@ -1642,9 +1647,10 @@ fill_another: if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len))) return -1; - if (ENCRYPT) + if (ENCRYPT) { if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt))) return -1; + } ucthread[s->uthread_no].s_buf = s_buf; ucthread[s->uthread_no].c_len = c_len;