Merge branch 'master' into compat

README.md

@@ -93,7 +93,7 @@ Simple 'n Easy™: `sudo make install`
 ### lrzip internals
 
 lrzip uses an extended version of [rzip](http://rzip.samba.org/) which does a first pass long distance
-redundancy reduction. lrzip's modifications allow it to scale to accomodate various memory sizes.
+redundancy reduction. lrzip's modifications allow it to scale to accommodate various memory sizes.
 
 Then, one of the following scenarios occurs:
 

configure.ac

@@ -20,7 +20,7 @@ AC_CONFIG_SRCDIR([configure.ac])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
 
-AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign])
+AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_USE_SYSTEM_EXTENSIONS
 

lrzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2015 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -513,13 +513,25 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
  * a pseudo-temporary file */
 static bool open_tmpoutbuf(rzip_control *control)
 {
+	i64 maxlen = control->maxram;
+	void *buf;
+
+	while (42) {
+		round_to_page(&maxlen);
+		buf = malloc(maxlen);
+		if (buf) {
+			print_maxverbose("Malloced %"PRId64" for tmp_outbuf\n", maxlen);
+			break;
+		}
+		maxlen = maxlen / 3 * 2;
+		if (maxlen < 100000000)
+			fatal_return(("Unable to even malloc 100MB for tmp_outbuf\n"), false);
+	}
 	control->flags |= FLAG_TMP_OUTBUF;
-	control->out_maxlen = control->maxram;
 	/* Allocate slightly more so we can cope when the buffer overflows and
 	 * fall back to a real temporary file */
-	control->tmp_outbuf = malloc(control->maxram + control->page_size);
+	control->out_maxlen = maxlen - control->page_size;
-	if (unlikely(!control->tmp_outbuf))
+	control->tmp_outbuf = buf;
-		fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false);
 	if (!DECOMPRESS && !TEST_ONLY)
 		control->out_ofs = control->out_len = MAGIC_LEN;\
 	return true;

lrzip_private.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2015 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 

lrztar

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Copyright (C) George Makrydakis 2009-2011,2013
-# Copyright (C) Con Kolivas 2011-2012
+# Copyright (C) Con Kolivas 2011-2012,2016
 
 # A bash wrapper for Con Kolivas' excellent lrzip utility. For the time
 # being, lrzip does not like pipes, so we had to do this. It is kind of
@@ -47,14 +47,14 @@ Notice:
           v_w=0 v_S=0 v_D=0 v_p=0 v_q=0 v_L=0 \
           v_n=0 v_l=0 v_b=0 v_g=0 v_z=0 v_U=0 \
           v_T=0 v_N=0 v_v=0 v_f=0 v_d=0 v_h=0 \
-          v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 x= i="$(pwd)"
+          v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 v_m=0 x= i="$(pwd)"
     which tar   &> /dev/null \
         || { printf "lrztar: no tar in your path\n"; return 1; }
     which lrzip &> /dev/null \
         || { printf "lrztar: no lrzip in your path\n"; return 1; }
     which lrzcat &> /dev/null \
         || { printf "lrztar: no lrzcat in your path\n"; return 1; }
-    while getopts w:O:S:DqL:nlbgzUTN:p:vfo:d:tVhHck x; do
+    while getopts w:O:S:DqL:nlbgzUm:TN:p:vfo:d:tVhHck x; do
         [[ $x == [tV] ]] && {
             printf "lrztar: invalid option for lrztar: %s\n" "$x";
             return 1;

rzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2015 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 1998 Andrew Tridgell
 
    Modified to use flat hash, memory limit and variable hash culling
@@ -50,6 +50,7 @@
 #ifdef HAVE_ARPA_INET_H
 # include <arpa/inet.h>
 #endif
+#include <inttypes.h>
 
 #include "md5.h"
 #include "stream.h"
@@ -729,23 +730,35 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st,
 		put_literal(control, st, st->last_match, st->chunk_size);
 
 	if (st->chunk_size > cksum_limit) {
+		i64 cksum_len = control->maxram;
+		void *buf;
+
+		while (42) {
+			round_to_page(&cksum_len);
+			buf = malloc(cksum_len);
+			if (buf) {
+				print_maxverbose("Malloced %"PRId64" for checksum ckbuf\n", cksum_len);
+				break;
+			}
+			cksum_len = cksum_len / 3 * 2;
+			if (cksum_len < control->page_size)
+				failure("Failed to malloc any ram for checksum ckbuf\n");
+		}
+		control->checksum.buf = buf;
+
 		/* Compute checksum. If the entire chunk is longer than maxram,
 		 * do it "per-partes" */
 		cksem_wait(control, &control->cksumsem);
 		control->checksum.len = st->chunk_size - cksum_limit;
-		cksum_chunks = control->checksum.len / control->maxram;
+		cksum_chunks = control->checksum.len / cksum_len;
-		cksum_remains = control->checksum.len % control->maxram;
+		cksum_remains = control->checksum.len % cksum_len;
-
-		control->checksum.buf = malloc(control->maxram);
-		if (unlikely(!control->checksum.buf))
-			failure("Failed to malloc ckbuf in hash_search2\n");
 
 		for (i = 0; i < cksum_chunks; i++) {
-			control->do_mcpy(control, control->checksum.buf, cksum_limit, control->maxram);
+			control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_len);
-			cksum_limit += control->maxram;
+			cksum_limit += cksum_len;
-			st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->maxram);
+			st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_len);
 			if (!NO_MD5)
-				md5_process_bytes(control->checksum.buf, control->maxram, &control->ctx);
+				md5_process_bytes(control->checksum.buf, cksum_len, &control->ctx);
 		}
 		/* Process end of the checksum buffer */
 		control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_remains);

stream.c

@@ -1,6 +1,6 @@
 /*
    Copyright (C) 2011 Serge Belyshev
-   Copyright (C) 2006-2015 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell
 
@@ -1122,7 +1122,8 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes)
 		}
 	}
 	sinfo->initial_pos = get_readseek(control, f);
-	if (unlikely(sinfo->initial_pos == -1)) goto failed;
+	if (unlikely(sinfo->initial_pos == -1))
+		goto failed;
 
 	for (i = 0; i < n; i++) {
 		uchar c, enc_head[25 + SALT_LEN];
@@ -1172,8 +1173,11 @@ again:
 		}
 		sinfo->total_read += header_length;
 
-		if (ENCRYPT)
+		if (ENCRYPT) {
-			if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed;
+			if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head)))
+				goto failed;
+			sinfo->total_read += SALT_LEN;
+		}
 
 		v1 = le64toh(v1);
 		v2 = le64toh(v2);
@@ -1228,16 +1232,17 @@ static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo,
 	if (unlikely(!head))
 		fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false);
 	buf = head + SALT_LEN;
-	if (unlikely(!get_rand(control, head, SALT_LEN))) goto error;
+	if (unlikely(!get_rand(control, head, SALT_LEN)))
+		goto error;
 	if (unlikely(seekto(control, sinfo, ofs - SALT_LEN)))
 		failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error);
 	if (unlikely(write_buf(control, head, SALT_LEN)))
 		failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error);
 	if (unlikely(read_buf(control, sinfo->fd, buf, 25)))
-
 		failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error);
 
-	if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error;
+	if (unlikely(!lrz_encrypt(control, buf, 25, head)))
+		goto error;
 
 	if (unlikely(seekto(control, sinfo, ofs)))
 		failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error);
@@ -1307,7 +1312,8 @@ retry:
 		cti->s_buf = realloc(cti->s_buf, MIN_SIZE);
 		if (unlikely(!cti->s_buf))
 			fatal_goto(("Failed to realloc s_buf in compthread\n"), error);
-		if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error;
+		if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len)))
+			goto error;
 	}
 
 	/* If compression fails for whatever reason multithreaded, then wait
@@ -1363,7 +1369,8 @@ retry:
 
 		/* First chunk of this stream, write headers */
 		ctis->initial_pos = get_seek(control, ctis->fd);
-		if (unlikely(ctis->initial_pos == -1)) goto error;
+		if (unlikely(ctis->initial_pos == -1))
+			goto error;
 
 		print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos);
 		for (j = 0; j < ctis->num_streams; j++) {
@@ -1419,10 +1426,12 @@ retry:
 	ctis->cur_pos += 1 + (write_len * 3);
 
 	if (ENCRYPT) {
-		if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error;
+		if (unlikely(!get_rand(control, cti->salt, SALT_LEN)))
+			goto error;
 		if (unlikely(write_buf(control, cti->salt, SALT_LEN)))
 			fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error);
-		if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error;
+		if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt)))
+			goto error;
 		ctis->cur_pos += SALT_LEN;
 	}
 
@@ -1572,8 +1581,11 @@ fill_another:
 	if (unlikely(read_seekto(control, sinfo, s->last_head)))
 		return -1;
 
-	if (unlikely(ENCRYPT && read_buf(control, sinfo->fd, enc_head, SALT_LEN)))
+	if (ENCRYPT) {
+		if (unlikely(read_buf(control, sinfo->fd, enc_head, SALT_LEN)))
 			return -1;
+		sinfo->total_read += SALT_LEN;
+	}
 
 	if (unlikely(read_u8(control, sinfo->fd, &c_type)))
 		return -1;
@@ -1608,7 +1620,6 @@ fill_another:
 			return -1;
 		header_length = 1 + (read_len * 3);
 	}
-	print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head);
 	sinfo->total_read += header_length;
 
 	if (ENCRYPT) {
@@ -1616,10 +1627,12 @@ fill_another:
 			return -1;
 		if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN)))
 			return -1;
+		sinfo->total_read += SALT_LEN;
 	}
 	c_len = le64toh(c_len);
 	u_len = le64toh(u_len);
 	last_head = le64toh(last_head);
+	print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head);
 
 	padded_len = MAX(c_len, MIN_SIZE);
 	sinfo->total_read += padded_len;
@@ -1635,9 +1648,10 @@ fill_another:
 	if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len)))
 		return -1;
 
-	if (ENCRYPT)
+	if (ENCRYPT) {
 		if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt)))
 			return -1;
+	}
 
 	ucthread[s->uthread_no].s_buf = s_buf;
 	ucthread[s->uthread_no].c_len = c_len;

util.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2015 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Serge Belyshev
    Copyright (C) 2008, 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell