ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into compat

Browse source
This commit is contained in:
Con Kolivas 2016-06-09 15:01:44 +10:00
parent cbc3789d5c 7e92f4cb47
commit 73213b9796
8 changed files with 77 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -93,7 +93,7 @@ Simple 'n Easy™: `sudo make install`
### lrzip internals
lrzip uses an extended version of [rzip](http://rzip.samba.org/) which does a first pass long distance
redundancy reduction. lrzip's modifications allow it to scale to accomodate various memory sizes.
redundancy reduction. lrzip's modifications allow it to scale to accommodate various memory sizes.
Then, one of the following scenarios occurs:

2
configure.ac
View file

@ -20,7 +20,7 @@ AC_CONFIG_SRCDIR([configure.ac])
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_HEADERS([config.h])
AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign])
AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
AC_USE_SYSTEM_EXTENSIONS

22
lrzip.c
View file

@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell
@ -513,13 +513,25 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
* a pseudo-temporary file */
static bool open_tmpoutbuf(rzip_control *control)
{
i64 maxlen = control->maxram;
void *buf;
while (42) {
round_to_page(&maxlen);
buf = malloc(maxlen);
if (buf) {
print_maxverbose("Malloced %"PRId64" for tmp_outbuf\n", maxlen);
break;
}
maxlen = maxlen / 3 * 2;
if (maxlen < 100000000)
fatal_return(("Unable to even malloc 100MB for tmp_outbuf\n"), false);
}
control->flags |= FLAG_TMP_OUTBUF;
control->out_maxlen = control->maxram;
/* Allocate slightly more so we can cope when the buffer overflows and
* fall back to a real temporary file */
control->tmp_outbuf = malloc(control->maxram + control->page_size);
if (unlikely(!control->tmp_outbuf))
fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false);
control->out_maxlen = maxlen - control->page_size;
control->tmp_outbuf = buf;
if (!DECOMPRESS && !TEST_ONLY)
control->out_ofs = control->out_len = MAGIC_LEN;\
return true;

2
lrzip_private.h
View file

@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998-2003 Andrew Tridgell

6
lrztar
View file

@ -1,7 +1,7 @@
#!/bin/bash
# Copyright (C) George Makrydakis 2009-2011,2013
# Copyright (C) Con Kolivas 2011-2012
# Copyright (C) Con Kolivas 2011-2012,2016
# A bash wrapper for Con Kolivas' excellent lrzip utility. For the time
# being, lrzip does not like pipes, so we had to do this. It is kind of
@ -47,14 +47,14 @@ Notice:
v_w=0 v_S=0 v_D=0 v_p=0 v_q=0 v_L=0 \
v_n=0 v_l=0 v_b=0 v_g=0 v_z=0 v_U=0 \
v_T=0 v_N=0 v_v=0 v_f=0 v_d=0 v_h=0 \
v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 x= i="$(pwd)"
v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 v_m=0 x= i="$(pwd)"
which tar &> /dev/null \
|| { printf "lrztar: no tar in your path\n"; return 1; }
which lrzip &> /dev/null \
|| { printf "lrztar: no lrzip in your path\n"; return 1; }
which lrzcat &> /dev/null \
|| { printf "lrztar: no lrzcat in your path\n"; return 1; }
while getopts w:O:S:DqL:nlbgzUTN:p:vfo:d:tVhHck x; do
while getopts w:O:S:DqL:nlbgzUm:TN:p:vfo:d:tVhHck x; do
[[ $x == [tV] ]] && {
printf "lrztar: invalid option for lrztar: %s\n" "$x";
return 1;

35
rzip.c
View file

@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 1998 Andrew Tridgell
Modified to use flat hash, memory limit and variable hash culling
@ -50,6 +50,7 @@
#ifdef HAVE_ARPA_INET_H
# include <arpa/inet.h>
#endif
#include <inttypes.h>
#include "md5.h"
#include "stream.h"
@ -729,23 +730,35 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st,
put_literal(control, st, st->last_match, st->chunk_size);
if (st->chunk_size > cksum_limit) {
i64 cksum_len = control->maxram;
void *buf;
while (42) {
round_to_page(&cksum_len);
buf = malloc(cksum_len);
if (buf) {
print_maxverbose("Malloced %"PRId64" for checksum ckbuf\n", cksum_len);
break;
}
cksum_len = cksum_len / 3 * 2;
if (cksum_len < control->page_size)
failure("Failed to malloc any ram for checksum ckbuf\n");
}
control->checksum.buf = buf;
/* Compute checksum. If the entire chunk is longer than maxram,
* do it "per-partes" */
cksem_wait(control, &control->cksumsem);
control->checksum.len = st->chunk_size - cksum_limit;
cksum_chunks = control->checksum.len / control->maxram;
cksum_remains = control->checksum.len % control->maxram;
control->checksum.buf = malloc(control->maxram);
if (unlikely(!control->checksum.buf))
failure("Failed to malloc ckbuf in hash_search2\n");
cksum_chunks = control->checksum.len / cksum_len;
cksum_remains = control->checksum.len % cksum_len;
for (i = 0; i < cksum_chunks; i++) {
control->do_mcpy(control, control->checksum.buf, cksum_limit, control->maxram);
cksum_limit += control->maxram;
st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->maxram);
control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_len);
cksum_limit += cksum_len;
st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_len);
if (!NO_MD5)
md5_process_bytes(control->checksum.buf, control->maxram, &control->ctx);
md5_process_bytes(control->checksum.buf, cksum_len, &control->ctx);
}
/* Process end of the checksum buffer */
control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_remains);

42
stream.c
View file

@ -1,6 +1,6 @@
/*
Copyright (C) 2011 Serge Belyshev
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 2011 Peter Hyman
Copyright (C) 1998 Andrew Tridgell
@ -1122,7 +1122,8 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes)
}
}
sinfo->initial_pos = get_readseek(control, f);
if (unlikely(sinfo->initial_pos == -1)) goto failed;
if (unlikely(sinfo->initial_pos == -1))
goto failed;
for (i = 0; i < n; i++) {
uchar c, enc_head[25 + SALT_LEN];
@ -1172,8 +1173,11 @@ again:
}
sinfo->total_read += header_length;
if (ENCRYPT)
if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed;
if (ENCRYPT) {
if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head)))
goto failed;
sinfo->total_read += SALT_LEN;
}
v1 = le64toh(v1);
v2 = le64toh(v2);
@ -1228,16 +1232,17 @@ static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo,
if (unlikely(!head))
fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false);
buf = head + SALT_LEN;
if (unlikely(!get_rand(control, head, SALT_LEN))) goto error;
if (unlikely(!get_rand(control, head, SALT_LEN)))
goto error;
if (unlikely(seekto(control, sinfo, ofs - SALT_LEN)))
failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error);
if (unlikely(write_buf(control, head, SALT_LEN)))
failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error);
if (unlikely(read_buf(control, sinfo->fd, buf, 25)))
failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error);
if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error;
if (unlikely(!lrz_encrypt(control, buf, 25, head)))
goto error;
if (unlikely(seekto(control, sinfo, ofs)))
failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error);
@ -1307,7 +1312,8 @@ retry:
cti->s_buf = realloc(cti->s_buf, MIN_SIZE);
if (unlikely(!cti->s_buf))
fatal_goto(("Failed to realloc s_buf in compthread\n"), error);
if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error;
if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len)))
goto error;
}
/* If compression fails for whatever reason multithreaded, then wait
@ -1363,7 +1369,8 @@ retry:
/* First chunk of this stream, write headers */
ctis->initial_pos = get_seek(control, ctis->fd);
if (unlikely(ctis->initial_pos == -1)) goto error;
if (unlikely(ctis->initial_pos == -1))
goto error;
print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos);
for (j = 0; j < ctis->num_streams; j++) {
@ -1419,10 +1426,12 @@ retry:
ctis->cur_pos += 1 + (write_len * 3);
if (ENCRYPT) {
if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error;
if (unlikely(!get_rand(control, cti->salt, SALT_LEN)))
goto error;
if (unlikely(write_buf(control, cti->salt, SALT_LEN)))
fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error);
if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error;
if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt)))
goto error;
ctis->cur_pos += SALT_LEN;
}
@ -1572,8 +1581,11 @@ fill_another:
if (unlikely(read_seekto(control, sinfo, s->last_head)))
return -1;
if (unlikely(ENCRYPT && read_buf(control, sinfo->fd, enc_head, SALT_LEN)))
if (ENCRYPT) {
if (unlikely(read_buf(control, sinfo->fd, enc_head, SALT_LEN)))
return -1;
sinfo->total_read += SALT_LEN;
}
if (unlikely(read_u8(control, sinfo->fd, &c_type)))
return -1;
@ -1608,7 +1620,6 @@ fill_another:
return -1;
header_length = 1 + (read_len * 3);
}
print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head);
sinfo->total_read += header_length;
if (ENCRYPT) {
@ -1616,10 +1627,12 @@ fill_another:
return -1;
if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN)))
return -1;
sinfo->total_read += SALT_LEN;
}
c_len = le64toh(c_len);
u_len = le64toh(u_len);
last_head = le64toh(last_head);
print_maxverbose("Fill_buffer stream %d c_len %lld u_len %lld last_head %lld\n", streamno, c_len, u_len, last_head);
padded_len = MAX(c_len, MIN_SIZE);
sinfo->total_read += padded_len;
@ -1635,9 +1648,10 @@ fill_another:
if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len)))
return -1;
if (ENCRYPT)
if (ENCRYPT) {
if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt)))
return -1;
}
ucthread[s->uthread_no].s_buf = s_buf;
ucthread[s->uthread_no].c_len = c_len;

2
util.c
View file

@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2015 Con Kolivas
Copyright (C) 2006-2016 Con Kolivas
Copyright (C) 2011 Serge Belyshev
Copyright (C) 2008, 2011 Peter Hyman
Copyright (C) 1998 Andrew Tridgell