From 21925e4a9c35484abd33624b14e593ec0c5f8288 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 1 Sep 2013 15:46:48 +1000 Subject: [PATCH] =?UTF-8?q?Massive=20files=20fail=20with=20-U=20due=20to?= =?UTF-8?q?=20trying=20to=20allocate=20the=20whole=20lot=20in=20ram=20whil?= =?UTF-8?q?e=20doing=20checksums.=20Do=20it=20piecemeal=20to=20avoid=20the?= =?UTF-8?q?=20problem.=20Patch=20and=20debugging=20courtesy=20of=20Adam=20?= =?UTF-8?q?Tk=C3=A1=C4=8D.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rzip.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/rzip.c b/rzip.c index 907b83f..00bf2f3 100644 --- a/rzip.c +++ b/rzip.c @@ -602,9 +602,9 @@ static inline void cksum_update(rzip_control *control) static inline bool hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple) { + i64 cksum_limit = 0, p, end, cksum_chunks, cksum_remains, i; struct sliding_buffer *sb = &control->sb; int lastpct = 0, last_chunkpct = 0; - i64 cksum_limit = 0, p, end; tag t = 0; struct { i64 p; @@ -738,15 +738,29 @@ static inline bool hash_search(rzip_control *control, struct rzip_state *st, put_literal(control, st, st->last_match, st->chunk_size); if (st->chunk_size > cksum_limit) { + /* Compute checksum. If the entire chunk is longer than maxram, + * do it "per-partes" */ lock_mutex(control, &control->cksumlock); control->checksum.len = st->chunk_size - cksum_limit; + cksum_chunks = control->checksum.len / control->maxram; + cksum_remains = control->checksum.len % control->maxram; + control->checksum.buf = malloc(control->checksum.len); if (unlikely(!control->checksum.buf)) - fatal_return(("Failed to malloc ckbuf in hash_search\n"), false); - control->do_mcpy(control, control->checksum.buf, cksum_limit, control->checksum.len); - st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->checksum.len); + fatal_return(("Failed to malloc ckbuf in hash_search2\n"), false); + + for (i = 0; i < cksum_chunks; i++) { + control->do_mcpy(control, control->checksum.buf, cksum_limit, control->maxram); + cksum_limit += control->maxram; + st->cksum = CrcUpdate(st->cksum, control->checksum.buf, control->maxram); + if (!NO_MD5) + md5_process_bytes(control->checksum.buf, control->maxram, &control->ctx); + } + /* Process end of the checksum buffer */ + control->do_mcpy(control, control->checksum.buf, cksum_limit, cksum_remains); + st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_remains); if (!NO_MD5) - md5_process_bytes(control->checksum.buf, control->checksum.len, &control->ctx); + md5_process_bytes(control->checksum.buf, cksum_remains, &control->ctx); free(control->checksum.buf); unlock_mutex(control, &control->cksumlock); } else