diff --git a/main.c b/main.c index dabdc4f..0a57b53 100644 --- a/main.c +++ b/main.c @@ -770,13 +770,6 @@ int main(int argc, char *argv[]) control.window = 1; } - /* malloc limited to 2GB on 32bit */ - if (sizeof(long) == 4 && control.window > 20) { - control.window = 20; - if (control.flags & FLAG_VERBOSE) - fprintf(stderr, "Limiting control window to 2GB due to 32bit limitations.\n"); - } - /* OK, if verbosity set, print summary of options selected */ if ((control.flags & FLAG_VERBOSE) && !(control.flags & FLAG_INFO)) { fprintf(stderr, "The following options are in effect for this %s.\n", diff --git a/runzip.c b/runzip.c index 0ff4c6d..f5ae3a7 100644 --- a/runzip.c +++ b/runzip.c @@ -23,6 +23,7 @@ static inline uchar read_u8(void *ss, int stream) { uchar b; + if (read_stream(ss, stream, &b, 1) != 1) fatal("Stream read u8 failed\n"); return b; @@ -69,11 +70,10 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum) if (len < 0) fatal("len %lld is negative in unzip_literal!\n",len); - if (sizeof(long) == 4 && len > (1UL << 31)) - fatal("Unable to allocate a chunk this big on 32bit userspace. Can't decompress this file on this hardware\n"); - - buf = malloc((size_t)len); - if (!buf) + /* We use anonymous mmap instead of malloc to allow us to allocate up + * to 2^44 even on 32 bits */ + buf = (uchar *)mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (buf == (uchar *)-1) fatal("Failed to allocate literal buffer of size %lld\n", len); read_stream(ss, 1, buf, len); @@ -82,7 +82,7 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum) *cksum = CrcUpdate(*cksum, buf, len); - free(buf); + munmap(buf, len); return len; } @@ -133,18 +133,18 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum */ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally, char chunk_bytes) { - i64 len, ofs, total = 0; uint32 good_cksum, cksum = 0; + i64 len, ofs, total = 0; int l = -1, p = 0; struct stat st; uchar head; void *ss; /* for display of progress */ - char *suffix[] = {"","KB","MB","GB"}; unsigned long divisor[] = {1,1024,1048576,1073741824U}; - int divisor_index; + char *suffix[] = {"","KB","MB","GB"}; double prog_done, prog_tsize; + int divisor_index; if (expected_size > (i64)10737418240ULL) /* > 10GB */ divisor_index = 3; @@ -166,7 +166,7 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i ss = open_stream_in(fd_in, NUM_STREAMS); if (!ss) - fatal(NULL); + fatal("Failed to open_stream_in in runzip_chunk\n"); while ((len = read_header(ss, &head)) || head) { switch (head) { diff --git a/rzip.c b/rzip.c index 1579f6b..8e5d4a0 100644 --- a/rzip.c +++ b/rzip.c @@ -121,7 +121,8 @@ static void put_match(struct rzip_state *st, uchar *p, uchar *buf, i64 offset, i do { i64 ofs; i64 n = len; - if (n > 0xFFFF) n = 0xFFFF; + if (n > 0xFFFF) + n = 0xFFFF; ofs = (p - (buf + offset)); put_header(st->ss, 1, n); @@ -139,7 +140,8 @@ static void put_literal(struct rzip_state *st, uchar *last, uchar *p) { do { i64 len = (i64)(p - last); - if (len > 0xFFFF) len = 0xFFFF; + if (len > 0xFFFF) + len = 0xFFFF; st->stats.literals++; st->stats.literal_bytes += len; @@ -147,7 +149,7 @@ static void put_literal(struct rzip_state *st, uchar *last, uchar *p) put_header(st->ss, 0, len); if (len && write_stream(st->ss, 1, last, len) != 0) - fatal(NULL); + fatal("Failed to write_stream in put_literal\n"); last += len; } while (p > last); } @@ -220,9 +222,8 @@ static void insert_hash(struct rzip_state *st, tag t, i64 offset) /* If we have lots of identical patterns, we end up with lots of the same hash number. Discard random. */ if (st->hash_table[h].t == t) { - if (round == victim_round) { + if (round == victim_round) victim_h = h; - } if (++round == st->level->max_chain_len) { h = victim_h; st->hash_count--; @@ -401,9 +402,9 @@ static void hash_search(struct rzip_state *st, uchar *buf, tag tag_mask = (1 << st->level->initial_freq) - 1; - if (st->hash_table) { + if (st->hash_table) memset(st->hash_table, 0, sizeof(st->hash_table[0]) * (1<hash_bits)); - } else { + else { i64 hashsize = st->level->mb_used * (1024 * 1024 / sizeof(st->hash_table[0])); for (st->hash_bits = 0; (1U << st->hash_bits) < hashsize; st->hash_bits++); @@ -527,7 +528,8 @@ static void init_hash_indexes(struct rzip_state *st) static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset, double pct_base, double pct_multiple, i64 limit) { - uchar *buf; + i64 prealloc_size = st->chunk_size; + uchar *buf = (void *)-1; /* Malloc'ing first will tell us if we can allocate this much ram * faster than slowly reading in the file and then failing. Filling @@ -535,12 +537,23 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset, * read in. */ if (control.flags & FLAG_VERBOSE) fprintf(control.msgout, "Preallocating ram...\n"); - buf = malloc(st->chunk_size); - if (!buf) - fatal("Failed to premalloc in rzip_chunk\n"); - if (!memset(buf, 0, st->chunk_size)) - fatal("Failed to memset in rzip_chunk\n"); - free(buf); + while (buf == (void*)-1) { + /* If we fail to mmap the full amount, it is worth trying to + * mmap ever smaller sizes till we succeed as we may be able + * to continue with file backed mmap in the presence of swap + * and defragmentation */ + buf = mmap(NULL, prealloc_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (buf == (void *)-1) { + prealloc_size = prealloc_size / 10 * 9; + continue; + } + if (control.flags & FLAG_VERBOSE) + fprintf(control.msgout, "Preallocated %lld ram...\n", prealloc_size); + if (!memset(buf, 0, prealloc_size)) + fatal("Failed to memset in rzip_chunk\n"); + if (munmap(buf, prealloc_size) != 0) + fatal("Failed to munmap in rzip_chunk\n"); + } if (control.flags & FLAG_VERBOSE) fprintf(control.msgout, "Reading file into mmapped ram...\n"); buf = (uchar *)mmap(buf, st->chunk_size, PROT_READ, MAP_SHARED, fd_in, offset); @@ -552,21 +565,13 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset, fatal("Failed to open streams in rzip_chunk\n"); hash_search(st, buf, pct_base, pct_multiple); /* unmap buffer before closing and reallocating streams */ - munmap(buf, st->chunk_size); + if (munmap(buf, st->chunk_size) != 0) + fatal("Failed to munmap in rzip_chunk\n"); if (close_stream_out(st->ss) != 0) fatal("Failed to flush/close streams in rzip_chunk\n"); } -/* Windows must be the width of _SC_PAGE_SIZE for offset to work in mmap */ -static void round_to_page_size(i64 *chunk) -{ - unsigned long page_size = sysconf(_SC_PAGE_SIZE); - i64 pages = *chunk / page_size; - - *chunk = pages * page_size; -} - /* compress a whole file chunks at a time */ void rzip_fd(int fd_in, int fd_out) { @@ -609,7 +614,6 @@ void rzip_fd(int fd_in, int fd_out) fprintf(control.msgout, "Byte width: %d\n", st->chunk_bytes); chunk_window = control.window * CHUNK_MULTIPLE; - round_to_page_size(&chunk_window); st->level = &levels[MIN(9, control.window)]; st->fd_in = fd_in; @@ -623,7 +627,7 @@ void rzip_fd(int fd_in, int fd_out) last.tv_sec = last.tv_usec = 0; gettimeofday(&start, NULL); - while (len > 0) { + while (len) { double pct_base, pct_multiple; i64 chunk, limit = 0;