ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2026-01-08 17:39:59 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

We were attempting to truncate mmap to page size when only the offset needed to be.

Browse source 
Fix the longstanding limit on 32 bits that allowed us to allocate only 2GB of ram by moving the big malloc calls to mmap equivalents which allow us to mmap up to 2^44 bytes of anonymous space.
Use progressively smaller preallocation to try and defragment ram prior to real mmap call to increase success rate of allocating ram when it's a significant proportion of total ram.
Don't fail if preallocation is unsuccessful.
Add more detailed error reporting.
Minor cleanups.
This commit is contained in:
Con Kolivas 2010-11-01 00:19:39 +11:00
parent 19c2769061
commit 2bacbc60d2
3 changed files with 40 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
main.c
View file

@ -770,13 +770,6 @@ int main(int argc, char *argv[])
control.window = 1;
}
/* malloc limited to 2GB on 32bit */
if (sizeof(long) == 4 && control.window > 20) {
control.window = 20;
if (control.flags & FLAG_VERBOSE)
fprintf(stderr, "Limiting control window to 2GB due to 32bit limitations.\n");
}
/* OK, if verbosity set, print summary of options selected */
if ((control.flags & FLAG_VERBOSE) && !(control.flags & FLAG_INFO)) {
fprintf(stderr, "The following options are in effect for this %s.\n",

20
runzip.c
View file

@ -23,6 +23,7 @@
static inline uchar read_u8(void *ss, int stream)
{
uchar b;
if (read_stream(ss, stream, &b, 1) != 1)
fatal("Stream read u8 failed\n");
return b;
@ -69,11 +70,10 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum)
if (len < 0)
fatal("len %lld is negative in unzip_literal!\n",len);
if (sizeof(long) == 4 && len > (1UL << 31))
fatal("Unable to allocate a chunk this big on 32bit userspace. Can't decompress this file on this hardware\n");
buf = malloc((size_t)len);
if (!buf)
/* We use anonymous mmap instead of malloc to allow us to allocate up
* to 2^44 even on 32 bits */
buf = (uchar *)mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (buf == (uchar *)-1)
fatal("Failed to allocate literal buffer of size %lld\n", len);
read_stream(ss, 1, buf, len);
@ -82,7 +82,7 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum)
*cksum = CrcUpdate(*cksum, buf, len);
free(buf);
munmap(buf, len);
return len;
}
@ -133,18 +133,18 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
*/
static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally, char chunk_bytes)
{
i64 len, ofs, total = 0;
uint32 good_cksum, cksum = 0;
i64 len, ofs, total = 0;
int l = -1, p = 0;
struct stat st;
uchar head;
void *ss;
/* for display of progress */
char *suffix[] = {"","KB","MB","GB"};
unsigned long divisor[] = {1,1024,1048576,1073741824U};
int divisor_index;
char *suffix[] = {"","KB","MB","GB"};
double prog_done, prog_tsize;
int divisor_index;
if (expected_size > (i64)10737418240ULL) /* > 10GB */
divisor_index = 3;
@ -166,7 +166,7 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
ss = open_stream_in(fd_in, NUM_STREAMS);
if (!ss)
fatal(NULL);
fatal("Failed to open_stream_in in runzip_chunk\n");
while ((len = read_header(ss, &head)) || head) {
switch (head) {

56
rzip.c
View file

@ -121,7 +121,8 @@ static void put_match(struct rzip_state *st, uchar *p, uchar *buf, i64 offset, i
do {
i64 ofs;
i64 n = len;
if (n > 0xFFFF) n = 0xFFFF;
if (n > 0xFFFF)
n = 0xFFFF;
ofs = (p - (buf + offset));
put_header(st->ss, 1, n);
@ -139,7 +140,8 @@ static void put_literal(struct rzip_state *st, uchar *last, uchar *p)
{
do {
i64 len = (i64)(p - last);
if (len > 0xFFFF) len = 0xFFFF;
if (len > 0xFFFF)
len = 0xFFFF;
st->stats.literals++;
st->stats.literal_bytes += len;
@ -147,7 +149,7 @@ static void put_literal(struct rzip_state *st, uchar *last, uchar *p)
put_header(st->ss, 0, len);
if (len && write_stream(st->ss, 1, last, len) != 0)
fatal(NULL);
fatal("Failed to write_stream in put_literal\n");
last += len;
} while (p > last);
}
@ -220,9 +222,8 @@ static void insert_hash(struct rzip_state *st, tag t, i64 offset)
/* If we have lots of identical patterns, we end up
with lots of the same hash number. Discard random. */
if (st->hash_table[h].t == t) {
if (round == victim_round) {
if (round == victim_round)
victim_h = h;
}
if (++round == st->level->max_chain_len) {
h = victim_h;
st->hash_count--;
@ -401,9 +402,9 @@ static void hash_search(struct rzip_state *st, uchar *buf,
tag tag_mask = (1 << st->level->initial_freq) - 1;
if (st->hash_table) {
if (st->hash_table)
memset(st->hash_table, 0, sizeof(st->hash_table[0]) * (1<<st->hash_bits));
} else {
else {
i64 hashsize = st->level->mb_used *
(1024 * 1024 / sizeof(st->hash_table[0]));
for (st->hash_bits = 0; (1U << st->hash_bits) < hashsize; st->hash_bits++);
@ -527,7 +528,8 @@ static void init_hash_indexes(struct rzip_state *st)
static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
double pct_base, double pct_multiple, i64 limit)
{
uchar *buf;
i64 prealloc_size = st->chunk_size;
uchar *buf = (void *)-1;
/* Malloc'ing first will tell us if we can allocate this much ram
* faster than slowly reading in the file and then failing. Filling
@ -535,12 +537,23 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
* read in. */
if (control.flags & FLAG_VERBOSE)
fprintf(control.msgout, "Preallocating ram...\n");
buf = malloc(st->chunk_size);
if (!buf)
fatal("Failed to premalloc in rzip_chunk\n");
if (!memset(buf, 0, st->chunk_size))
fatal("Failed to memset in rzip_chunk\n");
free(buf);
while (buf == (void*)-1) {
/* If we fail to mmap the full amount, it is worth trying to
* mmap ever smaller sizes till we succeed as we may be able
* to continue with file backed mmap in the presence of swap
* and defragmentation */
buf = mmap(NULL, prealloc_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (buf == (void *)-1) {
prealloc_size = prealloc_size / 10 * 9;
continue;
}
if (control.flags & FLAG_VERBOSE)
fprintf(control.msgout, "Preallocated %lld ram...\n", prealloc_size);
if (!memset(buf, 0, prealloc_size))
fatal("Failed to memset in rzip_chunk\n");
if (munmap(buf, prealloc_size) != 0)
fatal("Failed to munmap in rzip_chunk\n");
}
if (control.flags & FLAG_VERBOSE)
fprintf(control.msgout, "Reading file into mmapped ram...\n");
buf = (uchar *)mmap(buf, st->chunk_size, PROT_READ, MAP_SHARED, fd_in, offset);
@ -552,21 +565,13 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
fatal("Failed to open streams in rzip_chunk\n");
hash_search(st, buf, pct_base, pct_multiple);
/* unmap buffer before closing and reallocating streams */
munmap(buf, st->chunk_size);
if (munmap(buf, st->chunk_size) != 0)
fatal("Failed to munmap in rzip_chunk\n");
if (close_stream_out(st->ss) != 0)
fatal("Failed to flush/close streams in rzip_chunk\n");
}
/* Windows must be the width of _SC_PAGE_SIZE for offset to work in mmap */
static void round_to_page_size(i64 *chunk)
{
unsigned long page_size = sysconf(_SC_PAGE_SIZE);
i64 pages = *chunk / page_size;
*chunk = pages * page_size;
}
/* compress a whole file chunks at a time */
void rzip_fd(int fd_in, int fd_out)
{
@ -609,7 +614,6 @@ void rzip_fd(int fd_in, int fd_out)
fprintf(control.msgout, "Byte width: %d\n", st->chunk_bytes);
chunk_window = control.window * CHUNK_MULTIPLE;
round_to_page_size(&chunk_window);
st->level = &levels[MIN(9, control.window)];
st->fd_in = fd_in;
@ -623,7 +627,7 @@ void rzip_fd(int fd_in, int fd_out)
last.tv_sec = last.tv_usec = 0;
gettimeofday(&start, NULL);
while (len > 0) {
while (len) {
double pct_base, pct_multiple;
i64 chunk, limit = 0;