diff --git a/lrzip_private.h b/lrzip_private.h index 1dc9b25..b3eb03f 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -213,6 +213,7 @@ struct rzip_control { const char *suffix; int compression_level; i64 overhead; // compressor overhead + i64 usable_ram; // the most ram we'll try to use on one activity i64 maxram; // the largest chunk of ram to allocate unsigned char lzma_properties[5]; // lzma properties, encoded i64 window; diff --git a/main.c b/main.c index fea8be1..2edd6d4 100644 --- a/main.c +++ b/main.c @@ -766,16 +766,15 @@ int main(int argc, char *argv[]) else control.maxram = control.ramsize / 3; if (BITS32) { - i64 usable_ram; - /* Decrease usable ram size on 32 bits due to kernel / * userspace split. Cannot allocate larger than a 1 * gigabyte chunk due to 32 bit signed long being * used in alloc */ - usable_ram = MAX(control.ramsize - 900000000ll, 900000000ll); - control.maxram = MIN(control.maxram, usable_ram); + control.usable_ram = MAX(control.ramsize - 900000000ll, 900000000ll); + control.maxram = MIN(control.maxram, control.usable_ram); control.maxram = MIN(control.maxram, one_g); - } + } else + control.usable_ram = control.maxram; round_to_page(&control.maxram); show_summary(); diff --git a/rzip.c b/rzip.c index e671711..12c8460 100644 --- a/rzip.c +++ b/rzip.c @@ -812,11 +812,6 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) * buffer to work on 2/3 ram size, leaving enough ram for the * compression backends */ control->max_mmap = control->maxram; - - /* On 32 bits we can have a big window with sliding mmap, but can - * not enable much per mmap/malloc */ - if (BITS32) - control->max_mmap = MIN((unsigned long long)control->max_mmap, two_gig); round_to_page(&control->max_mmap); /* Set maximum chunk size to 2/3 of ram if not unlimited or specified diff --git a/stream.c b/stream.c index 76b70dc..b0840eb 100644 --- a/stream.c +++ b/stream.c @@ -985,14 +985,9 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li else testbufs = 2; - /* Serious limits imposed on 32 bit capabilities */ - if (BITS32) - limit = MIN((unsigned long long)limit, (two_gig / testbufs) - - (control->overhead * control->threads)); - testsize = (limit * testbufs) + (control->overhead * control->threads); - if (testsize > control->maxram) - limit = (control->maxram - (control->overhead * control->threads)) / testbufs; + if (testsize > control->usable_ram) + limit = (control->usable_ram - (control->overhead * control->threads)) / testbufs; /* If we don't have enough ram for the number of threads, decrease the * number of threads till we do, or only have one thread. */ @@ -1001,17 +996,30 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li --control->threads; else break; - limit = (control->maxram - (control->overhead * control->threads)) / testbufs; + limit = (control->usable_ram - (control->overhead * control->threads)) / testbufs; limit = MIN(limit, chunk_limit); } + if (BITS32) // Shouldn't be higher than this by now, but just in case + limit = MIN(limit, one_g); + /* Use a nominal minimum size should we fail all previous shrinking */ limit = MAX(limit, STREAM_BUFSIZE); retest_malloc: - testsize = (limit * testbufs) + (control->overhead * control->threads); + testsize = limit + (control->overhead * control->threads); testmalloc = malloc(testsize); if (!testmalloc) { limit = limit / 10 * 9; goto retest_malloc; } + if (!NO_COMPRESS) { + char *testmalloc2 = malloc(limit); + + if (!testmalloc2) { + free(testmalloc); + limit = limit / 10 * 9; + goto retest_malloc; + } + free(testmalloc2); + } free(testmalloc); print_maxverbose("Succeeded in testing %lld sized malloc for back end compression\n", testsize);