ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Unify maxram allocation and limit threads when there isn't enough ram.

Browse source
This commit is contained in:
Con Kolivas 2011-03-07 13:23:14 +11:00
parent 23f82e924d
commit 38eca38743
4 changed files with 19 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
main.c
View file

@ -997,6 +997,7 @@ int main(int argc, char *argv[])
/* Decrease usable ram size on 32 bits due to kernel/userspace split */ /* Decrease usable ram size on 32 bits due to kernel/userspace split */
if (BITS32) if (BITS32)
control.ramsize = MAX(control.ramsize - 900000000ll, 900000000ll); control.ramsize = MAX(control.ramsize - 900000000ll, 900000000ll);
control.maxram = control.ramsize / 3;
/* Set the main nice value to half that of the backend threads since /* Set the main nice value to half that of the backend threads since
* the rzip stage is usually the rate limiting step */ * the rzip stage is usually the rate limiting step */

2
rzip.c
View file

@ -780,7 +780,7 @@ void rzip_fd(int fd_in, int fd_out)
* allocate 1/3 of it to the main buffer and use a sliding mmap * allocate 1/3 of it to the main buffer and use a sliding mmap
* buffer to work on 2/3 ram size, leaving enough ram for the * buffer to work on 2/3 ram size, leaving enough ram for the
* compression backends */ * compression backends */
control.max_mmap = control.ramsize / 3; control.max_mmap = control.maxram;
/* On 32 bits we can have a big window with sliding mmap, but can /* On 32 bits we can have a big window with sliding mmap, but can
* not enable much per mmap/malloc */ * not enable much per mmap/malloc */

1
rzip.h
View file

@ -270,6 +270,7 @@ struct rzip_control {
const char *suffix; const char *suffix;
int compression_level; int compression_level;
i64 overhead; // compressor overhead i64 overhead; // compressor overhead
i64 maxram; // the largest chunk of ram to allocate
unsigned char lzma_properties[5]; // lzma properties, encoded unsigned char lzma_properties[5]; // lzma properties, encoded
i64 window; i64 window;
unsigned long flags; unsigned long flags;

21
stream.c
View file

@ -763,18 +763,18 @@ void close_streamout_threads(void)
/* open a set of output streams, compressing with the given /* open a set of output streams, compressing with the given
compression level and algorithm */ compression level and algorithm */
void *open_stream_out(int f, int n, i64 limit, char cbytes) void *open_stream_out(int f, int n, i64 chunk_limit, char cbytes)
{ {
struct stream_info *sinfo; struct stream_info *sinfo;
i64 testsize, limit;
uchar *testmalloc; uchar *testmalloc;
i64 testsize;
int i, testbufs; int i, testbufs;
sinfo = calloc(sizeof(struct stream_info), 1); sinfo = calloc(sizeof(struct stream_info), 1);
if (unlikely(!sinfo)) if (unlikely(!sinfo))
return NULL; return NULL;
sinfo->bufsize = limit; sinfo->bufsize = limit = chunk_limit;
sinfo->chunk_bytes = cbytes; sinfo->chunk_bytes = cbytes;
sinfo->num_streams = n; sinfo->num_streams = n;
@ -802,8 +802,19 @@ void *open_stream_out(int f, int n, i64 limit, char cbytes)
(control.overhead * control.threads)); (control.overhead * control.threads));
testsize = (limit * testbufs) + (control.overhead * control.threads); testsize = (limit * testbufs) + (control.overhead * control.threads);
if (testsize > control.ramsize / 3) if (testsize > control.maxram)
limit = (control.ramsize / 3 - (control.overhead * control.threads)) / testbufs; limit = (control.maxram - (control.overhead * control.threads)) / testbufs;
/* If we don't have enough ram for the number of threads, decrease the
* number of threads till we do, or only have one thread. */
while (limit < STREAM_BUFSIZE && limit < chunk_limit) {
if (control.threads > 1)
--control.threads;
else
break;
limit = (control.maxram - (control.overhead * control.threads)) / testbufs;
limit = MIN(limit, chunk_limit);
}
retest_malloc: retest_malloc:
testsize = (limit * testbufs) + (control.overhead * control.threads); testsize = (limit * testbufs) + (control.overhead * control.threads);
testmalloc = malloc(testsize); testmalloc = malloc(testsize);