ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove the slightly fragile exponential growth buffer size.

Browse source 
It was only speeding up compression a small amount, yet adversely affected compression and would segfault due to the size not being consistent on successive passes.
This commit is contained in:
ckolivas 2011-02-25 10:10:22 +11:00
parent 3a818196cc
commit f9f880908c
3 changed files with 7 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ChangeLog
View file

@ -6,9 +6,8 @@ FEBRUARY 2011, version 0.561 Con Kolivas
* Delete temporary files generated when testing from stdin. * Delete temporary files generated when testing from stdin.
* Don't call perror on failures that aren't system related. * Don't call perror on failures that aren't system related.
* Improve visual output. * Improve visual output.
* Significant multi-threading speed-ups - Use one more thread than CPUs, and * Significant multi-threading speed-ups - as the spawning of threads is
use a smaller buffer on the first thread and slightly more each next thread partially serialised, use one more thread than CPUs.
until the next stream. Then repeat the cycle.
* Don't split up the chunks into multiple threads when no back end * Don't split up the chunks into multiple threads when no back end
compression is being used. compression is being used.
* Take into account the precise amount of overhead that the memory hungry * Take into account the precise amount of overhead that the memory hungry

1
rzip.h
View file

@ -301,7 +301,6 @@ struct stream_info {
int num_streams; int num_streams;
int fd; int fd;
i64 bufsize; i64 bufsize;
i64 max_bufsize;
i64 cur_pos; i64 cur_pos;
i64 initial_pos; i64 initial_pos;
i64 total_read; i64 total_read;

21
stream.c
View file

@ -814,24 +814,14 @@ retest_malloc:
free(testmalloc); free(testmalloc);
print_maxverbose("Succeeded in testing %lld sized malloc for back end compression\n", testsize); print_maxverbose("Succeeded in testing %lld sized malloc for back end compression\n", testsize);
sinfo->max_bufsize = MIN(limit, /* Make the bufsize no smaller than STREAM_BUFSIZE. Round up the
MAX((limit + control.threads - 1) / control.threads, * bufsize to fit X threads into it */
sinfo->bufsize = MIN(limit, MAX((limit + control.threads - 1) / control.threads,
STREAM_BUFSIZE)); STREAM_BUFSIZE));
sinfo->bufsize = sinfo->max_bufsize;
/* We start with slightly smaller buffers to start loading CPUs as soon
* as possible and make them exponentially larger approaching the
* tested maximum size. We ensure the buffers are of a minimum size,
* though, as compression efficency drops off dramatically with tiny
* buffers. */
if (control.threads > 1 && sinfo->max_bufsize > STREAM_BUFSIZE) {
sinfo->bufsize = sinfo->max_bufsize * 63 / 100;
sinfo->bufsize = MAX(sinfo->bufsize, STREAM_BUFSIZE);
}
if (control.threads > 1) if (control.threads > 1)
print_maxverbose("Using up to %d threads to compress up to %lld bytes each.\n", print_maxverbose("Using up to %d threads to compress up to %lld bytes each.\n",
control.threads, sinfo->max_bufsize); control.threads, sinfo->bufsize);
else else
print_maxverbose("Using only 1 thread to compress up to %lld bytes\n", print_maxverbose("Using only 1 thread to compress up to %lld bytes\n",
sinfo->bufsize); sinfo->bufsize);
@ -1072,9 +1062,6 @@ static void clear_buffer(struct stream_info *sinfo, int stream, int newbuf)
i, cthread[i].s_len, stream); i, cthread[i].s_len, stream);
create_pthread(&threads[i], NULL, compthread, (void *)i); create_pthread(&threads[i], NULL, compthread, (void *)i);
if (control.threads > 1)
sinfo->bufsize += (sinfo->max_bufsize - sinfo->bufsize) * 63 / 100;
if (newbuf) { if (newbuf) {
/* The stream buffer has been given to the thread, allocate a new one */ /* The stream buffer has been given to the thread, allocate a new one */
sinfo->s[stream].buf = malloc(sinfo->bufsize); sinfo->s[stream].buf = malloc(sinfo->bufsize);