diff --git a/BUGS b/BUGS index cb30eaa..744fac5 100644 --- a/BUGS +++ b/BUGS @@ -1,5 +1,4 @@ BUGME March 2011 Mac may not be able to work with STDIN/STDOUT on very large files. -MD5 displayed on large files on Mac may not match file produced, but file -generated should match original file. +MD5 is disabled on Mac due to not working properly. diff --git a/ChangeLog b/ChangeLog index bfe804b..fb2bced 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,21 @@ lrzip ChangeLog +APRIL 2011, version 0.604 Con Kolivas +* Detach threads after creating them on the compression side. Not joining them +meant that compressing massive files requiring hundreds of threads would +eventually hit the resource limit of number of threads created even though +the threads themselves would exit. + +APRIL 2011, version 0.603 Con Kolivas, George Makrydakis, Jari Aalto. +* lseek in stream.c wasn't being compiled to the lseek64 variant on Apple +due to missing includes, breaking >2GB files. Added includes. +* Detect when stdout is being redirected and automatically direct output to +stdout unless a filename is specified. +* Update lrztar to properly support -S -O and -o, and use new syntax not +requiring '-o -' for stdout. +* Update lrzip.conf to support encryption. +* Do a sanity check to ensure lrzip is not attempting to work on a directory. +* Typo fixes. + APRIL 2011, version 0.602 Con Kolivas * Fixed the symlinks breaking package generation. * Made maximum chunk allocable on 32bits 2/3 of a GB again limiting total ram diff --git a/TODO b/TODO index 48ebcb1..6653cbb 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,12 @@ MAYBE TODO for lrzip program +Upgrade to newer version of zpaq supporting 3 compression levels without +relying on open_memstream so it works without temporary files on apple. + +Get MD5 working on apple. + +Make sure STDIO works properly on large files on apple. + Make a liblrzip library. Other posix/windows builds?? Need help there... diff --git a/WHATS-NEW b/WHATS-NEW index a3eac1a..660f51b 100644 --- a/WHATS-NEW +++ b/WHATS-NEW @@ -1,3 +1,22 @@ +lrzip-0.604 + +lrzip will no longer fail with a "resource temporarily unavailable" error +when compressing files over 100GB that require hundreds of threads to +complete. + +lrzip-0.603 + +lrzip now supports stdout without requiring the '-o -' option. It detects when +output is being redirected without a filename and will automatically output to +stdout so you can do: + lrunzip patch-2.6.38.4.lrz | patch -p1 +Apple builds will not have errors on compressing files >2GB in size which +broke with 0.600. +lrztar will properly support -o, -O and -S. +lrzip.conf file now supports encryption. +lrzip will now warn if it's inappropriately passed a directory as an argument +directly. + lrzip-0.602 Fixed wrong symlinks which broke some package generation. diff --git a/configure.ac b/configure.ac index a2d3f20..9a6e279 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_maj], [0]) m4_define([v_min], [6]) -m4_define([v_mic], [02]) +m4_define([v_mic], [04]) ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_v], m4_join([], v_min, v_mic)) m4_define([v_ver], [v_maj.v_v]) diff --git a/doc/lrzip.conf.example b/doc/lrzip.conf.example index e083de1..9d0e814 100644 --- a/doc/lrzip.conf.example +++ b/doc/lrzip.conf.example @@ -57,3 +57,7 @@ # Override for Temporary Directory. Only valid when stdin/out or Test is used # TMPDIR = /tmp + +# ENCRYPT = NO + +# Whether to use encryption on compression \ No newline at end of file diff --git a/main.c b/main.c index 18a934e..e6d6663 100644 --- a/main.c +++ b/main.c @@ -35,6 +35,12 @@ #ifdef HAVE_SYS_RESOURCE_H # include #endif +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif #include #include #ifdef HAVE_ENDIAN_H @@ -427,6 +433,9 @@ static void read_config(rzip_control *control) strcpy(control->tmpdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->tmpdir, "/"); + } else if (isparameter(parameter, "encrypt")) { + if (isparameter(parameter, "YES")) + control->flags |= FLAG_ENCRYPT; } else /* oops, we have an invalid parameter, display */ print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ @@ -715,8 +724,18 @@ int main(int argc, char *argv[]) control.infile = argv[i]; else if (!(i == 0 && STDIN)) break; - if (control.infile && (strcmp(control.infile, "-") == 0)) - control.flags |= FLAG_STDIN; + if (control.infile) { + if ((strcmp(control.infile, "-") == 0)) + control.flags |= FLAG_STDIN; + else { + struct stat infile_stat; + + stat(control.infile, &infile_stat); + if (unlikely(S_ISDIR(infile_stat.st_mode))) + failure("lrzip only works directly on FILES.\n" + "Use lrztar or pipe through tar for compressing directories.\n"); + } + } if (INFO && STDIN) failure("Will not get file info from STDIN\n"); diff --git a/stream.c b/stream.c index 6f840e7..c11da00 100644 --- a/stream.c +++ b/stream.c @@ -135,13 +135,19 @@ static void cond_broadcast(pthread_cond_t *cond) fatal("pthread_cond_broadcast failed"); } -void create_pthread(pthread_t * thread, pthread_attr_t * attr, +void create_pthread(pthread_t *thread, pthread_attr_t * attr, void * (*start_routine)(void *), void *arg) { - if (pthread_create(thread, attr, start_routine, arg)) + if (unlikely(pthread_create(thread, attr, start_routine, arg))) fatal("pthread_create"); } +void detach_pthread(pthread_t *thread) +{ + if (unlikely(pthread_detach(*thread))) + fatal("pthread_detach"); +} + void join_pthread(pthread_t th, void **thread_return) { if (pthread_join(th, thread_return)) @@ -1435,6 +1441,7 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s s->i = i; s->control = control; create_pthread(&threads[i], NULL, compthread, s); + detach_pthread(&threads[i]); if (newbuf) { /* The stream buffer has been given to the thread, allocate a