From 7a74991f3f21f25661ab83e3360419b5f8a89c06 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 18:40:39 +1000 Subject: [PATCH 01/10] Update bugs file. --- BUGS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/BUGS b/BUGS index cb30eaa..744fac5 100644 --- a/BUGS +++ b/BUGS @@ -1,5 +1,4 @@ BUGME March 2011 Mac may not be able to work with STDIN/STDOUT on very large files. -MD5 displayed on large files on Mac may not match file produced, but file -generated should match original file. +MD5 is disabled on Mac due to not working properly. From 30e990fbcf0fb00cc6eeba01a66bf2d45febaf2b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 18:49:24 +1000 Subject: [PATCH 02/10] Add encrypt support to lrzip.conf --- doc/lrzip.conf.example | 4 ++++ main.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/doc/lrzip.conf.example b/doc/lrzip.conf.example index e083de1..9d0e814 100644 --- a/doc/lrzip.conf.example +++ b/doc/lrzip.conf.example @@ -57,3 +57,7 @@ # Override for Temporary Directory. Only valid when stdin/out or Test is used # TMPDIR = /tmp + +# ENCRYPT = NO + +# Whether to use encryption on compression \ No newline at end of file diff --git a/main.c b/main.c index 18a934e..33662d8 100644 --- a/main.c +++ b/main.c @@ -427,6 +427,9 @@ static void read_config(rzip_control *control) strcpy(control->tmpdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->tmpdir, "/"); + } else if (isparameter(parameter, "encrypt")) { + if (isparameter(parameter, "YES")) + control->flags |= FLAG_ENCRYPT; } else /* oops, we have an invalid parameter, display */ print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ From c1c44352e0069b7a1372d2e6fb17cbacc053229e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 19:10:06 +1000 Subject: [PATCH 03/10] Updated documentation. --- ChangeLog | 10 ++++++++++ WHATS-NEW | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/ChangeLog b/ChangeLog index bfe804b..e2ccefc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,14 @@ lrzip ChangeLog +APRIL 2011, version 0.603 Con Kolivas, George Makrydakis, Jari Aalto. +* lseek in stream.c wasn't being compiled to the lseek64 variant on Apple +due to missing includes, breaking >2GB files. Added includes. +* Detect when stdout is being redirected and automatically direct output to +stdout unless a filename is specified. +* Update lrztar to properly support -S -O and -o, and use new syntax not +requiring '-o -' for stdout. +* Update lrzip.conf to support encryption. +* Typo fixes. + APRIL 2011, version 0.602 Con Kolivas * Fixed the symlinks breaking package generation. * Made maximum chunk allocable on 32bits 2/3 of a GB again limiting total ram diff --git a/WHATS-NEW b/WHATS-NEW index a3eac1a..ed36d32 100644 --- a/WHATS-NEW +++ b/WHATS-NEW @@ -1,3 +1,14 @@ +lrzip-0.603 + +lrzip now supports stdout without requiring the '-o -' option. It detects when +output is being redirected without a filename and will automatically output to +stdout so you can do: + lrunzip patch-2.6.38.4.lrz | patch -p1 +Apple builds will not have errors on compressing files >2GB in size which +broke with 0.600. +lrztar will properly support -o, -O and -S. +lrzip.conf file now supports encryption. + lrzip-0.602 Fixed wrong symlinks which broke some package generation. From e8824afa1ba9cec6b91428d991c6b0d9bc0c3877 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 19:23:40 +1000 Subject: [PATCH 04/10] Do a sanity check on infile in lrzip to ensure it's working on a file it can compress. --- main.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index 33662d8..c2a6a7b 100644 --- a/main.c +++ b/main.c @@ -35,6 +35,12 @@ #ifdef HAVE_SYS_RESOURCE_H # include #endif +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif #include #include #ifdef HAVE_ENDIAN_H @@ -718,8 +724,19 @@ int main(int argc, char *argv[]) control.infile = argv[i]; else if (!(i == 0 && STDIN)) break; - if (control.infile && (strcmp(control.infile, "-") == 0)) - control.flags |= FLAG_STDIN; + if (control.infile) { + if ((strcmp(control.infile, "-") == 0)) + control.flags |= FLAG_STDIN; + else { + struct stat infile_stat; + + stat(control.infile, &infile_stat); + if (unlikely(!S_ISREG(infile_stat.st_mode) && + !S_ISLNK(infile_stat.st_mode))) + failure("lrzip only works directly on FILES.\n" + "Use lrztar or pipe through tar for compressing directories.\n"); + } + } if (INFO && STDIN) failure("Will not get file info from STDIN\n"); From efa8f0f6a5fa2f0840076a8ba9c0e3445a9f19b4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 19:34:11 +1000 Subject: [PATCH 05/10] Add to changelog --- ChangeLog | 1 + WHATS-NEW | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index e2ccefc..addeb47 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,6 +7,7 @@ stdout unless a filename is specified. * Update lrztar to properly support -S -O and -o, and use new syntax not requiring '-o -' for stdout. * Update lrzip.conf to support encryption. +* Do a sanity check to ensure lrzip is not attempting to work on a directory. * Typo fixes. APRIL 2011, version 0.602 Con Kolivas diff --git a/WHATS-NEW b/WHATS-NEW index ed36d32..7fe5ee6 100644 --- a/WHATS-NEW +++ b/WHATS-NEW @@ -8,6 +8,8 @@ Apple builds will not have errors on compressing files >2GB in size which broke with 0.600. lrztar will properly support -o, -O and -S. lrzip.conf file now supports encryption. +lrzip will now warn if it's inappropriately passed a directory as an argument +directly. lrzip-0.602 From 5352debf15c35c1e85bc7e7d1ca37ccc30c48f17 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 19:35:04 +1000 Subject: [PATCH 06/10] Change to detecting only whether lrzip is inappropriately being passed a directory since there may be other valid file types. --- main.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main.c b/main.c index c2a6a7b..e6d6663 100644 --- a/main.c +++ b/main.c @@ -731,10 +731,9 @@ int main(int argc, char *argv[]) struct stat infile_stat; stat(control.infile, &infile_stat); - if (unlikely(!S_ISREG(infile_stat.st_mode) && - !S_ISLNK(infile_stat.st_mode))) - failure("lrzip only works directly on FILES.\n" - "Use lrztar or pipe through tar for compressing directories.\n"); + if (unlikely(S_ISDIR(infile_stat.st_mode))) + failure("lrzip only works directly on FILES.\n" + "Use lrztar or pipe through tar for compressing directories.\n"); } } From dcc3606507c5bf486b8e6ebde6876c70adbb03fd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 22 Apr 2011 19:38:47 +1000 Subject: [PATCH 07/10] Update version to 0.603 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a2d3f20..e3d1fa6 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_maj], [0]) m4_define([v_min], [6]) -m4_define([v_mic], [02]) +m4_define([v_mic], [03]) ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_v], m4_join([], v_min, v_mic)) m4_define([v_ver], [v_maj.v_v]) From 7ed977b1c18bc9956946cf2b4253aa2850411b85 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 23 Apr 2011 08:15:44 +1000 Subject: [PATCH 08/10] Detach threads from the compression side since we don't explicitly join them as they may count towards max thread count otherwise. --- stream.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/stream.c b/stream.c index 6f840e7..c11da00 100644 --- a/stream.c +++ b/stream.c @@ -135,13 +135,19 @@ static void cond_broadcast(pthread_cond_t *cond) fatal("pthread_cond_broadcast failed"); } -void create_pthread(pthread_t * thread, pthread_attr_t * attr, +void create_pthread(pthread_t *thread, pthread_attr_t * attr, void * (*start_routine)(void *), void *arg) { - if (pthread_create(thread, attr, start_routine, arg)) + if (unlikely(pthread_create(thread, attr, start_routine, arg))) fatal("pthread_create"); } +void detach_pthread(pthread_t *thread) +{ + if (unlikely(pthread_detach(*thread))) + fatal("pthread_detach"); +} + void join_pthread(pthread_t th, void **thread_return) { if (pthread_join(th, thread_return)) @@ -1435,6 +1441,7 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s s->i = i; s->control = control; create_pthread(&threads[i], NULL, compthread, s); + detach_pthread(&threads[i]); if (newbuf) { /* The stream buffer has been given to the thread, allocate a From 3f0a1124eb7a7f09d273851777d7ccbe4e92f6db Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 26 Apr 2011 10:09:27 +1000 Subject: [PATCH 09/10] Update docs. --- ChangeLog | 6 ++++++ TODO | 7 +++++++ WHATS-NEW | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/ChangeLog b/ChangeLog index addeb47..fb2bced 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ lrzip ChangeLog +APRIL 2011, version 0.604 Con Kolivas +* Detach threads after creating them on the compression side. Not joining them +meant that compressing massive files requiring hundreds of threads would +eventually hit the resource limit of number of threads created even though +the threads themselves would exit. + APRIL 2011, version 0.603 Con Kolivas, George Makrydakis, Jari Aalto. * lseek in stream.c wasn't being compiled to the lseek64 variant on Apple due to missing includes, breaking >2GB files. Added includes. diff --git a/TODO b/TODO index 48ebcb1..6653cbb 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,12 @@ MAYBE TODO for lrzip program +Upgrade to newer version of zpaq supporting 3 compression levels without +relying on open_memstream so it works without temporary files on apple. + +Get MD5 working on apple. + +Make sure STDIO works properly on large files on apple. + Make a liblrzip library. Other posix/windows builds?? Need help there... diff --git a/WHATS-NEW b/WHATS-NEW index 7fe5ee6..660f51b 100644 --- a/WHATS-NEW +++ b/WHATS-NEW @@ -1,3 +1,9 @@ +lrzip-0.604 + +lrzip will no longer fail with a "resource temporarily unavailable" error +when compressing files over 100GB that require hundreds of threads to +complete. + lrzip-0.603 lrzip now supports stdout without requiring the '-o -' option. It detects when From 74d532e9d6748b2a798f88667853a238920267b2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 26 Apr 2011 10:10:26 +1000 Subject: [PATCH 10/10] Update version to 0.604 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e3d1fa6..9a6e279 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_maj], [0]) m4_define([v_min], [6]) -m4_define([v_mic], [03]) +m4_define([v_mic], [04]) ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--## m4_define([v_v], m4_join([], v_min, v_mic)) m4_define([v_ver], [v_maj.v_v])