Merge pull request #258 from eli-schwartz/dont-use-echo

build: fix incorrect use of shell commands in handmade libtool file

.github/workflows/c-cpp.yml

@@ -0,0 +1,29 @@
+name: check_build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: prepare repo
+      run: git fetch --prune --unshallow
+    - name: autogen
+      run: ./autogen.sh
+    - name: install liblzo2-dev
+      run: sudo apt install -y liblzo2-dev
+    - name: install liblz4-dev
+      run: sudo apt install -y liblz4-dev
+    - name: configure
+      run: ./configure
+    - name: make
+      run: make
+    - name: make check
+      run: make check

.gitignore

@@ -16,6 +16,7 @@ libtool
 lrzip
 lrzip*.tar.bz2
 lrzip*.tar.gz
+lrzip*.tar.lrz
 ltmain.sh
 missing
 stamp-h1
@@ -24,7 +25,15 @@ ltoptions.m4
 ltsugar.m4
 ltversion.m4
 lt~obsolete.m4
+compile
 man/lrunzip.1
 man/lrzcat.1
 man/lrztar.1
 man/lrzuntar.1
+man/lrz.1
+libzpaq/.dirstamp
+lrzip.pc
+regressiontest.out
+decompress_demo
+liblrzip_demo
+

Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine as builder
+
+RUN apk add --update git autoconf automake libtool gcc musl-dev zlib-dev bzip2-dev lzo-dev coreutils make g++ lz4-dev && \
+    git clone https://github.com/ckolivas/lrzip.git && \
+    cd /lrzip && ./autogen.sh && ./configure && make -j `nproc` && make install
+
+FROM alpine
+
+RUN apk add --update --no-cache lzo libbz2 libstdc++ lz4-dev && \
+    rm -rf /tmp/* /var/tmp/*
+
+COPY --from=builder /usr/local/bin/lrzip /usr/local/bin/lrzip
+
+CMD ["/bin/sh"]

Lrzip.h

@@ -1,766 +0,0 @@
-/*
-   Copyright (C) 2006-2011 Con Kolivas
-   Copyright (C) 2011 Peter Hyman
-   Copyright (C) 1998-2003 Andrew Tridgell
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef LIBLRZIP_H
-#define LIBLRZIP_H
-
-#include <stdbool.h>
-#include <stdio.h>
-#ifdef _WIN32
-# include <stddef.h>
-#else
-# include <inttypes.h>
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- @brief LRZIP library
-
- @mainpage lrzip
- @version 1.0
- @date 2011
-
- @section intro What is LRZIP?
- 
- LRZIP is a compression program optimised for large files. The larger the file
- and the more memory you have, the better the compression advantage this will
- provide, especially once the files are larger than 100MB. The advantage can
- be chosen to be either size (much smaller than bzip2) or speed (much faster
- than bzip2).
- * @link Lrzip.h LRZIP API @endlink
- */
-
-/** @file Lrzip.h */
-
-/**
- * @typedef Lrzip
- * @brief The overall struct for managing all operations
- */
-typedef struct Lrzip Lrzip;
-
-/**
- * @typedef Lrzip_Log_Level
- * @brief The amount of information to display using logging functions
- * This enum is used when setting or getting the log level of an #Lrzip
- * struct. It determines how much information is shown about the current operation,
- * either in stdout/stderr or using logging callbacks.
- * @see lrzip_log_level_set()
- * @see lrzip_log_level_get()
- */
-typedef enum {
-	LRZIP_LOG_LEVEL_ERROR = 0, /**< Only display errors */
-	LRZIP_LOG_LEVEL_INFO, /**< Display information and errors */
-	LRZIP_LOG_LEVEL_PROGRESS, /**< Display progress updates, information, and errors */
-	LRZIP_LOG_LEVEL_VERBOSE, /**< Display verbose progress updates, information, and errors */
-	LRZIP_LOG_LEVEL_DEBUG /**< Display all possible information */
-} Lrzip_Log_Level;
-
-/**
- * @typedef Lrzip_Mode
- * @brief The mode of operation for an #Lrzip struct
- * This enum is used when setting or getting the operation mode of an #Lrzip
- * struct. It determines what will happen when lrzip_run() is called.
- * @see lrzip_mode_set()
- * @see lrzip_mode_get()
- */
-typedef enum {
-	LRZIP_MODE_NONE = 0, /**< No operation set */
-	LRZIP_MODE_INFO, /**< Retrieve info about an archive */
-	LRZIP_MODE_TEST, /**< Test an archive's integrity */
-	LRZIP_MODE_DECOMPRESS, /**< Decompress an archive */
-	LRZIP_MODE_COMPRESS_NONE, /**< RZIP preprocess only */
-	LRZIP_MODE_COMPRESS_LZO, /**< Use LZO compression */
-	LRZIP_MODE_COMPRESS_ZLIB, /**< Use ZLIB (GZIP) compression */
-	LRZIP_MODE_COMPRESS_BZIP2, /**< Use BZIP2 compression */
-	LRZIP_MODE_COMPRESS_LZMA, /**< Use LZMA compression */
-	LRZIP_MODE_COMPRESS_ZPAQ /**< Use ZPAQ compression */
-} Lrzip_Mode;
-
-/**
- * @typedef Lrzip_Flag
- * @brief The extra params for an #Lrzip struct's operations
- * This enum is used when setting or getting the flags of an #Lrzip
- * struct. It determines some of the miscellaneous extra abilities of LRZIP.
- * @see lrzip_flags_set()
- * @see lrzip_flags_get()
- */
-typedef enum {
-	LRZIP_FLAG_REMOVE_SOURCE = (1 << 0), /**< Remove the input file after the operation completes */
-	LRZIP_FLAG_REMOVE_DESTINATION = (1 << 1), /**< Remove matching destination file if it exists */
-	LRZIP_FLAG_KEEP_BROKEN = (1 << 2), /**< Do not remove broken files */
-	LRZIP_FLAG_VERIFY = (1 << 3), /**< Only verify the archive, do not perform any compression/decompression */
-	LRZIP_FLAG_DISABLE_LZO_CHECK = (1 << 4), /**< Disable test to determine if LZO compression will be useful */
-	LRZIP_FLAG_UNLIMITED_RAM = (1 << 5), /**< Use unlimited ram window size for compression */
-	LRZIP_FLAG_ENCRYPT = (1 << 6) /**< Encrypt archive during compression; @see lrzip_pass_cb_set() */
-} Lrzip_Flag;
-
-/**
- * @typedef Lrzip_Info_Cb
- * @brief The callback to call when an operation's progress changes
- * @param data The data param passed in lrzip_info_cb_set()
- * @param pct The overall operation progress as a percent
- * @param chunk_pct The current chunk's operation progress as a percent
- */
-typedef void (*Lrzip_Info_Cb)(void *data, int pct, int chunk_pct);
-/**
- * @typedef Lrzip_Log_Cb
- * @brief The callback to call when a log message is to be shown
- * @param data The data param passed in lrzip_log_cb_set()
- * @param level The Lrzip_Log_Level of the message
- * @param line The line in LRZIP code where the message originated
- * @param file The file in LRZIP code where the message originated
- * @param func The function in LRZIP code where the message originated
- * @param format The printf-style format of the message
- * @param args The matching va_list for @p format
- */
-typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *func, const char *format, va_list args);
-/**
- * @typedef Lrzip_Password_Cb
- * @brief The callback to call for operations requiring a password
- * @param data The data param passed in lrzip_pass_cb_set()
- * @param buffer The pre-allocated buffer to write the password into
- * @param buf_size The size, in bytes, of @p buffer
- */
-typedef void (*Lrzip_Password_Cb)(void *data, char *buffer, size_t buf_size);
-
-/**
- * @brief Initialize liblrzip
- * This function must be called prior to running any other liblrzip
- * functions to initialize compression algorithms. It does not allocate.
- * @return true on success, false on failure
- */
-bool lrzip_init(void);
-
-/**
- * @brief Create a new #Lrzip struct
- * Use this function to allocate a new struct for immediate or later use,
- * optionally setting flags and changing modes at a later time.
- * @param mode The optional Lrzip_Mode to set, or LRZIP_MODE_NONE to allow
- * setting a mode later.
- * @return The new #Lrzip struct, or NULL on failure
- * @see lrzip_mode_set()
- */
-Lrzip *lrzip_new(Lrzip_Mode mode);
-
-/**
- * @brief Free an #Lrzip struct
- * Use this function to free all memory associated with an existing struct.
- * @param lr The struct to free
- */
-void lrzip_free(Lrzip *lr);
-
-/**
- * @brief Set up an #Lrzip struct using environment settings
- * Use this function to acquire and utilize settings already existing in
- * either environment variables or configuration files for LRZIP. For more detailed
- * information, see the LRZIP manual.
- * @param lr The struct to configure
- * @note This function cannot fail.
- */
-void lrzip_config_env(Lrzip *lr);
-
-/**
- * @brief Retrieve the operation mode of an #Lrzip struct
- * @param lr The struct to query
- * @return The Lrzip_Mode of @p lr, or LRZIP_MODE_NONE on failure
- */
-Lrzip_Mode lrzip_mode_get(Lrzip *lr);
-
-/**
- * @brief Set the operation mode of an #Lrzip struct
- * @param lr The struct to change the mode for
- * @param mode The Lrzip_Mode to set for @p lr
- * @return true on success, false on failure
- */
-bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode);
-
-/**
- * @brief Set the compression level of an #Lrzip struct
- * @param lr The struct to change the compression level for
- * @param level The value, 1-9, to use as the compression level for operations with @p lr
- * @return true on success, false on failure
- * @note This function is only valid for compression operations
- */
-bool lrzip_compression_level_set(Lrzip *lr, unsigned int level);
-
-/**
- * @brief Get the compression level of an #Lrzip struct
- * @param lr The struct to get the compression level of
- * @return The value, 1-9, used as the compression level for operations with @p lr,
- * or 0 on failure
- * @note This function is only valid for compression operations
- */
-unsigned int lrzip_compression_level_get(Lrzip *lr);
-
-/**
- * @brief Set the operation specific parameters
- * @param lr The struct to set parameters for
- * @param flags A bitwise ORed set of Lrzip_Flags
- * @note This function does not perform any error checking. Any errors in flags
- * will be determined when lrzip_run() is called.
- */
-void lrzip_flags_set(Lrzip *lr, unsigned int flags);
-
-/**
- * @brief Get the operation specific parameters
- * @param lr The struct to get parameters of
- * @return A bitwise ORed set of Lrzip_Flags
- */
-unsigned int lrzip_flags_get(Lrzip *lr);
-
-/**
- * @brief Set the nice level for operations in a struct
- * @param lr The struct to set the nice level for
- * @param nice The value to use when nicing during operations
- */
-void lrzip_nice_set(Lrzip *lr, int nice);
-
-/**
- * @brief Get the nice level for operations in a struct
- * @param lr The struct to get the nice level of
- * @return The value to use when nicing during operations
- */
-int lrzip_nice_get(Lrzip *lr);
-
-/**
- * @brief Explicitly set the number of threads to use during operations
- * @param lr The struct to set the threads for
- * @param threads The number of threads to use for operations
- * @note LRZIP will automatically determine the optimal number of threads to use,
- * so this function should only be used to specify FEWER than optimal threads.
- */
-void lrzip_threads_set(Lrzip *lr, unsigned int threads);
-
-/**
- * @brief Get the number of threads used during operations
- * @param lr The struct to query
- * @return The number of threads to use for operations
- */
-unsigned int lrzip_threads_get(Lrzip *lr);
-
-/**
- * @brief Set the maximum compression window for operations
- * @param lr The struct to set the maximum compression window for
- * @param size The size (in hundreds of MB) to use for the maximum size of compression
- * chunks.
- * @note LRZIP will automatically determine the optimal maximum compression window to use,
- * so this function should only be used to specify a LOWER value.
- */
-void lrzip_compression_window_max_set(Lrzip *lr, int64_t size);
-
-/**
- * @brief Get the maximum compression window for operations
- * @param lr The struct to query
- * @return The size (in hundreds of MB) to use for the maximum size of compression
- * chunks.
- */
-int64_t lrzip_compression_window_max_get(Lrzip *lr);
-
-/**
- * @brief Return the size of the stream queue in a struct
- * This function returns the current count of streams added for processing
- * using lrzip_file_add. It always returns instantly.
- * @param lr The struct to query
- * @return The current number of streams in the queue
- */
-unsigned int lrzip_files_count(Lrzip *lr);
-
-/**
- * @brief Return the size of the file queue in a struct
- * This function returns the current count of files added for processing
- * using lrzip_filename_add. It always returns instantly.
- * @param lr The struct to query
- * @return The current number of files in the queue
- */
-unsigned int lrzip_filenames_count(Lrzip *lr);
-
-/**
- * @brief Return the array of the stream queue in a struct
- * This function returns the current queue of streams added for processing
- * using lrzip_file_add. It always returns instantly.
- * @param lr The struct to query
- * @return The current stream queue
- */
-FILE **lrzip_files_get(Lrzip *lr);
-
-/**
- * @brief Return the array of the filename queue in a struct
- * This function returns the current queue of files added for processing
- * using lrzip_filename_add. It always returns instantly.
- * @param lr The struct to query
- * @return The current filename queue
- */
-char **lrzip_filenames_get(Lrzip *lr);
-
-/**
- * @brief Add a stream (FILE) to the operation queue
- * This function adds a stream to the input queue. Each time lrzip_run()
- * is called, it will run the current operation (specified by the Lrzip_Mode)
- * on either a stream or file in the queue.
- * @param lr The struct
- * @param file The stream descriptor to queue
- * @return true on success, false on failure
- * @note The file queue will be fully processed prior to beginning processing
- * the stream queue.
- * @warning Any streams added to this queue MUST NOT be closed until they have
- * either been processed or removed from the queue!
- */
-bool lrzip_file_add(Lrzip *lr, FILE *file);
-
-/**
- * @brief Remove a stream from the operation queue
- * This function removes a previously added stream from the operation queue by
- * iterating through the queue and removing the stream if found.
- * @param lr The struct
- * @param file The stream to remove
- * @return true only on successful removal, else false
- */
-bool lrzip_file_del(Lrzip *lr, FILE *file);
-
-/**
- * @brief Pop the current head of the stream queue
- * This function is used to remove the current head of the stream queue. It can be called
- * immediately following any lrzip_run() stream operation to remove the just-processed stream. This
- * function modifies the stream queue array, reordering and updating the index count.
- * @param lr The struct to pop the stream queue of
- * @return The stream removed from the queue, or NULL on failure
- */
-FILE *lrzip_file_pop(Lrzip *lr);
-
-/**
- * @brief Clear the stream queue
- * This function is used to free and reset the stream queue. The streams
- * themselves are untouched.
- * @param lr The struct
- */
-void lrzip_files_clear(Lrzip *lr);
-
-/**
- * @brief Add a file to the operation queue
- * This function adds a file to the input queue. Each time lrzip_run()
- * is called, it will run the current operation (specified by the Lrzip_Mode)
- * on either a stream or file in the queue.
- * @param lr The struct
- * @param file The file (by absolute path) to queue
- * @return true on success, false on failure
- * @note The file queue will be fully processed prior to beginning processing
- * the stream queue.
- */
-bool lrzip_filename_add(Lrzip *lr, const char *file);
-
-/**
- * @brief Remove a filename from the operation queue
- * This function removes a previously added filename from the operation queue by
- * iterating through the queue and removing the filename if found.
- * @param lr The struct
- * @param file The file to remove
- * @return true only on successful removal, else false
- */
-bool lrzip_filename_del(Lrzip *lr, const char *file);
-
-/**
- * @brief Pop the current head of the file queue
- * This function is used to remove the current head of the file queue. It can be called
- * immediately following any lrzip_run() file operation to remove the just-processed file. This
- * function modifies the file queue array, reordering and updating the index count.
- * @param lr The struct to pop the filename queue of
- * @return The filename removed from the queue, or NULL on failure
- */
-const char *lrzip_filename_pop(Lrzip *lr);
-
-/**
- * @brief Clear the file queue
- * This function is used to free and reset the file queue.
- * @param lr The struct
- */
-void lrzip_filenames_clear(Lrzip *lr);
-
-/**
- * @brief Set the default suffix for LRZIP compression operations
- * This function is used to change the default ".lrz" suffix for operations
- * to @p suffix.
- * @param lr The struct
- * @param suffix The suffix to use for compression operations
- */
-void lrzip_suffix_set(Lrzip *lr, const char *suffix);
-
-/**
- * @brief Get the default suffix for LRZIP compression operations
- * @param lr The struct
- * @return The suffix to use for compression operations, or NULL on failure
- */
-const char *lrzip_suffix_get(Lrzip *lr);
-
-/**
- * @brief Set the output directory for operations
- * This function can be used to set the output directory for operations.
- * Files will be stored according to their basename and lrzip suffix where
- * applicable.
- * @param lr The struct
- * @param dir The absolute path of the output directory
- */
-void lrzip_outdir_set(Lrzip *lr, const char *dir);
-
-/**
- * @brief Get the output directory for operations
- * @param lr The struct
- * @return The previously set output directory
- */
-const char *lrzip_outdir_get(Lrzip *lr);
-
-/**
- * @brief Set the output stream for operations
- * This function can be used to set the output stream for operations.
- * Raw data will be written to this stream for the duration of lrzip_run().
- * @param lr The struct
- * @param file The stream to write to
- * @warning @p file is NOT created by this library and must be opened by the user!
- */
-void lrzip_outfile_set(Lrzip *lr, FILE *file);
-
-/**
- * @brief Get the output stream for operations
- * @param lr The struct
- * @return The previously set output stream
- */
-FILE *lrzip_outfile_get(Lrzip *lr);
-
-/**
- * @brief Set the output file for operations
- * This function can be used to set the output file for operations.
- * Raw data will be written to the file with this name for the duration of lrzip_run().
- * @param lr The struct
- * @param file The name of the file to write to
- */
-void lrzip_outfilename_set(Lrzip *lr, const char *file);
-
-/**
- * @brief Get the output filename for operations
- * @param lr The struct
- * @return The previously set output filename
- */
-const char *lrzip_outfilename_get(Lrzip *lr);
-
-/**
- * @brief Retrieve the MD5 digest of an LRZIP file
- * Use this function after calling lrzip_run() to retrieve the digest of
- * the processed archive.
- * @param lr The struct having run an operation
- * @return The MD5 digest of the operation's associated archive
- * @note The return value of this function will change after each operation
- */
-const unsigned char *lrzip_md5digest_get(Lrzip *lr);
-
-/**
- * @brief Run the current operation
- * This function is called when all necessary parameters have been set for an operation.
- * The calling thread will then block until the operation has fully completed, writing
- * output using logging and progress callbacks and calling password callbacks as required.
- * @param lr The struct to run an operation with
- * @return true if the operation successfully completed, else false
- */
-bool lrzip_run(Lrzip *lr);
-
-/**
- * @brief Set the logging level
- * @param lr The struct
- * @param level The #Lrzip_Log_Level to use
- */
-void lrzip_log_level_set(Lrzip *lr, int level);
-
-/**
- * @brief Get the logging level
- * @param lr The struct to query
- * @return The #Lrzip_Log_Level of @p lr
- */
-int lrzip_log_level_get(Lrzip *lr);
-
-/**
- * @brief Set a logging callback for use with all operations
- * This function sets an Lrzip_Log_Cb which will be called any time logging
- * output is to be displayed. The callback will be called as many times as the #Lrzip_Log_Level
- * requires.
- * @param lr The struct
- * @param cb The callback
- * @param log_data The data param to use in the logging callback
- */
-void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data);
-
-/**
- * @brief Redirect stdout log messages to another stream
- * This function sends any logging messages which would normally go to stdout into another stream.
- * Useful for when stdout is the target set by lrzip_outfile_set().
- * @param lr The struct
- * @param out The stream to use instead of stdout
- */
-void lrzip_log_stdout_set(Lrzip *lr, FILE *out);
-
-/**
- * @brief Return the stream currently used as stdout
- * @param lr The struct to query
- * @return A stream where stdout messages will be sent, NULL on failure
- */
-FILE *lrzip_log_stdout_get(Lrzip *lr);
-
-/**
- * @brief Redirect stderr log messages to another stream
- * This function sends any logging messages which would normally go to stderr into another stream.
- * @param lr The struct
- * @param err The stream to use instead of stderr
- */
-void lrzip_log_stderr_set(Lrzip *lr, FILE *err);
-
-/**
- * @brief Return the stream currently used as stderr
- * @param lr The struct to query
- * @return A stream where stderr messages will be sent, NULL on failure
- */
-FILE *lrzip_log_stderr_get(Lrzip *lr);
-
-/**
- * @brief Set a password callback for use with all operations
- * This function sets an Lrzip_Password_Cb which will be used when working with encrypted
- * LRZIP archives. It will be called both when compressing and decompressing archives.
- * @param lr The struct
- * @param cb The callback to set
- * @param data The data param to use in the password callback
- */
-void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data);
-
-/**
- * @brief Set an info callback for use with all operations
- * This function sets an Lrzip_Info_Cb which will be called any time there is a
- * progress update in an operation.
- * @param lr The struct
- * @param cb The callback to set
- * @param data The data param to use in the info callback
- */
-void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data);
-
-/**
- * @brief Quick setup for performing a decompression
- * This function performs all the required allocations and sets necessary parameters
- * to decompress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len);
-
-/**
- * @brief Quick setup for performing a compression
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param mode The compression mode to use
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level);
-
-/**
- * @brief Quick setup for performing a compression using LZMA
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_compress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, 7); }
-
-/**
- * @brief Quick setup for performing a compression using LZO
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_lcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, 7); }
-
-/**
- * @brief Quick setup for performing a compression using ZLIB (GZIP)
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_gcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, 7); }
-
-/**
- * @brief Quick setup for performing a compression using ZPAQ
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_zcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, 7); }
-
-/**
- * @brief Quick setup for performing a compression using BZIP
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_bcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, 7); }
-
-/**
- * @brief Quick setup for performing RZIP preprocessing
- * This function performs all the required allocations and sets necessary parameters
- * to preprocess @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @return true on success, else false
- */
-static inline bool lrzip_rcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, 7); }
-
-/**
- * @brief Quick setup for performing a compression using LZMA and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_compress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, compress_level); }
-
-/**
- * @brief Quick setup for performing a compression using LZO and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_lcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, compress_level); }
-
-/**
- * @brief Quick setup for performing a compression using ZLIB (GZIP) and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_gcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, compress_level); }
-
-/**
- * @brief Quick setup for performing a compression using ZPAQ and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_zcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, compress_level); }
-
-/**
- * @brief Quick setup for performing a compression using BZIP and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to compress @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_bcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, compress_level); }
-
-/**
- * @brief Quick setup for performing RZIP preprocessing and a user-defined compression level
- * This function performs all the required allocations and sets necessary parameters
- * to preprocess @p source to @p dest. No extra functions are necessary to call, and
- * this function will block until it completes.
- * @param dest A pointer to the LRZIP-allocated destination buffer
- * @param dest_len A pointer to the length of @p dest
- * @param source The allocated source buffer to read from
- * @param source_len The length of @p source
- * @param compress_level The value, 1-9, to use as a compression level
- * @return true on success, else false
- */
-static inline bool lrzip_rcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
-{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, compress_level); }
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-

Makefile.am

@@ -14,6 +14,8 @@ ltmain.sh \
 missing \
 $(PACKAGE_TARNAME)-$(PACKAGE_VERSION).tar.gz \
 $(PACKAGE_TARNAME)-$(PACKAGE_VERSION).tar.bz2 \
+$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).tar.xz \
+$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).tar.lrz \
 $(PACKAGE_TARNAME)-$(PACKAGE_VERSION)-doc.tar.bz2 \
 m4/libtool.m4 \
 m4/lt~obsolete.m4 \
@@ -32,7 +34,6 @@ lrztar_SCRIPTS = lrztar
 noinst_LTLIBRARIES = libtmplrzip.la
 libtmplrzip_la_SOURCES = \
   lrzip_private.h \
-  liblrzip_private.h \
   lrzip.c \
   lrzip_core.h \
   rzip.h \
@@ -55,13 +56,6 @@ libtmplrzip_la_SOURCES = \
 libtmplrzip_la_LIBADD = lzma/C/liblzma.la
 
 
-lib_LTLIBRARIES = liblrzip.la
-liblrzip_la_SOURCES = \
-  liblrzip.c \
-  liblrzip_private.h
-nodist_EXTRA_liblrzip_la_SOURCES = dummy.cxx
-liblrzip_la_LIBADD = libtmplrzip.la
-
 bin_PROGRAMS = lrzip
 lrzip_SOURCES = \
   main.c
@@ -72,13 +66,6 @@ if STATIC
   lrzip_LDFLAGS = -all-static
 endif
 
-noinst_PROGRAMS = decompress_demo liblrzip_demo
-decompress_demo_SOURCES = decompress_demo.c
-decompress_demo_LDADD = liblrzip.la
-
-liblrzip_demo_SOURCES = liblrzip_demo.c
-liblrzip_demo_LDADD = liblrzip.la
-
 dist_doc_DATA = \
   AUTHORS \
   BUGS \
@@ -89,14 +76,9 @@ dist_doc_DATA = \
   TODO \
   WHATS-NEW
 
-pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = lrzip.pc
-
-lrzip_HEADERS = Lrzip.h
 lrzipdir = $(includedir)
 
 EXTRA_DIST = \
-		lrzip.pc.in \
 		lrztar \
 		description-pak \
 		autogen.sh \

README.md

@@ -15,7 +15,7 @@ Just change the word `directory` to the name of the directory you wish to compre
 #### Compression:
 
 ```bash
-lrzdir=directory; tar cvf $lrzdir; lrzip -Ubvvp `nproc` -S .bzip2-lrz -L 9 $lrzdir.tar; rm -fv $lrzdir.tar; unset lrzdir
+lrzdir=directory; tar cvf $lrzdir.tar $lrzdir; lrzip -Ubvvp `nproc` -S .bzip2-lrz -L 9 $lrzdir.tar; rm -fv $lrzdir.tar; unset lrzdir
 ```
 
 `tar`s the directory, then maxes out all of the system's processor cores
@@ -49,8 +49,9 @@ A quick guide on building and installing.
  - libz-dev
  - libbz2-dev
  - liblzo2-dev
+ - liblz4-dev
  - coreutils
- - nasm on x86, not needed on x64
+ - Optional nasm
  - git if you want a repo-fresh copy
  - an OS with the usual *nix headers and libraries
 
@@ -60,7 +61,7 @@ Two different ways of doing this:
 
 Stable: Packaged tarball that is known to work:
 
-Go to <https://github.com/ckolivas/lrzip/releases> and downlaod the `tar.gz`
+Go to <https://github.com/ckolivas/lrzip/releases> and download the `tar.gz`
 file from the top. `cd` to the directory you downloaded, and use `tar xvzf lrzip-X.X.tar.gz`
 to extract the files (don't forget to replace `X.X` with the correct version). Finally, cd
 into the directory you just extracted.
@@ -247,21 +248,20 @@ lzma compression can't currently be tracked when handing over 100+MB chunks
 over to the lzma library. Therefore you'll see progress percentage until
 each chunk is handed over to the lzma library.
 
-> Q: What's this "lzo testing for incompressible data" message?
+> Q: What's this "lz4 testing for incompressible data" message?
 
-> A: Other compression is much slower, and lzo is the fastest. To help speed up
-the process, lzo compression is performed on the data first to test that the
+> A: Other compression is much slower, and lz4 is the fastest. To help speed up
+the process, lz4 compression is performed on the data first to test that the
 data is at all compressible. If a small block of data is not compressible, it
 tests progressively larger blocks until it has tested all the data (if it fails
 to compress at all). If no compressible data is found, then the subsequent
 compression is not even attempted. This can save a lot of time during the
-compression phase when there is incompressible dat
-> A: Theoretically it may be
-possible that data is compressible by the other backend (zpaq, lzma etc) and not
-at all by lzo, but in practice such data achieves only minuscule amounts of
+compression phase when there is incompressible data. Theoretically it may be
+possible that data is compressible by the other backend (zpaq, lzma etc) and
+not at all by lz4, but in practice such data achieves only minuscule amounts of
 compression which are not worth pursuing. Most of the time it is clear one way
-or the other that data is compressible or not. If you wish to disable this
-test and force it to try compressing it anyway, use -T.
+or the other that data is compressible or not. If you wish to disable this test
+and force it to try compressing it anyway, use -T.
 
 > Q: I have truckloads of ram so I can compress files much better, but can my
 generated file be decompressed on machines with less ram?
@@ -279,18 +279,16 @@ other modes are more useful).
 > Q: What about multimedia?
 
 > A: Most multimedia is already in a heavily compressed "lossy" format which by
-its very nature has very little redundancy. This means that there is not
-much that can actually be compressed. If your video/audio/picture is in a
-high bitrate, there will be more redundancy than a low bitrate one making it
-more suitable to compression. None of the compression techniques in lrzip are
-optimised for this sort of data.
-> A: However, the nature of rzip preparation
-means that you'll still get better compression than most normal compression
+its very nature has very little redundancy. This means that there is not much
+that can actually be compressed. If your video/audio/picture is in a high
+bitrate, there will be more redundancy than a low bitrate one making it more
+suitable to compression. None of the compression techniques in lrzip are
+optimised for this sort of data. However, the nature of rzip preparation means
+that you'll still get better compression than most normal compression
 algorithms give you if you have very large files. ISO images of dvds for
 example are best compressed directly instead of individual .VOB files. ZPAQ is
 the only compression format that can do any significant compression of
 multimedia.
-> A:
 
 > Q: Is this multithreaded?
 
@@ -337,8 +335,7 @@ permanent storage I compress it with the default options. When compressing
 small files for distribution I use the -z option for the smallest possible
 size.
 
-> Q: I found a file that compressed better with plain lzm
-> A: How can that be?
+> Q: I found a file that compressed better with plain lzma. How can that be?
 
 > A: When the file is more than 5 times the size of the compression window
 you have available, the efficiency of rzip preparation drops off as a means
@@ -361,14 +358,14 @@ cpu process scheduler how to prioritise workloads, and if your application is
 the _only_ thing running it will be no faster at nice -20 nor will it be any
 slower at +19.
 
-> Q: What is the LZO Testing option, -T?
+> Q: What is the LZ4 Testing option, -T?
 
-> A: LZO testing is normally performed for the slower back-end compression of LZMA
-and ZPA> Q: The reasoning is that if it is completely incompressible by LZO then
-it will also be incompressible by them. Thus if a block fails to be compressed
-by the very fast LZO, lrzip will not attempt to compress that block with the
-slower compressor, thereby saving time. If this option is enabled, it will
-bypass the LZO testing and attempt to compress each block regardless.
+> A: LZ4 testing is normally performed for the slower back-end compression of
+LZMA and ZPAQ. The reasoning is that if it is completely incompressible by LZ4
+then it will also be incompressible by them. Thus if a block fails to be
+compressed by the very fast LZ4, lrzip will not attempt to compress that block
+with the slower compressor, thereby saving time. If this option is enabled, it
+will bypass the LZ4 testing and attempt to compress each block regardless.
 
 > Q: Compression and decompression progress on large archives slows down and
 speeds up. There's also a jump in the percentage at the end?
@@ -385,11 +382,10 @@ compression backend (lzma) needs to compress.
    what does this mean?
 
 > A: LZMA requests large amounts of memory. When a higher compression window is
-   used, there may not be enough contiguous memory for LZM
-> A: LZMA may request
-   up to 25% of TOTAL ram depending on compression level. If contiguous blocks
-   of memory are not free, LZMA will return an error. This is not a fatal
-   error, and a backup mode of compression will be used.
+used, there may not be enough contiguous memory for LZMA: LZMA may request up
+to 25% of TOTAL ram depending on compression level. If contiguous blocks of
+memory are not free, LZMA will return an error. This is not a fatal error, and
+a backup mode of compression will be used.
 
 > Q: Where can I get more information about the internals of LZMA?
 
@@ -472,7 +468,7 @@ Persons above are listed in chronological order of first contribution to **lrzip
 #### README Authors
 
 Con Kolivas (`ckolivas` on GitHub) <kernel@kolivas.org>
-Fri, 10 June 2016: README
+Tuesday, 16 February 2021: README
 
 Also documented by
 Peter Hyman <pete@peterhyman.com>

TODO

@@ -21,5 +21,3 @@ Consider ncurses version or even GUI one.
 
 Consider using LZMA Filters for processor-optimised
 coding to increase compression.
-
-Get the ASM working on 64bit - it's only the CRC check so probably no point.

WHATS-NEW

@@ -1,3 +1,45 @@
+lrzip-0.651
+
+Remove redundant files
+Revert locale dependent output
+Add warnings for low memory and threads
+
+lrzip-0.650
+
+Minor optimisations.
+Exit status fixes.
+Update and beautify information output.
+Fix Android build.
+Enable MD5 on Apple build.
+Deprecate and remove liblrzip which was unused and at risk of bitrot.
+Fix failures with compressing to STDOUT with inadequate memory.
+Fix possible race conditions.
+Fix memory leaks.
+Fix -q to only hide progress.
+Add -Q option for very quiet.
+
+
+lrzip-0.641
+
+Critical bugfix for broken lz4 testing which would prevent secondary
+compression from being enabled.
+
+lrzip-0.640
+
+Numerous bugfixes and build fixes.
+lz4 now used for compressibility testing (only) making lz4-dev a build
+requirement.
+Fixes for handling of corrupt archives without crashing.
+Fixes for creating small lzma based archives to stdout.
+Incomplete files are now deleted on interrupting lrzip unless the keep-broken
+option is enabled.
+Version prints to stdout instead of stderr.
+
+
+lrzip-0.631
+
+Assembler code is back and works with x86_64
+
 lrzip-0.621
 
 Substantial speed ups for the rzip stage in both regular and unlimited modes.

configure.ac

@@ -2,7 +2,7 @@
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_define([v_maj], [0])
 m4_define([v_min], [6])
-m4_define([v_mic], [30])
+m4_define([v_mic], [51])
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_define([v_v], m4_join([], v_min, v_mic))
 m4_define([v_ver], [v_maj.v_v])
@@ -15,7 +15,7 @@ m4_define([lt_age], v_min)
 
 dnl Process this file with autoconf to produce a configure script.
 AC_INIT([lrzip],[v_ver],[kernel@kolivas.org])
-AC_PREREQ([2.59])
+AC_PREREQ([2.71])
 AC_CONFIG_SRCDIR([configure.ac])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
@@ -24,7 +24,7 @@ AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_USE_SYSTEM_EXTENSIONS
 
-AC_PROG_LIBTOOL
+LT_INIT
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_ifdef([v_rev], , [m4_define([v_rev], [0])])
@@ -51,23 +51,22 @@ AC_PROG_INSTALL
 AC_PROG_LN_S
 AC_SUBST(SHELL)
 AC_SYS_LARGEFILE
+AC_FUNC_FSEEKO
 AC_FUNC_ALLOCA
-AC_PROG_CC_C99
-AS_IF([test "x$ac_cv_prog_cc_c99" = "xno"], 
-	AC_MSG_ERROR([C compiler does not support C99], 1))
+
 AC_CHECK_PROG([HAVE_POD2MAN], [pod2man], [yes])
 AS_IF([test "$HAVE_POD2MAN" != "yes"],
 	AC_MSG_FAILURE([pod2man is needed to generate manual from POD]))
 
-
 AC_ARG_ENABLE(
 		asm,
-		[AC_HELP_STRING([--enable-asm],[Enable native Assembly code])],
+		[AS_HELP_STRING([--enable-asm],[Enable native Assembly code])],
 		ASM=$enableval,
 		ASM=yes
              )
-if test x"$ASM" = xyes; then
-	AC_CHECK_PROG( ASM_PROG, nasm, yes, no )
+
+if test x"$ASM" = x"yes"; then
+	AC_CHECK_PROG( ASM_PROG, nasm, nasm, no ) # fix to set ASM_PROG to nasm, not yes.
 	if test x"$ASM_PROG" = x"no "; then
 		ASM=no
 	fi
@@ -75,10 +74,10 @@ fi
 
 static=no
 AC_ARG_ENABLE([static-bin],
-	[AC_HELP_STRING([--enable-static-bin],[Build statically linked binary @<:@default=no@:>@])],
+	[AS_HELP_STRING([--enable-static-bin],[Build statically linked binary @<:@default=no@:>@])],
 	[static=$enableval]
 	)
-AM_CONDITIONAL([STATIC], [test "x$static" = "xyes"])
+AM_CONDITIONAL([STATIC], [test x"$static" = x"yes"])
 
 AC_CHECK_HEADERS(fcntl.h sys/time.h unistd.h sys/mman.h)
 AC_CHECK_HEADERS(ctype.h errno.h sys/resource.h)
@@ -92,12 +91,6 @@ AC_CHECK_SIZEOF(int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(short)
 
-if test $ac_cv_sizeof_long != 4 -a "x$ASM" = "xyes" ; then
-	AC_MSG_WARN([64bit arch detected, disabling ASM])
-	ASM=no
-fi
-
-
 AC_CACHE_CHECK([for large file support],rzip_cv_HAVE_LARGE_FILES,[
 AC_RUN_IFELSE([AC_LANG_SOURCE([[
 #include <stdio.h>
@@ -121,36 +114,42 @@ AC_CHECK_LIB(bz2, BZ2_bzBuffToBuffCompress, ,
         AC_MSG_ERROR([Could not find bz2 library - please install libbz2-dev]))
 AC_CHECK_LIB(lzo2, lzo1x_1_compress, ,
         AC_MSG_ERROR([Could not find lzo2 library - please install liblzo2-dev]))
+AC_CHECK_LIB(lz4, LZ4_compress_default, ,
+	AC_MSG_ERROR([Could not find lz4 library - please install liblz4-dev]))
 
 AC_CHECK_FUNCS(mmap strerror)
 AC_CHECK_FUNCS(getopt_long)
 
-# final checks for x86 and/or assembler
-if test x"$ASM" = x"no"; then
-	ASM_OBJ=7zCrc.o
-	ASM=no
-else
+AX_PTHREAD
+LIBS="$PTHREAD_LIBS $LIBS"
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+CXXFLAGS="$CXXFLAGS $PTHREAD_CXXFLAGS"
+
+# final checks for assembler
+# ASM is back for x86_64 by using newer CRC code from p7zip-16.02
+# object files handled in lzma/C/Makefile.am
+if test x"$ASM" = x"yes"; then
+	ASM_OPT="-I../ASM/x86/"
 	case $host in
 		i?86-*)
-			ASM_OBJ="7zCrcT8.o 7zCrcT8U.o"
-			ASM_CMD="$ASM_PROG -f elf" ;;
-# x86_64 code is broken still
-#		x86_64-*)
-#			ASM_OBJ="7zCrcT8.o 7zCrcT8U_64.o"
-#			ASM_CMD="$ASM_PROG -f elf64" ;;
-		*) ASM_OBJ=7zCrc.o ;;
+			ASM_OPT="$ASM_OPT -g -f elf" ;;
+		x86_64-*)
+			ASM_OPT="$ASM_OPT -Dx64 -g -f elf64" ;;
+		*) ASM_OPT= ;;
 	esac
+else
+	ASM_OPT=
 fi
-AM_CONDITIONAL([USE_ASM], [test "x$ASM" != "xyes" -a "x$ASM" != "xno"])
-AC_SUBST([ASM_OBJ])
+AM_CONDITIONAL([USE_ASM], [test x"$ASM" = x"yes"])
+AC_SUBST([ASM_OPT])
 AC_SUBST([ASM_CMD])
 EFL_CHECK_DOXYGEN([build_doc="yes"], [build_doc="no"])
 
 AC_CONFIG_FILES([
 Makefile
-lrzip.pc
 lzma/Makefile
 lzma/C/Makefile
+lzma/ASM/x86/Makefile
 doc/Makefile
 man/Makefile
 ])
@@ -166,7 +165,7 @@ echo
 echo
 echo "Configuration Options Summary:"
 echo
-echo "  ASM.(32 bit only)..: $ASM"
+echo "  ASM................: $ASM"
 echo "  Static binary......: $static"
 echo
 echo "Documentation..........: ${build_doc}"

doc/README.Assembler

@@ -1,5 +1,20 @@
 README.Assembler
 
+Update November 2019
+
+Assembler is enabled by
+./configure --enable-asm
+and disabled by
+./configure --disable-asm
+not
+ASM=no ./configure 
+
+New files replace 32 and 64 bit assembler code.
+fixes to lzma/C/Makefile.am permit libtool linking.
+
+Original text follows.
+==========================
+
 Notes about CRC Assembly Language Coding.
 
 lrzip-0.21 makes use of an x86 assembly language file

doc/README.benchmarks

@@ -45,7 +45,7 @@ purpose compressor at the moment:
 
 These are benchmarks performed on a 2.53Ghz dual core Intel Core2 with 4GB ram
 using lrzip v0.5.1. Note that it was running with a 32 bit userspace so only
-2GB addressing was posible. However the benchmark was run with the -U option
+2GB addressing was possible. However the benchmark was run with the -U option
 allowing the whole file to be treated as one large compression window.
 
 Tarball of 6 consecutive kernel trees.

doc/lrzip.conf.example

@@ -1,55 +1,47 @@
 # lrzip.conf example file
 # anything beginning with a # or whitespace will be ignored
 # valid parameters are separated with an = and a value
-# parameters and values are not case sensitive
+# parameters and values are not case sensitive except where specified
 # 
 # lrzip 0.24+, peter hyman, pete@peterhyman.com
 # ignored by earlier versions.
 
-# Compression Window size in 100MB. Normally selected by program.
+# Compression Window size in 100MB. Normally selected by program. (-w)
 # WINDOW = 20
-
-# Compression Level 1-9 (7 Default).
+# Compression Level 1-9 (7 Default). (-L)
 # COMPRESSIONLEVEL = 7
-
 # Use -U setting, Unlimited ram. Yes or No
 # UNLIMITED = NO
-
-# Compression Method, rzip, gzip, bzip2, lzo, or lzma (default), or zpaq.
-# If specified here, command line options not usable.
+# Compression Method, rzip, gzip, bzip2, lzo, or lzma (default), or zpaq. (-n -g -b -l --lzma -z)
+# May be overridden by command line compression choice.
 # COMPRESSIONMETHOD = lzma
-
-# Perform LZO Test. Default = YES (-T option, NO)
+# Perform LZO Test. Default = YES (-T )
 # LZOTEST = NO
-
-# Hash Check on decompression, YES
+# Hash Check on decompression, (-c)
 # HASHCHECK = YES
-
-# Show HASH value on Compression even if Verbose is off, YES
+# Show HASH value on Compression even if Verbose is off, YES (-H)
 # SHOWHASH = YES
 
-# Default output directory
+# Default output directory (-O)
 # OUTPUTDIRECTORY = location
-
-# Verbosity, Yes or Max
+# Verbosity, YES or MAX (v, vv)
 # VERBOSITY = max
+# Show Progress as file is parsed, YES or no (NO = -q option)
+# SHOWPROGRESS = YES
 
-# Show Progress as file is parsed, Yes or no
-# SHOWPROGRESS = true
-
-# Set Niceness. 19 is default. -20 to 19 is the allowable range
+# Set Niceness. 19 is default. -20 to 19 is the allowable range (-N)
 # NICE = 19
 
-# Keep broken or damaged output files, YES
+# Keep broken or damaged output files, YES (-K)
 # KEEPBROKEN = YES
 
-# Delete source file after compression
+# Delete source file after compression (-D)
 # this parameter and value are case sensitive
 # value must be YES to activate
 
 # DELETEFILES = NO
 
-# Replace existing lrzip file when compressing
+# Replace existing lrzip file when compressing (-f)
 # this parameter and value are case sensitive
 # value must be YES to activate
 
@@ -58,6 +50,6 @@
 # Override for Temporary Directory. Only valid when stdin/out or Test is used
 # TMPDIR = /tmp
 
+# Whether to use encryption on compression YES, NO (-e)
 # ENCRYPT = NO
 
-# Whether to use encryption on compression

liblrzip.c

@@ -1,741 +0,0 @@
-/*
-   Copyright (C) 2012-2016 Con Kolivas
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <liblrzip_private.h>
-
-#ifdef HAVE_SYS_TIME_H
-# include <sys/time.h>
-#endif
-#ifdef HAVE_SYS_RESOURCE_H
-# include <sys/resource.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-# include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-# include <sys/stat.h>
-#endif
-
-/* needed for CRC routines */
-#include "lzma/C/7zCrc.h"
-#include "util.h"
-#include "lrzip_core.h"
-#include "rzip.h"
-
-#if defined(__APPLE__) || defined(__FreeBSD__)
-# define fmemopen(s, len, modes) fake_fmemopen((s), (len), (modes))
-static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode)
-{
-	FILE *in;
-
-	in = tmpfile();
-	if (!in)
-		return NULL;
-	if (fwrite(buf, buflen, 1, in) != 1) {
-		fclose(in);
-		return NULL;
-	}
-	rewind(in);
-	return in;
-}
-#endif
-
-static void liblrzip_index_update(size_t x, size_t *idx, void **queue)
-{
-	for (; x < *idx; x++)
-		queue[x] = queue[x + 1];
-	(*idx)--;
-}
-
-static bool liblrzip_setup_flags(Lrzip *lr)
-{
-	if (!lr)
-		return false;
-#define MODE_CHECK(X) \
-	case LRZIP_MODE_COMPRESS_##X: \
-	lr->control->flags ^= FLAG_NOT_LZMA; \
-	lr->control->flags |= FLAG_##X##_COMPRESS; \
-	break
-
-
-	switch (lr->mode) {
-	case LRZIP_MODE_DECOMPRESS:
-		lr->control->flags |= FLAG_DECOMPRESS;
-		break;
-	case LRZIP_MODE_TEST:
-		lr->control->flags |= FLAG_TEST_ONLY;
-		break;
-	case LRZIP_MODE_INFO:
-		lr->control->flags |= FLAG_INFO;
-		break;
-	case LRZIP_MODE_COMPRESS_NONE:
-		lr->control->flags ^= FLAG_NOT_LZMA;
-		lr->control->flags |= FLAG_NO_COMPRESS;
-		break;
-	case LRZIP_MODE_COMPRESS_LZMA:
-		lr->control->flags ^= FLAG_NOT_LZMA;
-		break;
-	MODE_CHECK(LZO);
-	MODE_CHECK(BZIP2);
-	MODE_CHECK(ZLIB);
-	MODE_CHECK(ZPAQ);
-#undef MODE_CHECK
-	default:
-		return false;
-	}
-	setup_overhead(lr->control);
-	if (lr->flags & LRZIP_FLAG_VERIFY) {
-		lr->control->flags |= FLAG_CHECK;
-		lr->control->flags |= FLAG_HASH;
-	}
-	if (lr->flags & LRZIP_FLAG_REMOVE_DESTINATION)
-		lr->control->flags |= FLAG_FORCE_REPLACE;
-	if (lr->flags & LRZIP_FLAG_REMOVE_SOURCE)
-		lr->control->flags &= ~FLAG_KEEP_FILES;
-	if (lr->flags & LRZIP_FLAG_KEEP_BROKEN)
-		lr->control->flags |= FLAG_KEEP_BROKEN;
-	if (lr->flags & LRZIP_FLAG_DISABLE_LZO_CHECK)
-		lr->control->flags &= ~FLAG_THRESHOLD;
-	if (lr->flags & LRZIP_FLAG_UNLIMITED_RAM)
-		lr->control->flags |= FLAG_UNLIMITED;
-	if (lr->flags & LRZIP_FLAG_ENCRYPT)
-		lr->control->flags |= FLAG_ENCRYPT;
-	if (lr->control->log_level > 0) {
-		lr->control->flags |= FLAG_SHOW_PROGRESS;
-		if (lr->control->log_level > 1) {
-			lr->control->flags |= FLAG_VERBOSITY;
-			if (lr->control->log_level > 2)
-				lr->control->flags |= FLAG_VERBOSITY_MAX;
-		}
-	} else lr->control->flags ^= (FLAG_VERBOSE | FLAG_SHOW_PROGRESS);
-	return true;
-}
-
-
-bool lrzip_init(void)
-{
-	/* generate crc table */
-	CrcGenerateTable();
-	return true;
-}
-
-void lrzip_config_env(Lrzip *lr)
-{
-	const char *eptr;
-	/* Get Preloaded Defaults from lrzip.conf
-	 * Look in ., $HOME/.lrzip/, /etc/lrzip.
-	 * If LRZIP=NOCONFIG is set, then ignore config
-	 */
-	eptr = getenv("LRZIP");
-	if (!eptr)
-		read_config(lr->control);
-	else if (!strstr(eptr,"NOCONFIG"))
-		read_config(lr->control);
-}
-
-void lrzip_free(Lrzip *lr)
-{
-	size_t x;
-
-	if ((!lr) || (!lr->infilename_buckets))
-		return;
-	rzip_control_free(lr->control);
-	for (x = 0; x < lr->infilename_idx; x++)
-		free(lr->infilenames[x]);
-	free(lr->infilenames);
-	free(lr->infiles);
-	free(lr);
-}
-
-Lrzip *lrzip_new(Lrzip_Mode mode)
-{
-	Lrzip *lr;
-
-	lr = calloc(1, sizeof(Lrzip));
-	if (!lr)
-		return NULL;
-	lr->control = calloc(1, sizeof(rzip_control));
-	if (!lr->control)
-		goto error;
-	if (!initialize_control(lr->control))
-		goto error;
-	lr->mode = mode;
-	lr->control->library_mode = 1;
-	return lr;
-error:
-	lrzip_free(lr);
-	return NULL;
-}
-
-Lrzip_Mode lrzip_mode_get(Lrzip *lr)
-{
-	if (!lr)
-		return LRZIP_MODE_NONE;
-	return lr->mode;
-}
-
-bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode)
-{
-	if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ))
-		return false;
-	lr->mode = mode;
-	return true;
-}
-
-bool lrzip_compression_level_set(Lrzip *lr, unsigned int level)
-{
-	if ((!lr) || (!level) || (level > 9))
-		return false;
-	lr->control->compression_level = level;
-	return true;
-}
-
-unsigned int lrzip_compression_level_get(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->control->compression_level;
-}
-
-void lrzip_flags_set(Lrzip *lr, unsigned int flags)
-{
-	if (!lr)
-		return;
-	lr->flags = flags;
-}
-
-unsigned int lrzip_flags_get(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->flags;
-}
-
-void lrzip_nice_set(Lrzip *lr, int nice)
-{
-	if ((!lr) || (nice < -19) || (nice > 20))
-		return;
-	lr->control->nice_val = nice;
-}
-
-int lrzip_nice_get(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->control->nice_val;
-}
-
-void lrzip_threads_set(Lrzip *lr, unsigned int threads)
-{
-	if ((!lr) || (!threads))
-		return;
-	lr->control->threads = threads;
-}
-
-unsigned int lrzip_threads_get(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->control->threads;
-}
-
-void lrzip_compression_window_max_set(Lrzip *lr, int64_t size)
-{
-	if (!lr)
-		return;
-	lr->control->window = size;
-}
-
-int64_t lrzip_compression_window_max_get(Lrzip *lr)
-{
-	if (!lr)
-		return -1;
-	return lr->control->window;
-}
-
-unsigned int lrzip_files_count(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->infile_idx;
-}
-
-unsigned int lrzip_filenames_count(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->infilename_idx;
-}
-
-FILE **lrzip_files_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->infiles;
-}
-
-char **lrzip_filenames_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->infilenames;
-}
-
-bool lrzip_file_add(Lrzip *lr, FILE *file)
-{
-	if ((!lr) || (!file))
-		return false;
-	if (lr->infilenames)
-		return false;
-	if (!lr->infile_buckets) {
-		/* no files added */
-		lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*));
-		lr->infile_buckets++;
-	} else if (lr->infile_idx == INFILE_BUCKET_SIZE * lr->infile_buckets + 1) {
-		/* all buckets full, create new bucket */
-		FILE **tmp;
-
-		tmp = realloc(lr->infiles, (++lr->infile_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*));
-		if (!tmp)
-			return false;
-		lr->infiles = tmp;
-	}
-
-	lr->infiles[lr->infile_idx++] = file;
-	return true;
-}
-
-bool lrzip_file_del(Lrzip *lr, FILE *file)
-{
-	size_t x;
-
-	if ((!lr) || (!file))
-		return false;
-	if (!lr->infile_buckets)
-		return true;
-
-	for (x = 0; x <= lr->infile_idx + 1; x++) {
-		if (!lr->infiles[x])
-			return true; /* not found */
-		if (lr->infiles[x] != file)
-			continue; /* not a match */
-		break;
-	}
-	/* update index */
-	liblrzip_index_update(x, &lr->infile_idx, (void**)lr->infiles);
-	return true;
-}
-
-FILE *lrzip_file_pop(Lrzip *lr)
-{
-	FILE *ret;
-	if ((!lr) || (!lr->infile_buckets))
-		return NULL;
-	ret = lr->infiles[0];
-	lrzip_file_del(lr, ret);
-	return ret;
-}
-
-void lrzip_files_clear(Lrzip *lr)
-{
-	if ((!lr) || (!lr->infile_buckets))
-		return;
-	free(lr->infiles);
-	lr->infiles = NULL;
-}
-
-bool lrzip_filename_add(Lrzip *lr, const char *file)
-{
-	struct stat st;
-
-	if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-")))
-		return false;
-	if (lr->infiles)
-		return false;
-	if (stat(file, &st))
-		return false;
-	if (S_ISDIR(st.st_mode))
-		return false;
-
-	if (!lr->infilename_buckets) {
-		/* no files added */
-		lr->infilenames = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*));
-		lr->infilename_buckets++;
-	} else if (lr->infilename_idx == INFILE_BUCKET_SIZE * lr->infilename_buckets + 1) {
-		/* all buckets full, create new bucket */
-		char **tmp;
-
-		tmp = realloc(lr->infilenames, (++lr->infilename_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*));
-		if (!tmp)
-			return false;
-		lr->infilenames = tmp;
-	}
-
-	lr->infilenames[lr->infilename_idx++] = strdup(file);
-	return true;
-}
-
-bool lrzip_filename_del(Lrzip *lr, const char *file)
-{
-	size_t x;
-
-	if ((!lr) || (!file) || (!file[0]))
-		return false;
-	if (!lr->infilename_buckets)
-		return true;
-
-	for (x = 0; x <= lr->infilename_idx + 1; x++) {
-		if (!lr->infilenames[x])
-			return true; /* not found */
-		if (strcmp(lr->infilenames[x], file))
-			continue; /* not a match */
-		free(lr->infilenames[x]);
-		break;
-	}
-	/* update index */
-	liblrzip_index_update(x, &lr->infilename_idx, (void**)lr->infilenames);
-	return true;
-}
-
-const char *lrzip_filename_pop(Lrzip *lr)
-{
-	static char buf[4096];
-	if ((!lr) || (!lr->infilename_buckets))
-		return NULL;
-	strcat(buf, lr->infilenames[0]);
-	lrzip_filename_del(lr, buf);
-	return &buf[0];
-}
-
-void lrzip_filenames_clear(Lrzip *lr)
-{
-	size_t x;
-	if ((!lr) || (!lr->infilename_buckets))
-		return;
-	for (x = 0; x < lr->infilename_idx; x++)
-		free(lr->infilenames[x]);
-	free(lr->infilenames);
-	lr->infilenames = NULL;
-}
-
-void lrzip_suffix_set(Lrzip *lr, const char *suffix)
-{
-	if ((!lr) || (!suffix) || (!suffix[0]))
-		return;
-	free(lr->control->suffix);
-	lr->control->suffix = strdup(suffix);
-}
-
-const char *lrzip_suffix_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->suffix;
-}
-
-void lrzip_outdir_set(Lrzip *lr, const char *dir)
-{
-	const char *slash;
-	char *buf;
-	size_t len;
-	if ((!lr) || (!dir) || (!dir[0]))
-		return;
-	free(lr->control->outdir);
-	slash = strrchr(dir, '/');
-	if (slash && (slash[1] == 0)) {
-		lr->control->outdir = strdup(dir);
-		return;
-	}
-	len = strlen(dir);
-	buf = malloc(len + 2);
-	if (!buf)
-		return;
-	memcpy(buf, dir, len);
-	buf[len] = '/';
-	buf[len + 1] = 0;
-	lr->control->outdir = buf;
-}
-
-const char *lrzip_outdir_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->outdir;
-}
-
-void lrzip_outfile_set(Lrzip *lr, FILE *file)
-{
-	if ((!lr) || (file && (file == stderr)))
-		return;
-	if (lr->control->outname)
-		return;
-	lr->control->outFILE = file;
-}
-
-FILE *lrzip_outfile_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->outFILE;
-}
-
-void lrzip_outfilename_set(Lrzip *lr, const char *file)
-{
-	if ((!lr) || (file && (!file[0])))
-		return;
-	if (lr->control->outFILE)
-		return;
-	if (lr->control->outname && file && (!strcmp(lr->control->outname, file)))
-		return;
-	free(lr->control->outname);
-	lr->control->outname = file ? strdup(file) : NULL;
-}
-
-const char *lrzip_outfilename_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->outname;
-}
-
-const unsigned char *lrzip_md5digest_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->md5_resblock;
-}
-
-bool lrzip_run(Lrzip *lr)
-{
-	struct timeval start_time, end_time;
-	rzip_control *control;
-	double seconds,total_time; // for timers
-	int hours,minutes;
-
-	if (!liblrzip_setup_flags(lr))
-		return false;
-	control = lr->control;
-
-	if ((!lr->infile_idx) && (!lr->infilename_idx))
-		return false;
-	if (lr->control->outFILE) {
-		if (lr->control->outFILE == lr->control->msgout)
-			lr->control->msgout = stderr;
-		lr->control->flags |= FLAG_STDOUT;
-		register_outputfile(lr->control, lr->control->msgout);
-	}
-
-	if (lr->infilenames)
-		lr->control->infile = lr->infilenames[0];
-	else {
-		lr->control->inFILE = lr->infiles[0];
-		control->flags |= FLAG_STDIN;
-	}
-
-	if ((!STDOUT) && (!lr->control->msgout)) lr->control->msgout = stdout;
-	register_outputfile(lr->control, lr->control->msgout);
-
-	setup_ram(lr->control);
-
-	gettimeofday(&start_time, NULL);
-
-	if (ENCRYPT && (!lr->control->pass_cb)) {
-		print_err("No password callback set!\n");
-		return false;
-	}
-
-	if (DECOMPRESS || TEST_ONLY) {
-		if (!decompress_file(lr->control))
-			return false;
-	} else if (INFO) {
-		if (!get_fileinfo(lr->control))
-			return false;
-	} else if (!compress_file(lr->control))
-		return false;
-
-	/* compute total time */
-	gettimeofday(&end_time, NULL);
-	total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) -
-	             (start_time.tv_sec + (double)start_time.tv_usec / 1000000);
-	hours = (int)total_time / 3600;
-	minutes = (int)(total_time / 60) % 60;
-	seconds = total_time - hours * 3600 - minutes * 60;
-	if (!INFO)
-		print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
-
-	return true;
-}
-
-void lrzip_log_level_set(Lrzip *lr, int level)
-{
-	if (!lr)
-		return;
-	lr->control->log_level = level;
-}
-
-int lrzip_log_level_get(Lrzip *lr)
-{
-	if (!lr)
-		return 0;
-	return lr->control->log_level;
-}
-
-void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data)
-{
-	if (!lr)
-		return;
-	lr->control->log_cb = cb;
-	lr->control->log_data = log_data;
-}
-
-void lrzip_log_stdout_set(Lrzip *lr, FILE *out)
-{
-	if (!lr)
-		return;
-	lr->control->msgout = out;
-}
-
-FILE *lrzip_log_stdout_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->msgout;
-}
-
-void lrzip_log_stderr_set(Lrzip *lr, FILE *err)
-{
-	if (!lr)
-		return;
-	lr->control->msgerr = err;
-}
-
-FILE *lrzip_log_stderr_get(Lrzip *lr)
-{
-	if (!lr)
-		return NULL;
-	return lr->control->msgerr;
-}
-
-void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data)
-{
-	if (!lr)
-		return;
-	lr->control->pass_cb = cb;
-	lr->control->pass_data = data;
-}
-
-void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data)
-{
-	if (!lr)
-		return;
-	lr->control->info_cb = cb;
-	lr->control->info_data = data;
-}
-
-bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level)
-{
-	FILE *s = NULL, *d = NULL;
-	Lrzip *lr = NULL;
-	bool ret = false;
-	struct stat st;
-	int fd;
-
-	if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE))
-		goto out;
-
-	lrzip_init();
-	if (!mode) mode = LRZIP_MODE_COMPRESS_LZMA;
-	lr = lrzip_new(mode);
-	if (!lr)
-		goto out;
-	lrzip_config_env(lr);
-
-	s = fmemopen((void*)source, source_len, "r");
-	d = tmpfile();
-	if ((!s) || (!d))
-		goto out;
-
-	if (!lrzip_file_add(lr, s))
-		goto out;
-	lrzip_outfile_set(lr, d);
-	if (!lrzip_compression_level_set(lr, compress_level))
-		goto out;
-	if (!lrzip_run(lr))
-		goto out;
-
-	fd = fileno(d);
-	if (fstat(fd, &st))
-		goto out;
-	*dest_len = st.st_size;
-	if (unlikely((i64)fread(dest, sizeof(char), st.st_size, d) != st.st_size))
-		goto out;
-	if (unlikely(ferror(d)))
-		goto out;
-	ret = true;
-
-out:
-	if (s) fclose(s);
-	if (d) fclose(d);
-	lrzip_free(lr);
-	return ret;
-}
-
-
-bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
-{
-	FILE *s = NULL, *d = NULL;
-	Lrzip *lr = NULL;
-	bool ret = false;
-	struct stat st;
-	int fd;
-
-	if ((!dest) || (!dest_len) || (!source) || (!source_len))
-		goto out;
-
-	lrzip_init();
-	lr = lrzip_new(LRZIP_MODE_DECOMPRESS);
-	if (!lr)
-		goto out;
-	lrzip_config_env(lr);
-
-	s = fmemopen((void*)source, source_len, "r");
-	d = tmpfile();
-	if ((!s) || (!d))
-		goto out;
-
-	if (!lrzip_file_add(lr, s))
-		goto out;
-	lrzip_outfile_set(lr, d);
-	if (!lrzip_run(lr))
-		goto out;
-
-	fd = fileno(d);
-	if (fstat(fd, &st))
-		goto out;
-	*dest_len = st.st_size;
-	if (unlikely((i64)fread(dest, sizeof(char), st.st_size, d) != st.st_size))
-		goto out;
-	if (unlikely(ferror(d)))
-		goto out;
-	ret = true;
-
-out:
-	if (s) fclose(s);
-	if (d) fclose(d);
-	lrzip_free(lr);
-	return ret;
-}

liblrzip_demo.c

@@ -1,345 +0,0 @@
-/*
-   Copyright (C) 2012 Con Kolivas
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#ifdef HAVE_CONFIG_H
-# include "config.h"
-#endif
-#undef NDEBUG
-#include <stdlib.h>
-#include <stdio.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#endif
-#include <assert.h>
-#ifdef HAVE_ERRNO_H
-# include <errno.h>
-#else
-extern int errno;
-#endif
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-#include <termios.h>
-#include <Lrzip.h>
-
-#define failure(...) do { \
-	fprintf(stderr, __VA_ARGS__); \
-	exit(1); \
-} while (0)
-
-static void usage(void)
-{
-	printf("lrzip version %s\n", PACKAGE_VERSION);
-	printf("Copyright (C) Con Kolivas 2006-2011\n");
-	printf("Based on rzip ");
-	printf("Copyright (C) Andrew Tridgell 1998-2003\n\n");
-	printf("Usage: lrzip [options] <file...>\n");
-	printf("General options:\n");
-	printf("     -c            check integrity of file written on decompression\n");
-	printf("     -d            decompress\n");
-	printf("     -e            password protected sha512/aes128 encryption on compression\n");
-	printf("     -h|-?         show help\n");
-	printf("     -H            display md5 hash integrity information\n");
-	printf("     -i            show compressed file information\n");
-	printf("     -q            don't show compression progress\n");
-	printf("     -t            test compressed file integrity\n");
-	printf("     -v[v]         Increase verbosity\n");
-	printf("     -V            show version\n");
-	printf("Options affecting output:\n");
-	printf("     -D            delete existing files\n");
-	printf("     -f            force overwrite of any existing files\n");
-	printf("     -k            keep broken or damaged output files\n");
-	printf("     -o filename   specify the output file name and/or path\n");
-	printf("     -O directory  specify the output directory when -o is not used\n");
-	printf("     -S suffix     specify compressed suffix (default '.lrz')\n");
-	printf("Options affecting compression:\n");
-	printf("     -b            bzip2 compression\n");
-	printf("     -g            gzip compression using zlib\n");
-	printf("     -l            lzo compression (ultra fast)\n");
-	printf("     -n            no backend compression - prepare for other compressor\n");
-	printf("     -z            zpaq compression (best, extreme compression, extremely slow)\n");
-	printf("Low level options:\n");
-	printf("     -L level      set lzma/bzip2/gzip compression level (1-9, default 7)\n");
-	printf("     -N value      Set nice value to value (default 19)\n");
-	printf("     -p value      Set processor count to override number of threads\n");
-	printf("     -T            Disable LZO compressibility testing\n");
-	printf("     -U            Use unlimited window size beyond ramsize (potentially much slower)\n");
-	printf("     -w size       maximum compression window in hundreds of MB\n");
-	printf("                   default chosen by heuristic dependent on ram and chosen compression\n");
-	printf("\nLRZIP=NOCONFIG environment variable setting can be used to bypass lrzip.conf.\n");
-	printf("TMP environment variable will be used for storage of temporary files when needed.\n");
-	printf("TMPDIR may also be stored in lrzip.conf file.\n");
-	printf("\nIf no filenames or \"-\" is specified, stdin/out will be used.\n");
-}
-
-static int get_pass(char *s, size_t slen)
-{
-	int len;
-
-	memset(s, 0, slen);
-	if (!fgets(s, slen, stdin)) {
-		fprintf(stderr, "Failed to retrieve passphrase\n");
-		return -1;
-	}
-	len = strlen(s);
-	if (len > 0 && ('\r' ==  s[len - 1] || '\n' == s[len - 1]))
-		s[len - 1] = '\0';
-	if (len > 1 && ('\r' ==  s[len - 2] || '\n' == s[len - 2]))
-		s[len - 2] = '\0';
-	len = strlen(s);
-	if (!len) {
-		fprintf(stderr, "Empty passphrase\n");
-		return -1;
-	}
-	return len;
-}
-
-static void pass_cb(void *data __UNUSED__, char *pass_string, size_t pass_len)
-{
-	int len;
-	struct termios termios_p;
-	/* Disable stdin echo to screen */
-	tcgetattr(fileno(stdin), &termios_p);
-	termios_p.c_lflag &= ~ECHO;
-	tcsetattr(fileno(stdin), 0, &termios_p);
-
-	printf("Enter passphrase: ");
-	len = get_pass(pass_string, pass_len);
-	printf("\n");
-
-	if (len < 1) exit(1);
-
-	termios_p.c_lflag |= ECHO;
-	tcsetattr(fileno(stdin), 0, &termios_p);
-}
-
-static void mode_check(Lrzip *lr, Lrzip_Mode mode)
-{
-	Lrzip_Mode current = lrzip_mode_get(lr);
-	if (current && (current != mode))
-		failure("Can only use one of -l, -b, -g, -z or -n\n");
-	lrzip_mode_set(lr, mode);
-}
-
-int main(int argc, char *argv[])
-{
-	Lrzip *lr;
-	extern int optind;
-	extern char *optarg;
-	int64_t x;
-	int c;
-	bool get_hash = false;
-
-	lrzip_init();
-	lr = lrzip_new(LRZIP_MODE_NONE);
-	assert(lr);
-	lrzip_config_env(lr);
-	lrzip_log_level_set(lr, LRZIP_LOG_LEVEL_PROGRESS);
-	while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) {
-		switch (c) {
-		case 'b':
-			mode_check(lr, LRZIP_MODE_COMPRESS_BZIP2);
-			break;
-		case 'c':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_VERIFY);
-			break;
-		case 'd':
-			mode_check(lr, LRZIP_MODE_DECOMPRESS);
-			break;
-		case 'D':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_SOURCE);
-			break;
-		case 'e':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_ENCRYPT);
-			break;
-		case 'f':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_DESTINATION);
-			break;
-		case 'g':
-			mode_check(lr, LRZIP_MODE_COMPRESS_ZLIB);
-			break;
-		case 'h':
-		case '?':
-			usage();
-			return -1;
-		case 'H':
-			get_hash = true;
-			break;
-		case 'i':
-			mode_check(lr, LRZIP_MODE_INFO);
-			break;
-		case 'k':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_KEEP_BROKEN);
-			break;
-		case 'l':
-			mode_check(lr, LRZIP_MODE_COMPRESS_LZO);
-			break;
-		case 'L':
-			errno = 0;
-			x = strtol(optarg, NULL, 10);
-			if (errno || ((x < 1) || (x > 9)))
-				failure("Invalid compression level (must be 1-9)\n");
-			lrzip_compression_level_set(lr, (unsigned int)x);
-			break;
-		case 'n':
-			mode_check(lr, LRZIP_MODE_COMPRESS_NONE);
-			break;
-		case 'N':
-			errno = 0;
-			x = strtol(optarg, NULL, 10);
-			if (errno || (x < -20 || x > 19))
-				failure("Invalid nice value (must be -20..19)\n");
-			lrzip_nice_set(lr, x);
-			break;
-		case 'o':
-			if (lrzip_outdir_get(lr))
-				failure("Cannot have -o and -O together\n");
-			if (!strcmp(optarg, "-"))
-				lrzip_outfile_set(lr, stdout);
-			else
-				lrzip_outfilename_set(lr, optarg);
-			break;
-		case 'O':
-			if (lrzip_outfilename_get(lr))	/* can't mix -o and -O */
-				failure("Cannot have options -o and -O together\n");
-			if (lrzip_outfile_get(lr))
-				failure("Cannot specify an output directory when outputting to stdout\n");
-			lrzip_outdir_set(lr, optarg);
-			break;
-		case 'p':
-			errno = 0;
-			x = strtol(optarg, NULL, 10);
-			if (errno || (x < 1))
-				failure("Must have at least one thread\n");
-			lrzip_threads_set(lr, (unsigned int)x);
-			break;
-		case 'q':
-			lrzip_log_level_set(lr, lrzip_log_level_get(lr) - 1);
-			break;
-		case 'S':
-			if (lrzip_outfilename_get(lr))
-				failure("Specified output filename already, can't specify an extension.\n");
-			if (lrzip_outfile_get(lr))
-				failure("Cannot specify a filename suffix when outputting to stdout\n");
-			lrzip_suffix_set(lr, optarg);
-			break;
-		case 't':
-			if (lrzip_outfilename_get(lr))
-				failure("Cannot specify an output file name when just testing.\n");
-			if (lrzip_flags_get(lr) & LRZIP_FLAG_REMOVE_SOURCE)
-				failure("Doubt that you want to delete a file when just testing.\n");
-			mode_check(lr, LRZIP_MODE_TEST);
-			break;
-		case 'T':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_DISABLE_LZO_CHECK);
-			break;
-		case 'U':
-			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_UNLIMITED_RAM);
-			break;
-		case 'v':
-			lrzip_log_level_set(lr, lrzip_log_level_get(lr) + 1);
-			break;
-		case 'V':
-			printf("lrzip version %s\n", PACKAGE_VERSION);
-			exit(0);
-			break;
-		case 'w':
-			errno = 0;
-			x = strtoll(optarg, NULL, 10);
-			if (errno || (x < 1))
-				failure("Invalid compression window '%s'!\n", optarg);
-			lrzip_compression_window_max_set(lr, x);
-			break;
-		case 'z':
-			mode_check(lr, LRZIP_MODE_COMPRESS_ZPAQ);
-			break;
-		}
-	}
-	/* LZMA is the default */
-	if (!lrzip_mode_get(lr)) lrzip_mode_set(lr, LRZIP_MODE_COMPRESS_LZMA);
-	argc -= optind, argv += optind;
-
-	if (lrzip_outfilename_get(lr) && (argc > 1))
-		failure("Cannot specify output filename with more than 1 file\n");
-
-	if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_compression_window_max_get(lr)) {
-		fprintf(stderr, "If -U used, cannot specify a window size with -w.\n");
-		lrzip_compression_window_max_set(lr, 0);
-	}
-
-	if (argc < 1) lrzip_file_add(lr, stdin);
-
-	if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_files_count(lr)) {
-		fprintf(stderr, "Cannot have -U and stdin, unlimited mode disabled.\n");
-		lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_UNLIMITED_RAM);
-	}
-
-	/* If no output filename is specified, and we're using stdin,
-	 * use stdout */
-	if (lrzip_files_count(lr) && (!lrzip_outfilename_get(lr)))
-		lrzip_outfile_set(lr, stdout);
-
-	if (lrzip_flags_get(lr) & LRZIP_FLAG_VERIFY) {
-		if (lrzip_mode_get(lr) != LRZIP_MODE_DECOMPRESS) {
-			fprintf(stderr, "Can only check file written on decompression.\n");
-			lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY);
-		} else if (lrzip_outfile_get(lr)) {
-			fprintf(stderr, "Can't check file written when writing to stdout. Checking disabled.\n");
-			lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY);
-		}
-	}
-
-	for (x = 0; x < argc; x++) {
-		if (argv[x][0] != '-') {
-			assert(lrzip_filename_add(lr, argv[x]));
-			continue;
-		}
-		if (argv[x][1] == 0) {
-			assert(lrzip_file_add(lr, stdin));
-			continue;
-		}
-	}
-	if (argc == 1) {
-		if (!lrzip_files_count(lr)) lrzip_file_add(lr, stdin);
-		if (lrzip_filenames_count(lr)) {
-			if (!lrzip_outfilename_get(lr)) {
-				char buf[4096];
-				const char *infile;
-				size_t len;
-
-				infile = lrzip_filenames_get(lr)[0];
-				len = strlen(infile);
-				if (!strcmp(infile + len - 4, ".lrz"))
-					strncat(buf, infile, len - 4);
-				else
-					snprintf(buf, sizeof(buf), "%s.out", infile);
-				lrzip_outfilename_set(lr, buf);
-			}
-		} else if (!lrzip_outfile_get(lr)) lrzip_outfile_set(lr, stdout);
-	}
-	lrzip_log_stdout_set(lr, stdout);
-	lrzip_log_stderr_set(lr, stderr);
-	lrzip_pass_cb_set(lr, pass_cb, NULL);
-	if (!lrzip_run(lr)) exit(1);
-	if (get_hash) {
-		const unsigned char *digest = lrzip_md5digest_get(lr);
-		for (x = 0; x < 16; x++)
-			fprintf(stdout, "%02x", digest[x] & 0xFF);
-	}
-	lrzip_free(lr);
-	return 0;
-}

liblrzip_private.h

@@ -1,22 +0,0 @@
-#ifdef HAVE_CONFIG_H
-# include "config.h"
-#endif
-#include <lrzip_private.h>
-#include <Lrzip.h>
-
-#define INFILE_BUCKET_SIZE 10
-
-struct Lrzip
-{
-	Lrzip_Mode mode;
-	unsigned int flags;
-	rzip_control *control;
-
-	/* bucket allocation is used here to avoid frequent calls to realloc */
-	char **infilenames;
-	size_t infilename_idx;
-	size_t infilename_buckets;
-	FILE **infiles;
-	size_t infile_idx;
-	size_t infile_buckets;
-};

libzpaq/libzpaq.h

@@ -465,7 +465,8 @@ struct bufRead: public libzpaq::Reader {
 
 	int get() {
 		if (progress && !(*s_len % 128)) {
-			int pct = (total_len - *s_len) * 100 / total_len;
+			int pct = (total_len > 0) ?
+				(total_len - *s_len) * 100 / total_len : 100;
 
 			if (pct / 10 != *last_pct / 10) {
 				int i;

lrzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -56,6 +56,7 @@
 #include "stream.h"
 
 #define MAGIC_LEN (24)
+#define STDIO_TMPFILE_BUFFER_SIZE (65536) // used in read_tmpinfile and dump_tmpoutfile
 
 static void release_hashes(rzip_control *control);
 
@@ -90,6 +91,32 @@ i64 get_ram(rzip_control *control)
 
 	return ramsize;
 }
+#elif defined(__OpenBSD__)
+# include <sys/resource.h>
+i64 get_ram(rzip_control *control)
+{
+	struct rlimit rl;
+	i64 ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE;
+
+	/* Raise limits all the way to the max */
+
+	if (getrlimit(RLIMIT_DATA, &rl) == -1)
+		fatal_return(("Failed to get limits in get_ram\n"), -1);
+
+	rl.rlim_cur = rl.rlim_max;
+	if (setrlimit(RLIMIT_DATA, &rl) == -1)
+		fatal_return(("Failed to set limits in get_ram\n"), -1);
+
+	/* Declare detected RAM to be either the max RAM available from
+	physical memory or the max RAM allowed by RLIMIT_DATA, whatever
+	is smaller, to prevent the heuristics from selecting
+	compression windows which cause lrzip to go into deep swap */
+
+	if (rl.rlim_max < ramsize)
+		return rl.rlim_max;
+
+	return ramsize;
+}
 #else /* __APPLE__ */
 i64 get_ram(rzip_control *control)
 {
@@ -218,11 +245,14 @@ static bool get_magic(rzip_control *control, char *magic)
 	if ((int) magic[16]) {
 		for (i = 0; i < 5; i++)
 			control->lzma_properties[i] = magic[i + 16];
+		/* Cludge to allow us to read possibly corrupted archives */
+		if (!control->lzma_properties[0])
+			control->lzma_properties[0] = 93;
 	}
 
 	/* Whether this archive contains md5 data at the end or not */
 	md5 = magic[21];
-	if (md5 && MD5_RELIABLE) {
+	if (md5) {
 		if (md5 == 1)
 			control->flags |= FLAG_MD5;
 		else
@@ -311,10 +341,11 @@ int open_tmpoutfile(rzip_control *control)
 
 	fd_out = mkstemp(control->outfile);
 	if (fd_out == -1) {
-		print_progress("WARNING: Failed to create out tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
+		print_output("WARNING: Failed to create out tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
 			       control->outfile, DECOMPRESS ? "de" : "");
 	} else
 		register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN);
+	print_maxverbose("Created temporary outfile %s\n", control->outfile);
 	return fd_out;
 }
 
@@ -326,12 +357,14 @@ static bool fwrite_stdout(rzip_control *control, void *buf, i64 len)
 
 	total = 0;
 	while (len > 0) {
-		if (len > one_g)
-			ret = one_g;
+		ssize_t wrote;
+
+		if (BITS32)
+			ret = MIN(len, one_g);
 		else
 			ret = len;
-		ret = fwrite(offset_buf, 1, ret, control->outFILE);
-		if (unlikely(ret <= 0))
+		wrote = fwrite(offset_buf, 1, ret, control->outFILE);
+		if (unlikely(wrote != ret))
 			fatal_return(("Failed to fwrite in fwrite_stdout\n"), false);
 		len -= ret;
 		offset_buf += ret;
@@ -347,7 +380,10 @@ bool write_fdout(rzip_control *control, void *buf, i64 len)
 	ssize_t ret;
 
 	while (len > 0) {
+		if (BITS32)
 			ret = MIN(len, one_g);
+		else
+			ret = len;
 		ret = write(control->fd_out, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			fatal_return(("Failed to write to fd_out in write_fdout\n"), false);
@@ -357,7 +393,7 @@ bool write_fdout(rzip_control *control, void *buf, i64 len)
 	return true;
 }
 
-bool flush_tmpoutbuf(rzip_control *control)
+static bool flush_tmpoutbuf(rzip_control *control)
 {
 	if (!TEST_ONLY) {
 		print_maxverbose("Dumping buffer to physical file.\n");
@@ -375,10 +411,10 @@ bool flush_tmpoutbuf(rzip_control *control)
 }
 
 /* Dump temporary outputfile to perform stdout */
-bool dump_tmpoutfile(rzip_control *control, int fd_out)
+static bool dump_tmpoutfile(rzip_control *control)
 {
+	int fd_out = control->fd_out;
 	FILE *tmpoutfp;
-	int tmpchar;
 
 	if (unlikely(fd_out == -1))
 		fatal_return(("Failed: No temporary outfile created, unable to do in ram\n"), false);
@@ -390,9 +426,35 @@ bool dump_tmpoutfile(rzip_control *control, int fd_out)
 	rewind(tmpoutfp);
 
 	if (!TEST_ONLY) {
+		char* buf;
+
 		print_verbose("Dumping temporary file to control->outFILE.\n");
-		while ((tmpchar = fgetc(tmpoutfp)) != EOF)
-			putchar(tmpchar);
+		fflush(control->outFILE);
+
+		buf = malloc(STDIO_TMPFILE_BUFFER_SIZE);
+		if (unlikely(!buf))
+			fatal_return(("Failed to allocate buffer in dump_tmpoutfile\n"), false);
+
+		while (1) {
+			ssize_t num_read, num_written;
+			num_read = fread(buf, 1, STDIO_TMPFILE_BUFFER_SIZE, tmpoutfp);
+			if (unlikely(num_read == 0)) {
+				if (ferror(tmpoutfp)) {
+					dealloc(buf);
+					fatal_return(("Failed read in dump_tmpoutfile\n"), false);
+				} else {
+					break; // must be at EOF
+				}
+			}
+
+			num_written = fwrite(buf, 1, num_read, control->outFILE);
+			if (unlikely(num_written != num_read)) {
+				dealloc(buf);
+				fatal_return(("Failed write in dump_tmpoutfile\n"), false);
+			}
+		}
+
+		dealloc(buf);
 		fflush(control->outFILE);
 		rewind(tmpoutfp);
 	}
@@ -402,6 +464,15 @@ bool dump_tmpoutfile(rzip_control *control, int fd_out)
 	return true;
 }
 
+bool flush_tmpout(rzip_control *control)
+{
+	if (!STDOUT)
+		return true;
+	if (TMP_OUTBUF)
+		return flush_tmpoutbuf(control);
+	return dump_tmpoutfile(control);
+}
+
 /* Used if we're unable to read STDIN into the temporary buffer, shunts data
  * to temporary file */
 bool write_fdin(rzip_control *control)
@@ -411,7 +482,10 @@ bool write_fdin(rzip_control *control)
 	ssize_t ret;
 
 	while (len > 0) {
+		if (BITS32)
 			ret = MIN(len, one_g);
+		else
+			ret = len;
 		ret = write(control->fd_in, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			fatal_return(("Failed to write to fd_in in write_fdin\n"), false);
@@ -438,7 +512,7 @@ int open_tmpinfile(rzip_control *control)
 
 	/* Try the current directory */
 	if (fd_in == -1) {
-		free(control->infile);
+		dealloc(control->infile);
 		control->infile = malloc(16);
 		if (unlikely(!control->infile))
 			fatal_return(("Failed to allocate infile name\n"), -1);
@@ -448,7 +522,7 @@ int open_tmpinfile(rzip_control *control)
 
 	/* Use /tmp if nothing is writeable so far */
 	if (fd_in == -1) {
-		free(control->infile);
+		dealloc(control->infile);
 		control->infile = malloc(20);
 		if (unlikely(!control->infile))
 			fatal_return(("Failed to allocate infile name\n"), -1);
@@ -457,7 +531,7 @@ int open_tmpinfile(rzip_control *control)
 	}
 
 	if (fd_in == -1) {
-		print_progress("WARNING: Failed to create in tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
+		print_output("WARNING: Failed to create in tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
 			       control->infile, DECOMPRESS ? "de" : "");
 	} else {
 		register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN);
@@ -492,6 +566,7 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
 {
 	FILE *tmpinfp;
 	int tmpchar;
+	char* buf;
 
 	if (fd_in == -1)
 		return false;
@@ -501,9 +576,30 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
 	if (unlikely(tmpinfp == NULL))
 		fatal_return(("Failed to fdopen in tmpfile\n"), false);
 
-	while ((tmpchar = getchar()) != EOF)
-		fputc(tmpchar, tmpinfp);
+	buf = malloc(STDIO_TMPFILE_BUFFER_SIZE);
+	if (unlikely(!buf))
+		fatal_return(("Failed to allocate buffer in read_tmpinfile\n"), false);
 
+	while (1) {
+		ssize_t num_read, num_written;
+		num_read = fread(buf, 1, STDIO_TMPFILE_BUFFER_SIZE, stdin);
+		if (unlikely(num_read == 0)) {
+			if (ferror(stdin)) {
+				dealloc(buf);
+				fatal_return(("Failed read in read_tmpinfile\n"), false);
+			} else {
+				break; // must be at EOF
+			}
+		}
+
+		num_written = fwrite(buf, 1, num_read, tmpinfp);
+		if (unlikely(num_written != num_read)) {
+			dealloc(buf);
+			fatal_return(("Failed write in read_tmpinfile\n"), false);
+		}
+	}
+
+	dealloc(buf);
 	fflush(tmpinfp);
 	rewind(tmpinfp);
 	return true;
@@ -543,7 +639,7 @@ static bool open_tmpoutbuf(rzip_control *control)
 void close_tmpoutbuf(rzip_control *control)
 {
 	control->flags &= ~FLAG_TMP_OUTBUF;
-	free(control->tmp_outbuf);
+	dealloc(control->tmp_outbuf);
 	if (!BITS32)
 		control->usable_ram = control->maxram += control->ramsize / 18;
 }
@@ -576,7 +672,7 @@ bool clear_tmpinfile(rzip_control *control)
 void close_tmpinbuf(rzip_control *control)
 {
 	control->flags &= ~FLAG_TMP_INBUF;
-	free(control->tmp_inbuf);
+	dealloc(control->tmp_inbuf);
 	if (!BITS32)
 		control->usable_ram = control->maxram += control->ramsize / 18;
 }
@@ -586,7 +682,9 @@ static int get_pass(rzip_control *control, char *s)
 	int len;
 
 	memset(s, 0, PASS_LEN - SALT_LEN);
-	if (unlikely(fgets(s, PASS_LEN - SALT_LEN, stdin) == NULL))
+	if (control->passphrase)
+		strncpy(s, control->passphrase, PASS_LEN - SALT_LEN - 1);
+	else if (unlikely(fgets(s, PASS_LEN - SALT_LEN, stdin) == NULL))
 		failure_return(("Failed to retrieve passphrase\n"), -1);
 	len = strlen(s);
 	if (len > 0 && ('\r' ==  s[len - 1] || '\n' == s[len - 1]))
@@ -603,6 +701,7 @@ static bool get_hash(rzip_control *control, int make_hash)
 {
 	char *passphrase, *testphrase;
 	struct termios termios_p;
+	int prompt = control->passphrase == NULL;
 
 	passphrase = calloc(PASS_LEN, 1);
 	testphrase = calloc(PASS_LEN, 1);
@@ -610,8 +709,8 @@ static bool get_hash(rzip_control *control, int make_hash)
 	control->hash = calloc(HASH_LEN, 1);
 	if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) {
 		fatal("Failed to calloc encrypt buffers in compress_file\n");
-		free(testphrase);
-		free(passphrase);
+		dealloc(testphrase);
+		dealloc(passphrase);
 		return false;
 	}
 	mlock(passphrase, PASS_LEN);
@@ -625,8 +724,8 @@ static bool get_hash(rzip_control *control, int make_hash)
 			fatal("Supplied password was null!");
 			munlock(passphrase, PASS_LEN);
 			munlock(testphrase, PASS_LEN);
-			free(testphrase);
-			free(passphrase);
+			dealloc(testphrase);
+			dealloc(passphrase);
 			release_hashes(control);
 			return false;
 		}
@@ -637,12 +736,16 @@ static bool get_hash(rzip_control *control, int make_hash)
 		termios_p.c_lflag &= ~ECHO;
 		tcsetattr(fileno(stdin), 0, &termios_p);
 retry_pass:
+		if (prompt)
 			print_output("Enter passphrase: ");
 		control->salt_pass_len = get_pass(control, passphrase) + SALT_LEN;
+		if (prompt)
 			print_output("\n");
 		if (make_hash) {
+			if (prompt)
 				print_output("Re-enter passphrase: ");
 			get_pass(control, testphrase);
+			if (prompt)
 				print_output("\n");
 			if (strcmp(passphrase, testphrase)) {
 				print_output("Passwords do not match. Try again.\n");
@@ -659,8 +762,8 @@ retry_pass:
 	memset(passphrase, 0, PASS_LEN);
 	munlock(passphrase, PASS_LEN);
 	munlock(testphrase, PASS_LEN);
-	free(testphrase);
-	free(passphrase);
+	dealloc(testphrase);
+	dealloc(passphrase);
 	return true;
 }
 
@@ -670,8 +773,23 @@ static void release_hashes(rzip_control *control)
 	memset(control->hash, 0, SALT_LEN);
 	munlock(control->salt_pass, PASS_LEN);
 	munlock(control->hash, HASH_LEN);
-	free(control->salt_pass);
-	free(control->hash);
+	dealloc(control->salt_pass);
+	dealloc(control->hash);
+}
+
+static void clear_rulist(rzip_control *control)
+{
+	while (control->ruhead) {
+		struct runzip_node *node = control->ruhead;
+		struct stream_info *sinfo = node->sinfo;
+
+		dealloc(sinfo->ucthreads);
+		dealloc(node->pthreads);
+		dealloc(sinfo->s);
+		dealloc(sinfo);
+		control->ruhead = node->prev;
+		dealloc(node);
+	}
 }
 
 /*
@@ -684,7 +802,7 @@ bool decompress_file(rzip_control *control)
 	i64 expected_size = 0, free_space;
 	struct statvfs fbuf;
 
-	if (!STDIN) {
+	if (!STDIN && !IS_FROM_FILE) {
 		struct stat fdin_stat;
 
 		stat(control->infile, &fdin_stat);
@@ -731,10 +849,13 @@ bool decompress_file(rzip_control *control)
 		}
 
 		if (!STDOUT)
-			print_progress("Output filename is: %s\n", control->outfile);
+			print_output("Output filename is: %s\n", control->outfile);
 	}
 
-	if (STDIN) {
+	if ( IS_FROM_FILE ) {
+		fd_in = fileno(control->inFILE);
+	}
+	else if (STDIN) {
 		fd_in = open_tmpinfile(control);
 		read_tmpinmagic(control);
 		if (ENCRYPT)
@@ -785,12 +906,16 @@ bool decompress_file(rzip_control *control)
 		}
 	}
 
+	if (STDOUT) {
 		if (unlikely(!open_tmpoutbuf(control)))
 			return false;
+	}
 
 	if (!STDIN) {
 		if (unlikely(!read_magic(control, fd_in, &expected_size)))
 			return false;
+		if (unlikely(expected_size < 0))
+			fatal_return(("Invalid expected size %lld\n", expected_size), false);
 	}
 
 	if (!STDOUT && !TEST_ONLY) {
@@ -821,26 +946,30 @@ bool decompress_file(rzip_control *control)
 		if (unlikely(!get_hash(control, 0)))
 			return false;
 
-	print_progress("Decompressing...\n");
+	print_output("Decompressing...\n");
 
-	if (unlikely(runzip_fd(control, fd_in, fd_out, fd_hist, expected_size) < 0))
-		return false;
-
-	if (STDOUT && !TMP_OUTBUF) {
-		if (unlikely(!dump_tmpoutfile(control, fd_out)))
+	if (unlikely(runzip_fd(control, fd_in, fd_hist, expected_size) < 0)) {
+		clear_rulist(control);
 		return false;
 	}
 
+	/* We can now safely delete sinfo and pthread data of all threads
+	 * created. */
+	clear_rulist(control);
+
 	/* if we get here, no fatal_return(( errors during decompression */
 	print_progress("\r");
 	if (!(STDOUT | TEST_ONLY))
-		print_progress("Output filename is: %s: ", control->outfile);
+		print_output("Output filename is: %s: ", control->outfile);
 	if (!expected_size)
 		expected_size = control->st_size;
 	if (!ENCRYPT)
-		print_progress("[OK] - %lld bytes                                \n", expected_size);
+		print_output("[OK] - %lld bytes                                \n", expected_size);
 	else
-		print_progress("[OK]                                             \n");
+		print_output("[OK]                                             \n");
+
+	if (TMP_OUTBUF)
+		close_tmpoutbuf(control);
 
 	if (fd_out > 0) {
 		if (unlikely(close(fd_hist) || close(fd_out)))
@@ -850,7 +979,9 @@ bool decompress_file(rzip_control *control)
 	if (unlikely(!STDIN && !STDOUT && !TEST_ONLY && !preserve_times(control, fd_in)))
 		return false;
 
+	if ( ! IS_FROM_FILE ) {
 		close(fd_in);
+	}
 
 	if (!KEEP_FILES && !STDIN) {
 		if (unlikely(unlink(control->infile)))
@@ -860,7 +991,7 @@ bool decompress_file(rzip_control *control)
 	if (ENCRYPT)
 		release_hashes(control);
 
-	free(control->outfile);
+	dealloc(control->outfile);
 	return true;
 }
 
@@ -924,13 +1055,14 @@ static double percentage(i64 num, i64 den)
 
 bool get_fileinfo(rzip_control *control)
 {
-	i64 u_len, c_len, last_head, utotal = 0, ctotal = 0, ofs = 25, stream_head[2];
+	i64 u_len, c_len, second_last, last_head, utotal = 0, ctotal = 0, ofs = 25, stream_head[2];
 	i64 expected_size, infile_size, chunk_size = 0, chunk_total = 0;
 	int header_length, stream = 0, chunk = 0;
 	char *tmp, *infilecopy = NULL;
 	char chunk_byte = 0;
 	long double cratio;
 	uchar ctype = 0;
+	uchar save_ctype = 255;
 	struct stat st;
 	int fd_in;
 
@@ -947,7 +1079,9 @@ bool get_fileinfo(rzip_control *control)
 			infilecopy = strdupa(control->infile);
 	}
 
-	if (STDIN)
+	if ( IS_FROM_FILE )
+		fd_in = fileno(control->inFILE);
+	else if (STDIN)
 		fd_in = 0;
 	else {
 		fd_in = open(infilecopy, O_RDONLY);
@@ -966,19 +1100,24 @@ bool get_fileinfo(rzip_control *control)
 
 	if (ENCRYPT) {
 		print_output("Encrypted lrzip archive. No further information available\n");
-		if (!STDIN) close(fd_in);
+		if (!STDIN && !IS_FROM_FILE)
+			close(fd_in);
 		goto out;
 	}
 
 	if (control->major_version == 0 && control->minor_version > 4) {
 		if (unlikely(read(fd_in, &chunk_byte, 1) != 1))
 			fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error);
+		if (unlikely(chunk_byte < 1 || chunk_byte > 8))
+			fatal_goto(("Invalid chunk bytes %d\n", chunk_byte), error);
 		if (control->major_version == 0 && control->minor_version > 5) {
 			if (unlikely(read(fd_in, &control->eof, 1) != 1))
 				fatal_goto(("Failed to read eof in get_fileinfo\n"), error);
 			if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte))
 				fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error);
 			chunk_size = le64toh(chunk_size);
+			if (unlikely(chunk_size < 0))
+				fatal_goto(("Invalid chunk size %lld\n", chunk_size), error);
 		}
 	}
 
@@ -1003,30 +1142,38 @@ next_chunk:
 	stream_head[0] = 0;
 	stream_head[1] = stream_head[0] + header_length;
 
-	print_verbose("Rzip chunk %d:\n", ++chunk);
+	print_verbose("Rzip chunk:       %d\n", ++chunk);
 	if (chunk_byte)
 		print_verbose("Chunk byte width: %d\n", chunk_byte);
 	if (chunk_size) {
 		chunk_total += chunk_size;
-		print_verbose("Chunk size: %lld\n", chunk_size);
+		print_verbose("Chunk size:       %"PRId64"\n", chunk_size);
 	}
+	if (unlikely(chunk_byte && (chunk_byte > 8 || chunk_size < 0)))
+		failure("Invalid chunk data\n");
 	while (stream < NUM_STREAMS) {
 		int block = 1;
 
+		second_last = 0;
 		if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET) == -1))
 			fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error);
 		if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte)))
 			return false;
 
 		print_verbose("Stream: %d\n", stream);
-		print_maxverbose("Offset: %lld\n", ofs);
-		print_verbose("Block\tComp\tPercent\tSize\n");
+		print_maxverbose("Offset: %"PRId64"\n", stream_head[stream] + ofs);
+		print_verbose("%s\t%s\t%s\t%16s / %14s", "Block","Comp","Percent","Comp Size", "UComp Size");
+		print_maxverbose("%18s : %14s", "Offset", "Head");
+		print_verbose("\n");
 		do {
 			i64 head_off;
 
+			if (unlikely(last_head && last_head <= second_last))
+				failure_goto(("Invalid earlier last_head position, corrupt archive.\n"), error);
+			second_last = last_head;
 			if (unlikely(last_head + ofs > infile_size))
 				failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error);
-			if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET) == -1))
+			if (unlikely((head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1))
 				fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error);
 			if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len,
 					&last_head, chunk_byte)))
@@ -1048,10 +1195,15 @@ next_chunk:
 				print_verbose("zpaq");
 			else
 				print_verbose("Dunno wtf");
+			if (save_ctype == 255)
+				save_ctype = ctype; /* need this for lzma when some chunks could have no compression
+						     * and info will show rzip + none on info display if last chunk
+						     * is not compressed. Adjust for all types in case it's used in
+						     * the future */
 			utotal += u_len;
 			ctotal += c_len;
-			print_verbose("\t%.1f%%\t%lld / %lld", percentage(c_len, u_len), c_len, u_len);
-			print_maxverbose("\tOffset: %lld\tHead: %lld", head_off, last_head);
+			print_verbose("\t%5.1f%%\t%16"PRId64" / %14"PRId64"", percentage(c_len, u_len), c_len, u_len);
+			print_maxverbose("%18"PRId64" : %14"PRId64"", head_off, last_head);
 			print_verbose("\n");
 			block++;
 		} while (last_head);
@@ -1067,6 +1219,8 @@ next_chunk:
 	if (control->major_version == 0 && control->minor_version > 4) {
 		if (unlikely(read(fd_in, &chunk_byte, 1) != 1))
 			fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error);
+		if (unlikely(chunk_byte < 1 || chunk_byte > 8))
+			fatal_goto(("Invalid chunk bytes %d\n", chunk_byte), error);
 		ofs++;
 		if (control->major_version == 0 && control->minor_version > 5) {
 			if (unlikely(read(fd_in, &control->eof, 1) != 1))
@@ -1074,70 +1228,93 @@ next_chunk:
 			if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte))
 				fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error);
 			chunk_size = le64toh(chunk_size);
+			if (unlikely(chunk_size < 0))
+				fatal_goto(("Invalid chunk size %lld\n", chunk_size), error);
 			ofs += 1 + chunk_byte;
 			header_length = 1 + (chunk_byte * 3);
 		}
 	}
 	goto next_chunk;
 done:
+	cratio = (long double)expected_size / (long double)infile_size;
 	if (unlikely(ofs > infile_size))
 		failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error);
-	print_verbose("Rzip compression: %.1f%% %lld / %lld\n",
+	print_output("\nSummary\n=======\n");
+	print_output("File: %s\nlrzip version: %d.%d \n\n", infilecopy,
+				control->major_version, control->minor_version);
+
+	if (!expected_size)
+		print_output("Due to %s, expected decompression size not available\n", "Compression to STDOUT");
+	print_verbose("  Stats         Percent       Compressed /   Uncompressed\n  -------------------------------------------------------\n");
+		/* If we can't show expected size, tailor output for it */
+	if (expected_size) {
+		print_verbose("  Rzip:         %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
 					percentage (utotal, expected_size),
 					utotal, expected_size);
-	print_verbose("Back end compression: %.1f%% %lld / %lld\n",
+		print_verbose("  Back end:     %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
 					percentage(ctotal, utotal),
 					ctotal, utotal);
-	print_verbose("Overall compression: %.1f%% %lld / %lld\n",
+		print_verbose("  Overall:      %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
 					percentage(ctotal, expected_size),
 					ctotal, expected_size);
+	} else {
+		print_verbose("  Rzip:         Unavailable\n");
+		print_verbose("  Back end:     %5.1f%%\t%16"PRId64" / %14"PRId64"\n", percentage(ctotal, utotal), ctotal, utotal);
+		print_verbose("  Overall:      Unavailable\n");
+	}
+	print_verbose("\n");
 
-	cratio = (long double)expected_size / (long double)infile_size;
+	print_output("  Compression Method: ");
 
-	print_output("%s:\nlrzip version: %d.%d file\n", infilecopy, control->major_version, control->minor_version);
-
-	print_output("Compression: ");
-	if (ctype == CTYPE_NONE)
+	if (save_ctype == CTYPE_NONE)
 		print_output("rzip alone\n");
-	else if (ctype == CTYPE_BZIP2)
+	else if (save_ctype == CTYPE_BZIP2)
 		print_output("rzip + bzip2\n");
-	else if (ctype == CTYPE_LZO)
+	else if (save_ctype == CTYPE_LZO)
 		print_output("rzip + lzo\n");
-	else if (ctype == CTYPE_LZMA)
+	else if (save_ctype == CTYPE_LZMA)
 		print_output("rzip + lzma\n");
-	else if (ctype == CTYPE_GZIP)
+	else if (save_ctype == CTYPE_GZIP)
 		print_output("rzip + gzip\n");
-	else if (ctype == CTYPE_ZPAQ)
+	else if (save_ctype == CTYPE_ZPAQ)
 		print_output("rzip + zpaq\n");
 	else
 		print_output("Dunno wtf\n");
-	print_output("Decompressed file size: %llu\n", expected_size);
-	print_output("Compressed file size: %llu\n", infile_size);
-	print_output("Compression ratio: %.3Lf\n", cratio);
 
+	print_output("\n");
+
+	if (expected_size) {
+		print_output("  Decompressed file size: %14"PRIu64"\n", expected_size);
+		print_output("  Compressed file size:   %14"PRIu64"\n", infile_size);
+		print_output("  Compression ratio:      %14.3Lfx\n", cratio);
+	} else {
+		print_output("  Decompressed file size:    Unavailable\n");
+		print_output("  Compressed file size:   %14"PRIu64"\n", infile_size);
+		print_output("  Compression ratio:         Unavailable\n");
+	}
 	if (HAS_MD5) {
 		char md5_stored[MD5_DIGEST_SIZE];
 		int i;
 
-		print_output("MD5 used for integrity testing\n");
 		if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END) == -1))
 			fatal_goto(("Failed to seek to md5 data in runzip_fd\n"), error);
 		if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
 			fatal_goto(("Failed to read md5 data in runzip_fd\n"), error);
-		print_output("MD5: ");
+		print_output("\n  MD5 Checksum: ");
 		for (i = 0; i < MD5_DIGEST_SIZE; i++)
 			print_output("%02x", md5_stored[i] & 0xFF);
 		print_output("\n");
 	} else
-		print_output("CRC32 used for integrity testing\n");
+		print_output("\n  CRC32 used for integrity testing\n");
+	if ( !IS_FROM_FILE )
 		if (unlikely(close(fd_in)))
 			fatal_return(("Failed to close fd_in in get_fileinfo\n"), false);
 
 out:
-	free(control->outfile);
+	dealloc(control->outfile);
 	return true;
 error:
-	if (!STDIN) close(fd_in);
+	if (!STDIN && ! IS_FROM_FILE) close(fd_in);
 	return false;
 }
 
@@ -1152,14 +1329,15 @@ bool compress_file(rzip_control *control)
 	int fd_in = -1, fd_out = -1;
 	char header[MAGIC_LEN];
 
-	if (MD5_RELIABLE)
 	control->flags |= FLAG_MD5;
 	if (ENCRYPT)
 		if (unlikely(!get_hash(control, 1)))
 			return false;
 	memset(header, 0, sizeof(header));
 
-	if (!STDIN) {
+	if ( IS_FROM_FILE )
+		fd_in = fileno(control->inFILE);
+	else if (!STDIN) {
 		 /* is extension at end of infile? */
 		if ((tmp = strrchr(control->infile, '.')) && !strcmp(tmp, control->suffix)) {
 			print_err("%s: already has %s suffix. Skipping...\n", control->infile, control->suffix);
@@ -1169,7 +1347,8 @@ bool compress_file(rzip_control *control)
         fd_in = open(control->infile, O_RDONLY);
 		if (unlikely(fd_in == -1))
 			fatal_return(("Failed to open %s\n", control->infile), false);
-	} else
+	} 
+	else
 		fd_in = 0;
 
 	if (!STDOUT) {
@@ -1206,7 +1385,7 @@ bool compress_file(rzip_control *control)
 			} else
 				strcpy(control->outfile, tmpinfile);
 			strcat(control->outfile, control->suffix);
-			print_progress("Output filename is: %s\n", control->outfile);
+			print_output("Output filename is: %s\n", control->outfile);
 		}
 
 		fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666);
@@ -1227,6 +1406,12 @@ bool compress_file(rzip_control *control)
 				goto error;
 		}
 	} else {
+		control->fd_out = fd_out = open_tmpoutfile(control);
+		if (likely(fd_out != -1)) {
+			/* Unlink temporary file as soon as possible */
+			if (unlikely(unlink(control->outfile)))
+				fatal_return(("Failed to unlink tmpfile: %s\n", control->outfile), false);
+		}
 		if (unlikely(!open_tmpoutbuf(control)))
 			goto error;
 	}
@@ -1237,7 +1422,7 @@ bool compress_file(rzip_control *control)
 
 	rzip_fd(control, fd_in, fd_out);
 
-	/* Wwrite magic at end b/c lzma does not tell us properties until it is done */
+	/* Write magic at end b/c lzma does not tell us properties until it is done */
 	if (!STDOUT) {
 		if (unlikely(!write_magic(control)))
 			goto error;
@@ -1246,6 +1431,11 @@ bool compress_file(rzip_control *control)
 	if (ENCRYPT)
 		release_hashes(control);
 
+	if (unlikely(!STDIN && !STDOUT && !preserve_times(control, fd_in))) {
+		fatal("Failed to preserve times on output file\n");
+		goto error;
+	}
+
 	if (unlikely(close(fd_in))) {
 		fatal("Failed to close fd_in\n");
 		fd_in = -1;
@@ -1261,25 +1451,28 @@ bool compress_file(rzip_control *control)
 			fatal_return(("Failed to unlink %s\n", control->infile), false);
 	}
 
-	free(control->outfile);
+	dealloc(control->outfile);
 	return true;
 error:
-	if (STDIN && (fd_in > 0)) close(fd_in);
-	if ((!STDOUT) && (fd_out > 0)) close(fd_out);
+	if (! IS_FROM_FILE && STDIN && (fd_in > 0))
+		close(fd_in);
+	if ((!STDOUT) && (fd_out > 0))
+		close(fd_out);
 	return false;
 }
 
 bool initialise_control(rzip_control *control)
 {
-	struct timeval tv;
-	char *eptr; /* for environment */
+	time_t now_t, tdiff;
+	char localeptr[] = "./", *eptr; /* for environment */
+	size_t len;
 
 	memset(control, 0, sizeof(rzip_control));
 	control->msgout = stderr;
 	control->msgerr = stderr;
 	register_outputfile(control, control->msgout);
 	control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD;
-	control->suffix = strdup(".lrz");
+	control->suffix = ".lrz";
 	control->compression_level = 7;
 	control->ramsize = get_ram(control);
 	if (unlikely(control->ramsize == -1))
@@ -1292,9 +1485,17 @@ bool initialise_control(rzip_control *control)
 	/* The first 5 bytes of the salt is the time in seconds.
 	 * The next 2 bytes encode how many times to hash the password.
 	 * The last 9 bytes are random data, making 16 bytes of salt */
-	if (unlikely(gettimeofday(&tv, NULL)))
-		fatal_return(("Failed to gettimeofday in main\n"), false);
-	control->secs = tv.tv_sec;
+	if (unlikely((now_t = time(NULL)) == ((time_t)-1)))
+		fatal_return(("Failed to call time in main\n"), false);
+	if (unlikely(now_t < T_ZERO)) {
+		print_output("Warning your time reads before the year 2011, check your system clock\n");
+		now_t = T_ZERO;
+	}
+	/* Workaround for CPUs no longer keeping up with Moore's law!
+	 * This way we keep the magic header format unchanged. */
+	tdiff = (now_t - T_ZERO) / 4;
+	now_t = T_ZERO + tdiff;
+	control->secs = now_t;
 	control->encloops = nloops(control->secs, control->salt, control->salt + 1);
 	if (unlikely(!get_rand(control, control->salt + 2, 6)))
 		return false;
@@ -1307,13 +1508,9 @@ bool initialise_control(rzip_control *control)
 		eptr = getenv("TEMPDIR");
 	if (!eptr)
 		eptr = getenv("TEMP");
-	if (!eptr) {
-		eptr = malloc(3);
-		if ( eptr == NULL )
-			fatal_return(("Failed to allocate for eptr\n"), false);
-		strcpy(eptr,"./");
-	}
-	size_t len = strlen(eptr);
+	if (!eptr)
+		eptr = localeptr;
+	len = strlen(eptr);
 
 	control->tmpdir = malloc(len + 2);
 	if (control->tmpdir == NULL)

lrzip.pc.in

@@ -1,10 +0,0 @@
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: lrzip
-Description: lrzip compression library
-Version: @VERSION@
-Libs: -L${libdir} -llrzip
-Libs.private: @LIBS@

lrzip_core.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -27,7 +27,7 @@ bool write_magic(rzip_control *control);
 bool read_magic(rzip_control *control, int fd_in, i64 *expected_size);
 bool preserve_perms(rzip_control *control, int fd_in, int fd_out);
 int open_tmpoutfile(rzip_control *control);
-bool dump_tmpoutfile(rzip_control *control, int fd_out);
+bool flush_tmpout(rzip_control *control);
 int open_tmpinfile(rzip_control *control);
 bool read_tmpinfile(rzip_control *control, int fd_in);
 bool decompress_file(rzip_control *control);
@@ -36,7 +36,6 @@ bool get_fileinfo(rzip_control *control);
 bool compress_file(rzip_control *control);
 bool write_fdout(rzip_control *control, void *buf, i64 len);
 bool write_fdin(rzip_control *control);
-bool flush_tmpoutbuf(rzip_control *control);
 void close_tmpoutbuf(rzip_control *control);
 void clear_tmpinbuf(rzip_control *control);
 bool clear_tmpinfile(rzip_control *control);
@@ -47,4 +46,5 @@ extern void zpaq_compress(uchar *c_buf, i64 *c_len, uchar *s_buf, i64 s_len, int
 			  FILE *msgout, bool progress, long thread);
 extern void zpaq_decompress(uchar *s_buf, i64 *d_len, uchar *c_buf, i64 c_len,
 			    FILE *msgout, bool progress, long thread);
+
 #endif

lrzip_private.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -144,7 +144,7 @@ extern int errno;
 #define unlikely(x)	__builtin_expect(!!(x), 0)
 #define __maybe_unused	__attribute__((unused))
 
-#if defined(__MINGW32__) || defined(__CYGWIN__) || defined(ANDROID) || defined(__APPLE__)
+#if defined(__MINGW32__) || defined(__CYGWIN__) || defined(__ANDROID__) || defined(__APPLE__) || defined(__OpenBSD__)
 # define ffsll __builtin_ffsll
 #endif
 
@@ -170,12 +170,6 @@ typedef sem_t cksem_t;
  #define mremap fake_mremap
 #endif
 
-#if defined(__APPLE__)
-# define MD5_RELIABLE (0)
-#else
-# define MD5_RELIABLE (1)
-#endif
-
 #define bswap_32(x) \
      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) |		      \
       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
@@ -235,6 +229,7 @@ typedef sem_t cksem_t;
 #define FLAG_TMP_OUTBUF		(1 << 21)
 #define FLAG_TMP_INBUF		(1 << 22)
 #define FLAG_ENCRYPT		(1 << 23)
+#define FLAG_OUTPUT		(1 << 24)
 
 #define NO_MD5		(!(HASH_CHECK) && !(HAS_MD5))
 
@@ -260,11 +255,18 @@ typedef sem_t cksem_t;
 # define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
 #endif
 
+#ifndef PAGE_SIZE
 # ifdef _SC_PAGE_SIZE
 #  define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
 # else
 #  define PAGE_SIZE (4096)
 # endif
+#endif
+
+#define dealloc(ptr) do { \
+	free(ptr); \
+	ptr = NULL; \
+} while (0)
 
 /* Determine how many times to hash the password when encrypting, based on
  * the date such that we increase the number of loops according to Moore's
@@ -303,10 +305,13 @@ typedef sem_t cksem_t;
 #define HAS_MD5		(control->flags & FLAG_MD5)
 #define CHECK_FILE	(control->flags & FLAG_CHECK)
 #define KEEP_BROKEN	(control->flags & FLAG_KEEP_BROKEN)
-#define LZO_TEST	(control->flags & FLAG_THRESHOLD)
+#define LZ4_TEST	(control->flags & FLAG_THRESHOLD)
 #define TMP_OUTBUF	(control->flags & FLAG_TMP_OUTBUF)
 #define TMP_INBUF	(control->flags & FLAG_TMP_INBUF)
 #define ENCRYPT		(control->flags & FLAG_ENCRYPT)
+#define SHOW_OUTPUT	(control->flags & FLAG_OUTPUT)
+
+#define IS_FROM_FILE ( !!(control->inFILE) && !STDIN )
 
 
 /* Structure to save state of computation between the single steps.  */
@@ -344,8 +349,21 @@ struct checksum {
 
 typedef i64 tag;
 
+struct node {
+	void *data;
+	struct node *prev;
+};
+
+struct runzip_node {
+	struct stream_info *sinfo;
+	pthread_t *pthreads;
+	struct runzip_node *prev;
+};
+
 struct rzip_state {
 	void *ss;
+	struct node *sslist;
+	struct node *head;
 	struct level *level;
 	tag hash_index[256];
 	struct hash_entry *hash_table;
@@ -405,6 +423,7 @@ struct rzip_control {
 	i64 max_mmap;
 	int threads;
 	char nice_val;		// added for consistency
+	int current_priority;
 	char major_version;
 	char minor_version;
 	i64 st_size;
@@ -420,6 +439,7 @@ struct rzip_control {
 	uchar *salt_pass;
 	int salt_pass_len;
 	uchar *hash;
+	char *passphrase;
 
 	pthread_mutex_t control_lock;
 	unsigned char eof;
@@ -435,10 +455,6 @@ struct rzip_control {
 	const char *util_infile;
 	char delete_infile;
 	const char *util_outfile;
-#define STREAM_BUCKET_SIZE 20
-	size_t sinfo_buckets;
-	size_t sinfo_idx;
-	struct stream_info **sinfo_queue;
 	char delete_outfile;
 	FILE *outputfile;
 	char library_mode;
@@ -454,6 +470,18 @@ struct rzip_control {
 	void (*next_tag)(rzip_control *, struct rzip_state *, i64, tag *);
 	tag (*full_tag)(rzip_control *, struct rzip_state *, i64);
 	i64 (*match_len)(rzip_control *, struct rzip_state *, i64, i64, i64, i64 *);
+
+	pthread_t *pthreads;
+	struct runzip_node *ruhead;
+};
+
+struct uncomp_thread {
+	uchar *s_buf;
+	i64 u_len, c_len;
+	i64 last_head;
+	uchar c_type;
+	int busy;
+	int streamno;
 };
 
 struct stream {
@@ -479,6 +507,7 @@ struct stream_info {
 	i64 total_read;
 	i64 ram_alloced;
 	i64 size;
+	struct uncomp_thread *ucthreads;
 	long thread_no;
 	long next_thread;
 	int chunks;
@@ -520,6 +549,7 @@ static inline void print_err(const rzip_control *control, unsigned int line, con
 } while (0)
 
 #define print_output(...)	do {\
+	if (SHOW_OUTPUT)	\
 	print_stuff(1, __VA_ARGS__); \
 } while (0)
 

lrztar

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Copyright (C) George Makrydakis 2009-2011,2013
-# Copyright (C) Con Kolivas 2011-2012,2016
+# Copyright (C) Con Kolivas 2011-2012,2016,2018,2021
 
 # A bash wrapper for Con Kolivas' excellent lrzip utility. For the time
 # being, lrzip does not like pipes, so we had to do this. It is kind of
@@ -24,7 +24,7 @@ function lrztar_local() {
     local hv="\
 lrztar GNU/bash wrapper script for lrzip and tar input/output over directories.
 Copyright (C) George Makrydakis 2009-2011,2013
-Copyright (C) Con Kolivas       2011,2012
+Copyright (C) Con Kolivas       2011-2012,2016,2018,2021
 
 Usage : lrztar [lrzip options] <directory>
 Result: a lrzip tarball is produced.
@@ -43,16 +43,20 @@ Notice:
  - This script exists because of how lrzip behaves.
  - Beware the -f flag, it stands for what it says...
 "
+    [[ $1 == "" ]] && {
+        printf "lrztar: no arguments given\n";
+        return 1;
+    }
     local p=("${@:1:$(($#-1))}") s="${!#}" vopt=("lrz") \
           v_w=0 v_S=0 v_D=0 v_p=0 v_q=0 v_L=0 \
           v_n=0 v_l=0 v_b=0 v_g=0 v_z=0 v_U=0 \
           v_T=0 v_N=0 v_v=0 v_f=0 v_d=0 v_h=0 \
           v_H=0 v_c=0 v_k=0 v_o=0 v_O=0 v_m=0 x= i="$(pwd)"
-    which tar   &> /dev/null \
+    tar --version &> /dev/null \
         || { printf "lrztar: no tar in your path\n"; return 1; }
-    which lrzip &> /dev/null \
+    lrzip --version &> /dev/null \
         || { printf "lrztar: no lrzip in your path\n"; return 1; }
-    which lrzcat &> /dev/null \
+    lrzcat --version &> /dev/null \
         || { printf "lrztar: no lrzcat in your path\n"; return 1; }
     while getopts w:O:S:DqL:nlbgzUm:TN:p:vfo:d:tVhHck x; do
         [[ $x == [tV] ]] && {
@@ -133,11 +137,13 @@ Notice:
             s="${s%/}"
             p+=(-o "$i/${s##*/}.tar.${vopt[v_S]}");
         fi
+        if ! ((v_o)); then 
             ! ((v_f)) && [[ -e $i/${s##*/}.tar.${vopt[v_S]} ]] && {
                 printf "lrztar: %s exists, use -f to overwrite\n" \
                     "$i/${s##*/}.tar.${vopt[v_S]}"
                 return 1
             }
+        fi
         tar c "$s" | lrzip "${p[@]}"
         x=$?
     }

lzma/ASM/x86/7zAsm.asm

@@ -0,0 +1,100 @@
+; 7zAsm.asm -- ASM macros
+; 2009-12-12 : Igor Pavlov : Public domain
+; 2011-10-12 : P7ZIP       : Public domain
+
+%define NOT ~
+
+%macro MY_ASM_START 0
+  SECTION .text
+%endmacro
+
+%macro MY_PROC 2 ; macro name:req, numParams:req
+  align 16
+  %define proc_numParams %2 ; numParams
+    global %1
+    global _%1
+    %1:
+    _%1:
+%endmacro
+
+%macro  MY_ENDP 0
+  %ifdef x64
+    ret
+    ; proc_name ENDP
+  %else
+    ret ; (proc_numParams - 2) * 4
+  %endif
+%endmacro
+
+%ifdef x64
+  REG_SIZE equ 8
+%else
+  REG_SIZE equ 4
+%endif
+
+  %define x0  EAX
+  %define x1  ECX
+  %define x2  EDX
+  %define x3  EBX
+  %define x4  ESP
+  %define x5  EBP
+  %define x6  ESI
+  %define x7  EDI
+
+  %define x0_L  AL
+  %define x1_L  CL
+  %define x2_L  DL
+  %define x3_L  BL
+
+  %define x0_H  AH
+  %define x1_H  CH
+  %define x2_H  DH
+  %define x3_H  BH
+
+%ifdef x64
+  %define r0  RAX
+  %define r1  RCX
+  %define r2  RDX
+  %define r3  RBX
+  %define r4  RSP
+  %define r5  RBP
+  %define r6  RSI
+  %define r7  RDI
+%else
+  %define r0  x0
+  %define r1  x1
+  %define r2  x2
+  %define r3  x3
+  %define r4  x4
+  %define r5  x5
+  %define r6  x6
+ %define  r7  x7
+%endif
+
+%macro MY_PUSH_4_REGS 0
+    push    r3
+    push    r5
+%ifdef x64
+  %ifdef CYGWIN64
+    push    r6
+    push    r7
+  %endif
+%else
+    push    r6
+    push    r7
+%endif
+%endmacro
+
+%macro MY_POP_4_REGS 0
+%ifdef x64
+  %ifdef CYGWIN64
+    pop     r7
+    pop     r6
+  %endif
+%else
+    pop     r7
+    pop     r6
+%endif
+    pop     r5
+    pop     r3
+%endmacro

lzma/ASM/x86/7zCrcOpt_asm.asm

@@ -0,0 +1,147 @@
+; 7zCrcOpt.asm -- CRC32 calculation : optimized version
+; 2009-12-12 : Igor Pavlov : Public domain
+
+%include "7zAsm.asm"
+
+MY_ASM_START
+
+%define rD    r2
+%define rN     r7
+
+%ifdef x64
+    %define num_VAR      r8
+    %define table_VAR    r9
+%else
+    data_size   equ (REG_SIZE * 7)
+    crc_table   equ (REG_SIZE + data_size)
+    %define num_VAR     [r4 + data_size]
+    %define table_VAR   [r4 + crc_table]
+%endif
+
+%define SRCDAT  rN + rD + 4 *
+
+%macro CRC 4  ;CRC macro op:req, dest:req, src:req, t:req
+    %1      %2, DWORD [r5 + %3 * 4 + 0400h * %4]  ; op      dest, DWORD [r5 + src * 4 + 0400h * t]
+%endmacro
+
+%macro CRC_XOR 3  ; CRC_XOR macro dest:req, src:req, t:req
+    CRC xor, %1, %2, %3
+%endmacro
+
+%macro CRC_MOV 3  ; CRC_MOV macro dest:req, src:req, t:req
+    CRC mov, %1, %2, %3   ; CRC mov, dest, src, t
+%endmacro
+
+%macro CRC1b 0
+    movzx   x6, BYTE [rD]
+    inc     rD
+    movzx   x3, x0_L
+    xor     x6, x3
+    shr     x0, 8
+    CRC     xor, x0, r6, 0
+    dec     rN
+%endmacro
+
+%macro  MY_PROLOG 1 ; MY_PROLOG macro crc_end:req
+    MY_PUSH_4_REGS
+
+
+%ifdef x64
+  %ifdef CYGWIN64
+    ;ECX=CRC, RDX=buf, R8=size R9=table
+    ; already in R8  : mov num_VAR,R8 ; LEN
+    ; already in RDX : mov rD, RDX ; BUF
+    ; already in R9  : mov table_VAR,R9; table
+    mov x0, ECX ; CRC
+  %else
+    ;EDI=CRC, RSI=buf, RDX=size RCX=table
+    mov num_VAR,RDX ; LEN
+    mov rD, RSI ; BUF
+    mov table_VAR,RCX; table
+    mov x0, EDI ; CRC
+  %endif
+%else
+    mov     x0, [r4 + 20]  ; CRC
+    mov     rD, [r4 + 24]  ; buf
+%endif
+    mov     rN, num_VAR
+    mov     r5, table_VAR
+    test    rN, rN
+    jz   near   %1 ; crc_end
+  %%sl:
+    test    rD, 7
+    jz     %%sl_end
+    CRC1b
+    jnz     %%sl
+  %%sl_end:
+    cmp     rN, 16
+    jb   near   %1; crc_end
+    add     rN, rD
+    mov     num_VAR, rN
+    sub     rN, 8
+    and     rN, NOT 7
+    sub     rD, rN
+    xor     x0, [SRCDAT 0]
+%endmacro
+
+%macro MY_EPILOG 1  ; MY_EPILOG macro crc_end:req
+    xor     x0, [SRCDAT 0]
+    mov     rD, rN
+    mov     rN, num_VAR
+    sub     rN, rD
+  %1:  ; crc_end:
+    test    rN, rN
+    jz      %%end ; @F
+    CRC1b
+    jmp     %1 ; crc_end
+  %%end:
+    MY_POP_4_REGS
+%endmacro
+
+MY_PROC CrcUpdateT8, 4
+    MY_PROLOG crc_end_8
+    mov     x1, [SRCDAT 1]
+    align 16
+  main_loop_8:
+    mov     x6, [SRCDAT 2]
+    movzx   x3, x1_L
+    CRC_XOR x6, r3, 3
+    movzx   x3, x1_H
+    CRC_XOR x6, r3, 2
+    shr     x1, 16
+    movzx   x3, x1_L
+    movzx   x1, x1_H
+    CRC_XOR x6, r3, 1
+    movzx   x3, x0_L
+    CRC_XOR x6, r1, 0
+
+    mov     x1, [SRCDAT 3]
+    CRC_XOR x6, r3, 7
+    movzx   x3, x0_H
+    shr     x0, 16
+    CRC_XOR x6, r3, 6
+    movzx   x3, x0_L
+    CRC_XOR x6, r3, 5
+    movzx   x3, x0_H
+    CRC_MOV x0, r3, 4
+    xor     x0, x6
+    add     rD, 8
+    jnz     main_loop_8
+
+    MY_EPILOG crc_end_8
+MY_ENDP
+
+; T4 CRC deleted
+
+; end
+
+%ifidn __OUTPUT_FORMAT__,elf
+section .note.GNU-stack noalloc noexec nowrite progbits
+%endif
+%ifidn __OUTPUT_FORMAT__,elf32
+section .note.GNU-stack noalloc noexec nowrite progbits
+%endif
+%ifidn __OUTPUT_FORMAT__,elf64
+section .note.GNU-stack noalloc noexec nowrite progbits
+%endif
+

lzma/ASM/x86/7zCrcT8U.s

@@ -1,102 +0,0 @@
-
-SECTION .text
-
-%macro CRC1b 0
-    movzx EDX, BYTE [ESI]
-    inc ESI
-    movzx EBX, AL
-    xor EDX, EBX 
-    shr EAX, 8
-    xor EAX, [EBP + EDX * 4]
-    dec EDI
-%endmacro
-
-data_size equ (28)
-crc_table equ (data_size + 4)
-
-	align 16
-	global CrcUpdateT8
-	global _CrcUpdateT8
-CrcUpdateT8:
-_CrcUpdateT8:
-    push EBX
-    push ESI
-    push EDI
-    push EBP
-
-    mov	EAX, [ESP + 20]        
-    mov	ESI, [ESP + 24]        
-    mov	EDI, [ESP + data_size] 
-    mov EBP, [ESP + crc_table] 
-
-    test EDI, EDI
-    jz sl_end
-  sl:
-    test ESI, 7
-    jz sl_end
-    CRC1b
-    jnz sl
-  sl_end:
-
-    cmp EDI, 16
-    jb NEAR crc_end
-    mov [ESP + data_size], EDI
-    sub EDI, 8
-    and EDI, ~ 7
-    sub [ESP + data_size], EDI
-
-    add EDI, ESI
-    xor EAX, [ESI]
-    mov EBX, [ESI + 4]
-    movzx ECX, BL
-    align 16
-  main_loop:
-    mov EDX, [EBP + ECX*4 + 0C00h]
-    movzx ECX, BH
-    xor EDX, [EBP + ECX*4 + 0800h]
-    shr EBX, 16
-    movzx ECX, BL
-    xor EDX, [EBP + ECX*4 + 0400h]
-    xor EDX, [ESI + 8]
-    movzx ECX, AL
-    movzx EBX, BH
-    xor EDX, [EBP + EBX*4 + 0000h]
-
-    mov EBX, [ESI + 12]
-
-    xor EDX, [EBP + ECX*4 + 01C00h]
-    movzx ECX, AH
-    add ESI, 8
-    shr EAX, 16
-    xor EDX, [EBP + ECX*4 + 01800h]
-    movzx ECX, AL
-    xor EDX, [EBP + ECX*4 + 01400h]
-    movzx ECX, AH
-    mov EAX, [EBP + ECX*4 + 01000h]
-    movzx ECX, BL
-    xor EAX,EDX
-
-    cmp ESI, EDI
-    jne	main_loop
-    xor	EAX, [ESI]
-
-    mov EDI, [ESP + data_size]
-
-  crc_end:
-
-    test EDI, EDI
-    jz fl_end
-  fl:
-    CRC1b
-    jnz fl
-  fl_end:
-
-    pop EBP
-    pop EDI
-    pop ESI
-    pop EBX
-    ret 
-
-%ifidn __OUTPUT_FORMAT__,elf
-    section .note.GNU-stack noalloc noexec nowrite progbits
-%endif

lzma/ASM/x86/Makefile.am

@@ -0,0 +1,7 @@
+MAINTAINERCLEANFILES = Makefile.in
+
+noinst_LTLIBRARIES = liblzmaasm.la
+
+liblzmaasm_la_SOURCES = \
+	7zAsm.asm \
+	7zCrcOpt_asm.asm

lzma/ASM/x86_64/7zCrcT8U_64.s

@@ -1,105 +0,0 @@
-
-SECTION .text
-
-
-
-
-%macro CRC1b 0
-    movzx EDX, BYTE [RSI]
-    inc RSI
-    movzx EBX, AL
-    xor EDX, EBX
-    shr EAX, 8
-    xor EAX, [RDI + RDX * 4]
-    dec R8
-%endmacro
-
-
-
-
-align 16
-global CrcUpdateT8
-
-CrcUpdateT8:
-
-    push RBX
-    push RSI
-    push RDI
-    push RBP
-
-    mov EAX, ECX
-    mov RSI, RDX
-    mov RDI, R9
-
-
-    test R8, R8
-    jz sl_end
-  sl:
-    test RSI, 7
-    jz sl_end
-    CRC1b
-    jnz sl
-  sl_end:
-
-    cmp R8, 16
-    jb crc_end
-    mov R9, R8
-    and R8, 7
-    add R8, 8
-    sub R9, R8
-            
-    add R9, RSI
-    xor EAX, [RSI]
-    mov EBX, [RSI + 4]
-    movzx ECX, BL
-    align 16
-  main_loop:
-    mov EDX, [RDI + RCX*4 + 0C00h]
-    movzx EBP, BH
-    xor EDX, [RDI + RBP*4 + 0800h]
-    shr EBX, 16
-    movzx ECX, BL
-    xor EDX, [RSI + 8]
-    xor EDX, [RDI + RCX*4 + 0400h]
-    movzx ECX, AL
-    movzx EBP, BH
-    xor EDX, [RDI + RBP*4 + 0000h]
-
-    mov EBX, [RSI + 12]
-
-    xor EDX, [RDI + RCX*4 + 01C00h]
-    movzx EBP, AH
-    shr EAX, 16
-    movzx ECX, AL
-    xor EDX, [RDI + RBP*4 + 01800h]
-    movzx EBP, AH
-    mov EAX, [RDI + RCX*4 + 01400h]
-    add RSI, 8
-    xor EAX, [RDI + RBP*4 + 01000h]
-    movzx ECX, BL
-    xor EAX,EDX
-
-    cmp RSI, R9
-    jne main_loop
-    xor EAX, [RSI]
-
-
-  
-  crc_end:
-
-    test R8, R8
-    jz fl_end
-  fl:
-    CRC1b
-    jnz fl
-  fl_end:
-
-    pop RBP
-    pop RDI
-    pop RSI
-    pop RBX
-    ret
-
-%ifidn __OUTPUT_FORMAT__,elf
-    section .note.GNU-stack noalloc noexec nowrite progbits
-%endif

lzma/C/Makefile.am

@@ -1,22 +1,28 @@
 MAINTAINERCLEANFILES = Makefile.in
 
+# Update -D
 AM_CFLAGS = \
-  -DCOMPRESS_MF_MT \
   -D_REENTRANT \
   -I@top_builddir@ \
   -I@top_srcdir@
 
-ASM_LIBS =
 ASM_S =
+ASM_7z =
+C_S =
 if USE_ASM
-  ASM_LIBS += @ASM_OBJ@
+  ASM_7z += 7zCrcOpt_asm
+  ASM_S += @abs_top_srcdir@/lzma/ASM/x86/$(ASM_7z).asm
+  C_S += 7zCrcT8.c
 else
-  ASM_S += 7zCrc.c 7zCrc.h
+  C_S += 7zCrc.c 
 endif
 
 noinst_LTLIBRARIES = liblzma.la
+# need separate variable for ASM so that make will compile later
+# to prevent an error even if -j## is used.
 liblzma_la_SOURCES = \
-  $(ASM_S) \
+	$(C_S) \
+	7zCrc.h \
 	LzmaDec.h \
 	LzmaEnc.h \
 	LzFind.c \
@@ -37,10 +43,22 @@ liblzma_la_SOURCES = \
 	basetyps.h \
 	MyWindows.h \
 	MyGuidDef.h
-liblzma_so_LIBS = $(ASM_LIBS)
 
-7zCrcT8U.o: @top_srcdir@/lzma/ASM/x86/7zCrcT8U.s
-	@ASM_CMD@ -o 7zCrcT8U.o @top_srcdir@/lzma/ASM/x86/7zCrcT8U.s
+## hack to force asm compilation and to trick libtool with .lo file
+if USE_ASM 
+liblzma_la_LIBADD = $(ASM_7z).lo
 
-7zCrcT8U_64.o: @top_srcdir@/lzma/ASM/x86_64/7zCrcT8U_64.s
-	@ASM_CMD@ -o 7zCrcT8U_64.o @top_srcdir@/lzma/ASM/x86_64/7zCrcT8U_64.s
+7ZIPASMLOFILE := \
+\# $(ASM_7z).lo - a libtool object file\
+\n\# Generated by libtool -- hack to allow asm linking\
+\n\# Peter Hyman\
+\npic_object='.libs/$(ASM_7z).o'\
+\nnon_pic_object='$(ASM_7z).o'\
+\n
+
+$(ASM_7z).lo: $(ASM_S)
+	$(ASM_PROG) $(ASM_OPT) -o $(ASM_7z).o $(ASM_S)
+	mkdir -p .libs
+	cp $(ASM_7z).o .libs/
+	@printf "$(7ZIPASMLOFILE)" > $(ASM_7z).lo
+endif

lzma/C/Threads.c

@@ -198,7 +198,7 @@ WRes Thread_Create(CThread *thread, THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE
 	ret = pthread_attr_setdetachstate(&attr,PTHREAD_CREATE_JOINABLE);
 	if (ret) return ret;
 
-	ret = pthread_create(&thread->_tid, &attr, (void * (*)(void *))startAddress, parameter);
+	ret = pthread_create(&thread->_tid, &attr, (void *)startAddress, parameter);
 
 	/* ret2 = */ pthread_attr_destroy(&attr);
 

lzma/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = C
+SUBDIRS = C ASM/x86
 MAINTAINERCLEANFILES = Makefile.in
 
 lzmadocdir = @docdir@/lzma

m4/ac_attribute.m4

@@ -18,8 +18,7 @@ AC_DEFUN([AC_C___ATTRIBUTE__],
 AC_MSG_CHECKING([for __attribute__])
 
 AC_CACHE_VAL([ac_cv___attribute__],
-   [AC_TRY_COMPILE(
-       [
+   [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <stdlib.h>
 
 int func(int x);
@@ -27,11 +26,8 @@ int foo(int x __attribute__ ((unused)))
 {
    exit(1);
 }
-       ],
-       [],
-       [ac_cv___attribute__="yes"],
-       [ac_cv___attribute__="no"]
-    )])
+       ]], [[]])],[ac_cv___attribute__="yes"],[ac_cv___attribute__="no"
+    ])])
 
 AC_MSG_RESULT($ac_cv___attribute__)
 

m4/efl_doxygen.m4

@@ -15,9 +15,7 @@ dnl
 dnl Disable the build of the documentation
 dnl
 AC_ARG_ENABLE([doc],
-   [AC_HELP_STRING(
-       [--disable-doc],
-       [Disable documentation build @<:@default=enabled@:>@])],
+   [AS_HELP_STRING([--disable-doc],[Disable documentation build @<:@default=enabled@:>@])],
    [
     if test "x${enableval}" = "xyes" ; then
        efl_enable_doc="yes"
@@ -39,9 +37,7 @@ dnl
    efl_doxygen="doxygen"
 
    AC_ARG_WITH([doxygen],
-      [AC_HELP_STRING(
-          [--with-doxygen=FILE],
-          [doxygen program to use @<:@default=doxygen@:>@])],
+      [AS_HELP_STRING([--with-doxygen=FILE],[doxygen program to use @<:@default=doxygen@:>@])],
 dnl
 dnl Check the given doxygen program.
 dnl

main.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2021-2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -68,7 +68,7 @@ static rzip_control base_control, local_control, *control;
 static void usage(bool compat)
 {
 	print_output("lrz%s version %s\n", compat ? "" : "ip", PACKAGE_VERSION);
-	print_output("Copyright (C) Con Kolivas 2006-2016\n");
+	print_output("Copyright (C) Con Kolivas 2006-2022\n");
 	print_output("Based on rzip ");
 	print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n");
 	print_output("Usage: lrz%s [options] <file...>\n", compat ? "" : "ip");
@@ -79,15 +79,17 @@ static void usage(bool compat)
 	} else
 		print_output("	-c, -C, --check		check integrity of file written on decompression\n");
 	print_output("	-d, --decompress	decompress\n");
-	print_output("	-e, --encrypt		password protected sha512/aes128 encryption on compression\n");
+	print_output("	-e, --encrypt[=password] password protected sha512/aes128 encryption on compression\n");
 	print_output("	-h, -?, --help		show help\n");
 	print_output("	-H, --hash		display md5 hash integrity information\n");
 	print_output("	-i, --info		show compressed file information\n");
 	if (compat) {
 		print_output("	-L, --license		display software version and license\n");
 		print_output("	-P, --progress		show compression progress\n");
-	} else
+	} else {
 		print_output("	-q, --quiet		don't show compression progress\n");
+		print_output("	-Q, --very-quiet	don't show any output\n");
+	}
 	print_output("	-r, --recursive		operate recursively on directories\n");
 	print_output("	-t, --test		test compressed file integrity\n");
 	print_output("	-v[v%s], --verbose	Increase verbosity\n", compat ? "v" : "");
@@ -98,11 +100,12 @@ static void usage(bool compat)
 	print_output("	-f, --force		force overwrite of any existing files\n");
 	if (compat)
 		print_output("	-k, --keep		don't delete source files on de/compression\n");
-	print_output("	-%s, --keep-broken	keep broken or damaged output files\n", compat ? "K" : "k, -K");
+	print_output("	-K, --keep-broken	keep broken or damaged output files\n");
 	print_output("	-o, --outfile filename	specify the output file name and/or path\n");
 	print_output("	-O, --outdir directory	specify the output directory when -o is not used\n");
 	print_output("	-S, --suffix suffix	specify compressed suffix (default '.lrz')\n");
 	print_output("Options affecting compression:\n");
+	print_output("	--lzma			lzma compression (default)\n");
 	print_output("	-b, --bzip2		bzip2 compression\n");
 	print_output("	-g, --gzip		gzip compression using zlib\n");
 	print_output("	-l, --lzo		lzo compression (ultra fast)\n");
@@ -114,12 +117,13 @@ static void usage(bool compat)
 		print_output("	--fast			alias for -1\n");
 		print_output("	--best			alias for -9\n");
 	}
+	if (!compat)
 		print_output("	-L, --level level	set lzma/bzip2/gzip compression level (1-9, default 7)\n");
 	print_output("	-N, --nice-level value	Set nice value to value (default %d)\n", compat ? 0 : 19);
 	print_output("	-p, --threads value	Set processor count to override number of threads\n");
-	print_output("	-m, --maxram size	Set maximim available ram in hundreds of MB\n");
-	print_output("				overrides detected ammount of available ram\n");
-	print_output("	-T, --threshold		Disable LZO compressibility testing\n");
+	print_output("	-m, --maxram size	Set maximum available ram in hundreds of MB\n");
+	print_output("				overrides detected amount of available ram\n");
+	print_output("	-T, --threshold		Disable LZ4 compressibility testing\n");
 	print_output("	-U, --unlimited		Use unlimited window size beyond ramsize (potentially much slower)\n");
 	print_output("	-w, --window size	maximum compression window in hundreds of MB\n");
 	print_output("				default chosen by heuristic dependent on ram and chosen compression\n");
@@ -130,17 +134,25 @@ static void usage(bool compat)
 
 }
 
+static void license(void)
+{
+	print_output("lrz version %s\n", PACKAGE_VERSION);
+	print_output("Copyright (C) Con Kolivas 2006-2016\n");
+	print_output("Based on rzip ");
+	print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n");
+	print_output("This is free software.  You may redistribute copies of it under the terms of\n");
+	print_output("the GNU General Public License <http://www.gnu.org/licenses/gpl.html>.\n");
+	print_output("There is NO WARRANTY, to the extent permitted by law.\n");
+}
+
 static void sighandler(int sig __UNUSED__)
 {
-	struct termios termios_p;
-
-	/* Make sure we haven't died after disabling stdin echo */
-	tcgetattr(fileno(stdin), &termios_p);
-	termios_p.c_lflag |= ECHO;
-	tcsetattr(fileno(stdin), 0, &termios_p);
-
-	unlink_files(control);
-	exit(0);
+	signal(sig, SIG_IGN);
+	signal(SIGTERM, SIG_IGN);
+	signal(SIGTTIN, SIG_IGN);
+	signal(SIGTTOU, SIG_IGN);
+	print_err("Interrupted\n");
+	fatal_exit(&local_control);
 }
 
 static void show_summary(void)
@@ -175,15 +187,15 @@ static void show_summary(void)
 		if (!DECOMPRESS && !TEST_ONLY) {
 			print_verbose("Compression mode is: ");
 			if (LZMA_COMPRESS)
-				print_verbose("LZMA. LZO Compressibility testing %s\n", (LZO_TEST? "enabled" : "disabled"));
+				print_verbose("LZMA. LZ4 Compressibility testing %s\n", (LZ4_TEST? "enabled" : "disabled"));
 			else if (LZO_COMPRESS)
 				print_verbose("LZO\n");
 			else if (BZIP2_COMPRESS)
-				print_verbose("BZIP2. LZO Compressibility testing %s\n", (LZO_TEST? "enabled" : "disabled"));
+				print_verbose("BZIP2. LZ4 Compressibility testing %s\n", (LZ4_TEST? "enabled" : "disabled"));
 			else if (ZLIB_COMPRESS)
 				print_verbose("GZIP\n");
 			else if (ZPAQ_COMPRESS)
-				print_verbose("ZPAQ. LZO Compressibility testing %s\n", (LZO_TEST? "enabled" : "disabled"));
+				print_verbose("ZPAQ. LZ4 Compressibility testing %s\n", (LZ4_TEST? "enabled" : "disabled"));
 			else if (NO_COMPRESS)
 				print_verbose("RZIP pre-processing only\n");
 			if (control->window)
@@ -215,7 +227,7 @@ static struct option long_options[] = {
 	{"check",	no_argument,	0,	'C'},
 	{"decompress",	no_argument,	0,	'd'},
 	{"delete",	no_argument,	0,	'D'},
-	{"encrypt",	no_argument,	0,	'e'}, /* 5 */
+	{"encrypt",	optional_argument,	0,	'e'}, /* 5 */
 	{"force",	no_argument,	0,	'f'},
 	{"gzip",	no_argument,	0,	'g'},
 	{"help",	no_argument,	0,	'h'},
@@ -224,24 +236,27 @@ static struct option long_options[] = {
 	{"keep-broken",	no_argument,	0,	'k'},
 	{"keep-broken",	no_argument,	0,	'K'},
 	{"lzo",		no_argument,	0,	'l'},
-	{"level",	required_argument,	0,	'L'},
-	{"maxram",	required_argument,	0,	'm'}, /* 15 */
+	{"lzma",       	no_argument,	0,	'/'},
+	{"level",	optional_argument,	0,	'L'}, /* 15 */
+	{"license",	no_argument,	0,	'L'},
+	{"maxram",	required_argument,	0,	'm'},
 	{"no-compress",	no_argument,	0,	'n'},
 	{"nice-level",	required_argument,	0,	'N'},
 	{"outfile",	required_argument,	0,	'o'},
-	{"outdir",	required_argument,	0,	'O'},
-	{"threads",	required_argument,	0,	'p'}, /* 20 */
+	{"outdir",	required_argument,	0,	'O'}, /* 20 */
+	{"threads",	required_argument,	0,	'p'},
 	{"progress",	no_argument,	0,	'P'},
 	{"quiet",	no_argument,	0,	'q'},
+	{"very-quiet",	no_argument,	0,	'Q'},
 	{"recursive",	no_argument,	0,	'r'},
 	{"suffix",	required_argument,	0,	'S'},
-	{"test",	no_argument,	0,	't'},
-	{"threshold",	required_argument,	0,	'T'}, /* 25 */
+	{"test",	no_argument,	0,	't'},  /* 25 */
+	{"threshold",	required_argument,	0,	'T'},
 	{"unlimited",	no_argument,	0,	'U'},
 	{"verbose",	no_argument,	0,	'v'},
 	{"version",	no_argument,	0,	'V'},
-	{"window",	required_argument,	0,	'w'},
-	{"zpaq",	no_argument,	0,	'z'}, /* 30 */
+	{"window",	required_argument,	0,	'w'},  /* 30 */
+	{"zpaq",	no_argument,	0,	'z'},
 	{"fast",	no_argument,	0,	'1'},
 	{"best",	no_argument,	0,	'9'},
 	{0,	0,	0,	0},
@@ -287,22 +302,29 @@ static void recurse_dirlist(char *indir, char **dirlist, int *entries)
 	closedir(dirp);
 }
 
+static const char *loptions = "bcCdDefghHiKlL:nN:o:O:p:PqQrS:tTUm:vVw:z?";
+static const char *coptions = "bcCdefghHikKlLnN:o:O:p:PrS:tTUm:vVw:z?123456789";
+
 int main(int argc, char *argv[])
 {
 	bool lrzcat = false, compat = false, recurse = false;
+	bool options_file = false, conf_file_compression_set = false; /* for environment and tracking of compression setting */
 	struct timeval start_time, end_time;
 	struct sigaction handler;
 	double seconds,total_time; // for timers
+	bool nice_set = false;
 	int c, i;
 	int hours,minutes;
 	extern int optind;
 	char *eptr, *av; /* for environment */
+	char *endptr = NULL;
 
         control = &base_control;
 
 	initialise_control(control);
 
 	av = basename(argv[0]);
+	control->flags |= FLAG_OUTPUT;
 	if (!strcmp(av, "lrunzip"))
 		control->flags |= FLAG_DECOMPRESS;
 	else if (!strcmp(av, "lrzcat")) {
@@ -311,7 +333,6 @@ int main(int argc, char *argv[])
 	} else if (!strcmp(av, "lrz")) {
 		/* Called in gzip compatible command line mode */
 		control->flags &= ~FLAG_SHOW_PROGRESS;
-		control->nice_val = 0;
 		control->flags &= ~FLAG_KEEP_FILES;
 		compat = true;
 		long_options[1].name = "stdout";
@@ -324,19 +345,48 @@ int main(int argc, char *argv[])
 	/* Get Preloaded Defaults from lrzip.conf
 	 * Look in ., $HOME/.lrzip/, /etc/lrzip.
 	 * If LRZIP=NOCONFIG is set, then ignore config
+	 * If lrzip.conf sets a compression mode, options_file will be true.
+	 * This will allow for a test to permit an override of compression mode.
+	 * If there is an override, then all compression settings will be reset
+	 * and command line switches will prevail, including for --lzma.
 	 */
 	eptr = getenv("LRZIP");
 	if (eptr == NULL)
-		read_config(control);
+		options_file = read_config(control);
 	else if (!strstr(eptr,"NOCONFIG"))
-		read_config(control);
+		options_file = read_config(control);
+	if (options_file && (control->flags & FLAG_NOT_LZMA))		/* if some compression set in lrzip.conf    */
+		conf_file_compression_set = true;			/* need this to allow command line override */
 
-	while ((c = getopt_long(argc, argv, "bcCdDefghHikKlL:nN:o:O:pPqrS:tTUm:vVw:z?123456789", long_options, &i)) != -1) {
+	while ((c = getopt_long(argc, argv, compat ? coptions : loptions, long_options, &i)) != -1) {
 		switch (c) {
 		case 'b':
-			if (control->flags & FLAG_NOT_LZMA)
+		case 'g':
+		case 'l':
+		case 'n':
+		case 'z':
+			/* If some compression was chosen in lrzip.conf, allow this one time
+			 * because conf_file_compression_set will be true
+			 */
+			if ((control->flags & FLAG_NOT_LZMA) && conf_file_compression_set == false)
 				failure("Can only use one of -l, -b, -g, -z or -n\n");
+			/* Select Compression Mode */
+			control->flags &= ~FLAG_NOT_LZMA; /* must clear all compressions first */
+			if (c == 'b')
 				control->flags |= FLAG_BZIP2_COMPRESS;
+			else if (c == 'g')
+				control->flags |= FLAG_ZLIB_COMPRESS;
+			else if (c == 'l')
+				control->flags |= FLAG_LZO_COMPRESS;
+			else if (c == 'n')
+				control->flags |= FLAG_NO_COMPRESS;
+			else if (c == 'z')
+				control->flags |= FLAG_ZPAQ_COMPRESS;
+			/* now FLAG_NOT_LZMA will evaluate as true */
+			conf_file_compression_set = false;
+			break;
+		case '/':							/* LZMA Compress selected */
+			control->flags &= ~FLAG_NOT_LZMA;			/* clear alternate compression flags */
 			break;
 		case 'c':
 			if (compat) {
@@ -344,6 +394,7 @@ int main(int argc, char *argv[])
 				set_stdout(control);
 				break;
 			}
+			/* FALLTHRU */
 		case 'C':
 			control->flags |= FLAG_CHECK;
 			control->flags |= FLAG_HASH;
@@ -356,55 +407,54 @@ int main(int argc, char *argv[])
 			break;
 		case 'e':
 			control->flags |= FLAG_ENCRYPT;
+			control->passphrase = optarg;
 			break;
 		case 'f':
 			control->flags |= FLAG_FORCE_REPLACE;
 			break;
-		case 'g':
-			if (control->flags & FLAG_NOT_LZMA)
-				failure("Can only use one of -l, -b, -g, -z or -n\n");
-			control->flags |= FLAG_ZLIB_COMPRESS;
-			break;
 		case 'h':
-		case '?':
 			usage(compat);
-			return -1;
+			exit(0);
+			break;
 		case 'H':
 			control->flags |= FLAG_HASH;
 			break;
 		case 'i':
 			control->flags |= FLAG_INFO;
+			control->flags &= ~FLAG_DECOMPRESS;
 			break;
 		case 'k':
 			if (compat) {
 				control->flags |= FLAG_KEEP_FILES;
 				break;
 			}
+			/* FALLTHRU */
 		case 'K':
 			control->flags |= FLAG_KEEP_BROKEN;
 			break;
-		case 'l':
-			if (control->flags & FLAG_NOT_LZMA)
-				failure("Can only use one of -l, -b, -g, -z or -n\n");
-			control->flags |= FLAG_LZO_COMPRESS;
-			break;
 		case 'L':
-			control->compression_level = atoi(optarg);
+			if (compat) {
+				license();
+				exit(0);
+			}
+			control->compression_level = strtol(optarg, &endptr, 10);
 			if (control->compression_level < 1 || control->compression_level > 9)
 				failure("Invalid compression level (must be 1-9)\n");
+			if (*endptr)
+				failure("Extra characters after compression level: \'%s\'\n", endptr);
 			break;
 		case 'm':
-			control->ramsize = atol(optarg) * 1024 * 1024 * 100;
-			break;
-		case 'n':
-			if (control->flags & FLAG_NOT_LZMA)
-				failure("Can only use one of -l, -b, -g, -z or -n\n");
-			control->flags |= FLAG_NO_COMPRESS;
+			control->ramsize = strtol(optarg, &endptr, 10) * 1024 * 1024 * 100;
+			if (*endptr)
+				failure("Extra characters after ramsize: \'%s\'\n", endptr);
 			break;
 		case 'N':
-			control->nice_val = atoi(optarg);
-			if (control->nice_val < -20 || control->nice_val > 19)
-				failure("Invalid nice value (must be -20..19)\n");
+			nice_set = true;
+			control->nice_val = strtol(optarg, &endptr, 10);
+			if (control->nice_val < PRIO_MIN || control->nice_val > PRIO_MAX)
+				failure("Invalid nice value (must be %d...%d)\n", PRIO_MIN, PRIO_MAX);
+			if (*endptr)
+				failure("Extra characters after nice level: \'%s\'\n", endptr);
 			break;
 		case 'o':
 			if (control->outdir)
@@ -427,9 +477,11 @@ int main(int argc, char *argv[])
 				strcat(control->outdir, "/");
 			break;
 		case 'p':
-			control->threads = atoi(optarg);
+			control->threads = strtol(optarg, &endptr, 10);
 			if (control->threads < 1)
 				failure("Must have at least one thread\n");
+			if (*endptr)
+				failure("Extra characters after number of threads: \'%s\'\n", endptr);
 			break;
 		case 'P':
 			control->flags |= FLAG_SHOW_PROGRESS;
@@ -437,6 +489,10 @@ int main(int argc, char *argv[])
 		case 'q':
 			control->flags &= ~FLAG_SHOW_PROGRESS;
 			break;
+		case 'Q':
+			control->flags &= ~FLAG_SHOW_PROGRESS;
+			control->flags &= ~FLAG_OUTPUT;
+			break;
 		case 'r':
 			recurse = true;
 			break;
@@ -474,16 +530,16 @@ int main(int argc, char *argv[])
 			}
 			break;
 		case 'V':
+			control->msgout = stdout;
 			print_output("lrzip version %s\n", PACKAGE_VERSION);
 			exit(0);
 			break;
 		case 'w':
-			control->window = atol(optarg);
-			break;
-		case 'z':
-			if (control->flags & FLAG_NOT_LZMA)
-				failure("Can only use one of -l, -b, -g, -z or -n\n");
-			control->flags |= FLAG_ZPAQ_COMPRESS;
+			control->window = strtol(optarg, &endptr, 10);
+			if (control->window < 1)
+				failure("Window must be positive\n");
+			if (*endptr)
+				failure("Extra characters after window size: \'%s\'\n", endptr);
 			break;
 		case '1':
 		case '2':
@@ -496,9 +552,15 @@ int main(int argc, char *argv[])
 		case '9':
 			control->compression_level = c - '0';
 			break;
+		default:
+			usage(compat);
+			return 2;
 		}
 	}
 
+	if (compat && !SHOW_PROGRESS)
+		control->flags &= ~FLAG_OUTPUT;
+
 	argc -= optind;
 	argv += optind;
 
@@ -531,12 +593,22 @@ int main(int argc, char *argv[])
 
 	/* Set the main nice value to half that of the backend threads since
 	 * the rzip stage is usually the rate limiting step */
-	if (control->nice_val > 0 && !NO_COMPRESS) {
-		if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val / 2) == -1))
-			print_err("Warning, unable to set nice value\n");
+	control->current_priority = getpriority(PRIO_PROCESS, 0);
+	if (nice_set) {
+		if (!NO_COMPRESS) {
+			/* If niceness can't be set. just reset process priority */
+			if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val/2) == -1)) {
+				print_err("Warning, unable to set nice value %d...Resetting to %d\n",
+					control->nice_val, control->current_priority);
+				setpriority(PRIO_PROCESS, 0, (control->nice_val=control->current_priority));
+			}
 		} else {
-		if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1))
-			print_err("Warning, unable to set nice value\n");
+			if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1)) {
+				print_err("Warning, unable to set nice value %d...Resetting to %d\n",
+					control->nice_val, control->current_priority);
+				setpriority(PRIO_PROCESS, 0, (control->nice_val=control->current_priority));
+			}
+		}
 	}
 
 	/* One extra iteration for the case of no parameters means we will default to stdin/out */
@@ -604,6 +676,7 @@ recursion:
 
 		if (STDIN)
 			control->inFILE = stdin;
+
 		/* Implement signal handler only once flags are set */
 		sigemptyset(&handler.sa_mask);
 		handler.sa_flags = 0;
@@ -639,8 +712,8 @@ recursion:
 
 		gettimeofday(&start_time, NULL);
 
-		if (unlikely(STDIN && ENCRYPT))
-			failure("Unable to work from STDIN while reading password\n");
+		if (!control->passphrase && (unlikely((STDIN || STDOUT) && ENCRYPT)))
+			failure("Unable to work from STDIO while reading password\n");
 
 		memcpy(&local_control, &base_control, sizeof(rzip_control));
 		if (DECOMPRESS || TEST_ONLY)
@@ -658,7 +731,7 @@ recursion:
 		minutes = (int)(total_time / 60) % 60;
 		seconds = total_time - hours * 3600 - minutes * 60;
 		if (!INFO)
-			print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
+			print_output("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
 		if (recurse)
 			goto recursion;
 	}

man/lrz.1.pod

@@ -1,6 +1,8 @@
+#!/usr/bin/perl -w
+
 #   Copyright
 #
-#      Copyright (C) 2016 Con Kolivas
+#      Copyright (C) 2021 Con Kolivas
 #
 #   License
 #
@@ -38,63 +40,278 @@
 
 =pod
 
+=encoding utf8
+
 =head1 NAME
 
 lrz - gzip compatible command line variant of lrzip
 
+
 =head1 SYNOPSIS
 
-  lrz [OPTIONS] <file>
+B<lrz> [options] I<file>
+
 
 =head1 DESCRIPTION
 
-lrz is identical to the lrzip application however its command line options and
-behaviour are made to be as compatible with gzip as possible.
+B<lrz> is identical to the B<lrzip> application, however, its command
+line options and behaviour are made to be as compatible with B<gzip>
+as possible.
 
 =head1 OPTIONS
 
-lrz differs from lrzip by accepting the following commands:
+=head2 General options
 
-General options:
-        -c, --stdout            output to STDOUT
-        -C, --check             check integrity of file written on decompression
-        -d, --decompress        decompress
-        -e, --encrypt           password protected sha512/aes128 encryption on compression
-        -h, -?, --help          show help
-        -H, --hash              display md5 hash integrity information
-        -i, --info              show compressed file information
-        -L, --license           display software version and license
-        -P, --progress          show compression progress
-        -r, --recursive         operate recursively on directories
-        -t, --test              test compressed file integrity
-        -v[vv], --verbose       Increase verbosity
-        -V, --version           show version
-Options affecting output:
-        -f, --force             force overwrite of any existing files
-        -k, --keep              don't delete source files on de/compression
-        -K, --keep-broken       keep broken or damaged output files
-        -o, --outfile filename  specify the output file name and/or path
-        -O, --outdir directory  specify the output directory when -o is not used
-        -S, --suffix suffix     specify compressed suffix (default '.lrz')
-Options affecting compression:
-        -b, --bzip2             bzip2 compression
-        -g, --gzip              gzip compression using zlib
-        -l, --lzo               lzo compression (ultra fast)
-        -n, --no-compress       no backend compression - prepare for other compressor
-        -z, --zpaq              zpaq compression (best, extreme compression, extremely slow)
-Low level options:
-        -1 .. -9                set lzma/bzip2/gzip compression level (1-9, default 7)
-        --fast                  alias for -1
-        --best                  alias for -9
-        -L, --level level       set lzma/bzip2/gzip compression level (1-9, default 7)
-        -N, --nice-level value  Set nice value to value (default 0)
-        -p, --threads value     Set processor count to override number of threads
-        -m, --maxram size       Set maximim available ram in hundreds of MB
-                                overrides detected ammount of available ram
-        -T, --threshold         Disable LZO compressibility testing
-        -U, --unlimited         Use unlimited window size beyond ramsize (potentially much slower)
-        -w, --window size       maximum compression window in hundreds of MB
-                                default chosen by heuristic dependent on ram and chosen compression
+=over 9
+
+=item B<--stdout>
+
+=item B<-c>
+
+Output to STDOUT.
+
+
+=item B<--check>
+
+=item B<-C>
+
+Check integrity of file written on decompression.
+
+
+=item B<--decompress>
+
+=item B<-d>
+
+Decompress.
+
+
+=item B<--encrypt>[=I<password>]
+
+=item B<-e>
+
+Password protect sha512/aes128 encryption on compression.
+
+
+=item B<--help>
+
+=item B<-h>
+
+=item B<-?>
+
+Show help.
+
+
+=item B<--hash>
+
+=item B<-H>
+
+Display md5 hash integrity information.
+
+
+=item B<--info>
+
+=item B<-i>
+
+Show compressed file information.
+
+
+=item B<--license>
+
+=item B<-L>
+
+Display software version and license.
+
+
+=item B<--progress>
+
+=item B<-P>
+
+Show compression progress.
+
+
+=item B<--recursive>
+
+=item B<-r>
+
+Operate recursively on directories.
+
+
+=item B<--test>
+
+=item B<-t>
+
+Test compressed file integrity.
+
+
+=item B<--verbose>
+
+=item B<-v[vv]>
+
+Increase verbosity.
+
+
+=item B<--version>
+
+=item B<-V>
+
+Show version.
+
+
+=back
+
+
+=head2 Options affecting output
+
+=over 9
+
+=item B<--force>
+
+=item B<-f>
+
+Force overwrite of any existing files.
+
+
+=item B<--keep>
+
+=item B<-k>
+
+Don't delete source files on de/compression.
+
+
+=item B<--keep-broken>
+
+=item B<-K>
+
+Keep broken or damaged output files.
+
+
+=item B<--outfile>  I<name>
+
+=item B<-o> I<name>
+
+Specify the output file name and/or path.
+
+
+=item B<--outdir> I<dir>
+
+=item B<-O> I<dir>
+
+Specify the output directory when B<-o> is not used.
+
+
+=item B<--suffix> I<suffix>
+
+=item B<-S> I<suffix>
+
+Specify compressed suffix (default '.lrz').
+
+
+=back
+
+=head2 Options affecting compression
+
+=over 9
+
+=item B<--bzip2>
+
+=item B<-b>
+
+Bzip2 compression.
+
+=item B<--gzip>
+
+=item B<-g>
+
+Gzip compression using zlib.
+
+=item B<--lzo>
+
+=item B<-l>
+
+Lzo compression (ultra fast).
+
+=item B<--lzma>
+
+Lzma compression (default).
+
+=item B<--no-compress>
+
+=item B<-n>
+
+No backend compression - prepare for other compressor.
+
+=item B<--zpaq>
+
+=item B<-z>
+
+Zpaq compression (best, extreme compression, extremely slow).
+
+
+=back
+
+=head2 Low level options
+
+=over 9
+
+=item B<-1> .. B<-9>
+
+=item B<--level> I<level>
+
+=item B<-L> I<level>
+
+Set lzma/bzip2/gzip compression level (1-9, default 7).
+
+
+=item B<--fast>
+
+Alias for B<-1>.
+
+
+=item B<--best>
+
+Alias for B<-9>.
+
+
+=item B<--nice-level> I<value>
+
+=item B<-N> I<value>
+
+Set nice value to I<value> (default 0).
+
+
+=item B<--threads> I<value>
+
+=item B<-P> I<value>
+
+Set processor count to override number of threads.
+
+=item B<--maxram> I<size>
+
+=item B<-m> I<size>
+
+Set maximum available ram as I<size> * 100 MB.
+Overrides detected amount of available ram.
+
+=item B<--threshold>
+
+=item B<-T>
+
+Disable LZ4 compressibility testing.
+
+=item B<--unlimited>
+
+=item B<-U>
+
+Use unlimited window size beyond ramsize (potentially much slower).
+
+=item B<--window> I<size>
+
+=item B<-w> I<size>
+
+Set maximum compression window as I<size> * 100 MB.
+Default chosen by heuristic dependent on ram and chosen compression.
+
+=back
 
 See also lrzip(1)
 

man/lrzip.1

@@ -1,4 +1,4 @@
-.TH "lrzip" "1" "June 2016" "" ""
+.TH "lrzip" "1" "February 2022" "" ""
 .SH "NAME"
 lrzip \- a large-file compression program
 .SH "SYNOPSIS"
@@ -38,11 +38,12 @@ Here is a summary of the options to lrzip\&.
 General options:
  \-c, \-\-check             check integrity of file written on decompression
  \-d, \-\-decompress        decompress
- \-e, \-\-encrypt           password protected sha512/aes128 encryption on compression
+ \-e, \-\-encrypt[=password] password protected sha512/aes128 encryption on compression
  \-h, \-?, \-\-help          show help
  \-H, \-\-hash              display md5 hash integrity information
  \-i, \-\-info              show compressed file information
  \-q, \-\-quiet             don't show compression progress
+ \-Q, \-\-very-quiet        don't show any output
  \-r, \-\-recursive         operate recursively on directories
  \-t, \-\-test              test compressed file integrity
  \-v[v], \-\-verbose        Increase verbosity
@@ -64,9 +65,9 @@ Low level options:
  \-L, \-\-level level       set lzma/bzip2/gzip compression level (1-9, default 7)
  \-N, \-\-nice-level value  Set nice value to value (default 19)
  \-p, \-\-threads value     Set processor count to override number of threads
- \-m, \-\-maxram size       Set maximim available ram in hundreds of MB
-                         overrides detected ammount of available ram
- \-T, \-\-threshold         Disable LZO compressibility testing
+ \-m, \-\-maxram size       Set maximum available ram in hundreds of MB
+                         overrides detected amount of available ram
+ \-T, \-\-threshold         Disable LZ4 compressibility testing
  \-U, \-\-unlimited         Use unlimited window size beyond ramsize (potentially much slower)
  \-w, \-\-window size       maximum compression window in hundreds of MB
                          default chosen by heuristic dependent on ram and chosen compression
@@ -102,6 +103,7 @@ the name used to launch the program. If it contains the string
 "lrzcat" then the \-d \-o \- options are automatically set.
 .IP
 .IP "\fB-e\fP"
+.IP "\fB\-\-encrypt\fP[=\fIpassword\fP]"
 Encrypt. This option enables high grade password encryption using a combination
 of multiply sha512 hashed password, random salt and aes128 CBC encryption.
 Passwords up to 500 characters long are supported, and the encryption mechanism
@@ -141,6 +143,10 @@ bursts with lzma compression which is the default compression. This means
 that it will progress very rapidly for short periods and then stop for
 long periods.
 .IP
+.IP "\fB-Q\fP"
+If this option is specified then lrzip will not show any output to the console
+except for error messages.
+.IP
 .IP "\fB-r\fP"
 If this option is specified, lrzip will recursively enter the directories
 specified, compressing or decompressing every file individually in the same
@@ -239,13 +245,13 @@ decrease the load on your machine, or to improve compression. Setting it to
 1 will maximise compression but will not attempt to use more than one CPU.
 .IP
 .IP "\fB-T\fP"
-Disables the LZO compressibility threshold testing when a slower compression
-back-end is used. LZO testing is normally performed for the slower back-end
+Disables the LZ4 compressibility threshold testing when a slower compression
+back-end is used. LZ4 testing is normally performed for the slower back-end
 compression of LZMA and ZPAQ. The reasoning is that if it is completely
-incompressible by LZO then it will also be incompressible by them. Thus if a
-block fails to be compressed by the very fast LZO, lrzip will not attempt to
+incompressible by LZ4 then it will also be incompressible by them. Thus if a
+block fails to be compressed by the very fast LZ4, lrzip will not attempt to
 compress that block with the slower compressor, thereby saving time. If this
-option is enabled, it will bypass the LZO testing and attempt to compress each
+option is enabled, it will bypass the LZ4 testing and attempt to compress each
 block regardless.
 .IP
 .IP "\fB-U \fP"
@@ -348,6 +354,12 @@ lzop(1),
 rzip(1),
 zip(1)
 
+.PP
+.SH "DIAGNOSTICS"
+.PP
+Exit status is normally 0; if an error occurs, exit status is 1, usage errors
+is 2.
+
 .PP
 .SH "AUTHOR and CREDITS"
 .br
@@ -364,8 +376,9 @@ zpaq was written by Matt Mahoney.
 Peter Hyman added informational output, updated LZMA SDK,
 and added lzma multi-threading capabilities.
 .PP
-If you wish to report a problem, or make a suggestion, then please email Con at
-kernel@kolivas.org
+If you wish to report a problem, or make a suggestion, then please consult the
+git repository at:
+https://github.com/ckolivas/lrzip
 .PP
 lrzip is released under the GNU General Public License version 2.
 Please see the file COPYING for license details.

man/lrzip.conf.5

@@ -1,4 +1,4 @@
-.TH "lrzip.conf" "5" "January 2009" "" ""
+.TH "lrzip.conf" "5" "January 2009, updated May 2019" "" ""
 .SH "NAME"
 lrzip.conf \- Configuration File for lrzip
 .SH "DESCRIPTION"
@@ -13,54 +13,63 @@ three places\&:
 .nf
 $PWD \- Current Directory
 /etc/lrzip
-$HOME/\&./lrzip
+$HOME/\&.lrzip
 .PP
 Parameters are set in \fBPARAMETER\&=VALUE\fP fashion where any line
 beginning with a \fB#\fP or that is blank will be ignored\&.
-Parameter values are not case sensitive\&.
+Parameter values are not case sensitive except where specified\&.
 .PP
 .SH "CONFIG FILE EXAMPLE"
 .nf
 # This is a comment.
 
-# Compression Window size in 100MB. Normally selected by program.
-# WINDOW = 5
-# Compression Level 1-9 (7 Default).
+# Compression Window size in 100MB. Normally selected by program. (-w)
+# WINDOW = 20
+# Compression Level 1-9 (7 Default). (-L)
 # COMPRESSIONLEVEL = 7
-# Unlimited Ram Compression
-# UNLIMITED = YES
-# Compression Method, rzip, gzip, bzip2, lzo, or lzma (default), zpaq.
-# COMPRESSIONMETHOD = LZMA
-# Perform LZO Test. Default = YES (\-T option, NO)
+# Use -U setting, Unlimited ram. Yes or No
+# UNLIMITED = NO
+# Compression Method, rzip, gzip, bzip2, lzo, or lzma (default), or zpaq. (-n -g -b -l --lzma -z)
+# If specified here, command line options not usable.
+# COMPRESSIONMETHOD = lzma
+# Perform LZO Test. Default = YES (-T )
 # LZOTEST = NO
-# Hash Check on decompression, YES
+# Hash Check on decompression, (-c)
 # HASHCHECK = YES
-# Show HASH value on Compression even if Verbose is off, YES
+# Show HASH value on Compression even if Verbose is off, YES (-H)
 # SHOWHASH = YES
 
-# Default output directory
+# Default output directory (-O)
 # OUTPUTDIRECTORY = location
-# Verbosity, Yes or Max
-# VERBOSITY = MAX
-# Show Progress as file is parsed, YES, NO (yes is default)
-# SHOWPROGRESS = NO
-# Set Niceness. 19 is default. \-20 to 19 is the allowable range
+# Verbosity, YES or MAX (v, vv)
+# VERBOSITY = max
+# Show Progress as file is parsed, YES or no (NO = -q option)
+# SHOWPROGRESS = YES
+
+# Set Niceness. 19 is default. -20 to 19 is the allowable range (-N)
 # NICE = 19
 
-# Keep broken or damaged output files, YES
+# Keep broken or damaged output files, YES (-K)
 # KEEPBROKEN = YES
-# Delete source file after compression
+
+# Delete source file after compression (-D)
 # this parameter and value are case sensitive
 # value must be YES to activate
+
 # DELETEFILES = NO
 
-# Replace existing lrzip file when compressing
+# Replace existing lrzip file when compressing (-f)
 # this parameter and value are case sensitive
 # value must be YES to activate
-# REPLACEFILE = NO
 
-# Select Temporary Directory when stdin/stdout or Test file is used
+# REPLACEFILE = YES 
+
+# Override for Temporary Directory. Only valid when stdin/out or Test is used
 # TMPDIR = /tmp
+
+# Whether to use encryption on compression YES, NO (-e)
+# ENCRYPT = NO
+
 .fi
 .PP
 .SH "NOTES"

regressiontest.good

@@ -0,0 +1,49 @@
+Test basic use
+Test decompression in read-only dir
+   1000    1000    3893
+this should be silent
+man page for lrz should exist
+0
+compress stdin to stdout
+Respect $TMPDIR
+   1000    1000    3893
+Decompress in read only dir
+   1000    1000    3893
+Test -cd
+   1000    1000    3893
+Test -cfd should not remove testfile.lrz
+   1000    1000    3893
+testfile.lrz
+Test -1c
+   1002    1002    3975
+Test -r
+t10.lrz
+t1.lrz
+t2.lrz
+t3.lrz
+t4.lrz
+t5.lrz
+t6.lrz
+t7.lrz
+t8.lrz
+t9.lrz
+Test tar compatibility
+t/
+t/t8
+t/t7
+t/t3
+t/t5
+t/t2
+t/t6
+t/t10
+t/t4
+t/t9
+t/t1
+11
+test compress of 1 GB data with parallel --pipe --compress
+1073741824
+test compress of 1 GB with sort --compress-program
+1073741825
+test should not lrz -dc removes file
+OK
+testfile.lrz

regressiontest.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# Regression test.
+#
+# Copyright (C) 2016
+# Ole Tange and Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+bash > regressiontest.out 2>&1  <<'_EOS'
+  rm -f testfile.lrz
+  seq 1000 > testfile
+  
+  echo 'Test basic use'
+    lrz testfile
+  
+  echo 'Test decompression in read-only dir'
+    mkdir -p ro
+    cp testfile.lrz ro
+    chmod 500 ro
+    cd ro
+    lrz -dc testfile.lrz | wc
+    cd ..
+  
+  echo 'this should be silent'
+    lrz -d testfile.lrz
+  
+  echo 'man page for lrz should exist'
+    man lrz >/dev/null
+    echo $?
+  
+  echo 'compress stdin to stdout'
+    cat testfile | lrz | cat > testfile.lrz
+  
+  echo 'Respect $TMPDIR'
+    mkdir -p t
+    chmod 111 t
+    cd t
+    TMPDIR=.. lrz -d < ../testfile.lrz | wc
+    cd ..
+    rm -rf t
+  
+  echo 'Decompress in read only dir'
+    mkdir -p t
+    chmod 111 t
+    cd t
+    lrz -d < ../testfile.lrz | wc
+    cd ..
+    rm -rf t
+  
+  echo 'Test -cd'
+    mkdir -p t
+    chmod 111 t
+    cd t
+    lrz -cd  ../testfile.lrz | wc
+    cd ..
+    rm -rf t
+
+  echo 'Test -cfd should not remove testfile.lrz'
+    mkdir -p t
+    chmod 111 t
+    cd t
+    lrz -cfd  ../testfile.lrz | wc
+    cd ..
+    rm -rf t
+    ls testfile.lrz
+
+  echo 'Test -1c'
+    lrz -1c testfile | wc
+
+  echo 'Test -r'
+    mkdir t
+    touch t/t{1..10}
+    lrz -r t
+    ls t
+    rm -r t
+
+  echo 'Test tar compatibility'
+    mkdir t
+    touch t/t{1..10}
+    tar --use-compress-program lrz -cvf testfile.tar.lrz t
+    tar --use-compress-program lrz -tvf testfile.tar.lrz | wc -l
+    rm -r t
+
+  echo 'test compress of 1 GB data with parallel --pipe --compress'
+    yes "`echo {1..100}`" |
+      head -c 1G |
+      parallel --pipe --block 100m --compress-program lrz cat |
+      wc -c
+
+  echo 'test compress of 1 GB with sort --compress-program'
+    yes "`echo {1..100}`" |
+      head -c 1G |
+      sort --compress-program lrz |
+      wc -c
+
+  echo 'test should not lrz -dc removes file'
+    rm testfile.lrz
+    echo OK > testfile
+    lrz testfile
+    lrz -dc testfile.lrz
+    ls testfile.lrz
+
+_EOS
+
+diff regressiontest.good regressiontest.out

runzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
    Copyright (C) 1998-2003 Andrew Tridgell
 
    This program is free software; you can redistribute it and/or modify
@@ -161,12 +161,12 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, uint32 *cksum
 
 	stream_read = read_stream(control, ss, 1, buf, len);
 	if (unlikely(stream_read == -1 )) {
-		free(buf);
+		dealloc(buf);
 		fatal_return(("Failed to read_stream in unzip_literal\n"), -1);
 	}
 
 	if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) {
-		free(buf);
+		dealloc(buf);
 		fatal_return(("Failed to write literal buffer of size %lld\n", stream_read), -1);
 	}
 
@@ -175,7 +175,7 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, uint32 *cksum
 	if (!NO_MD5)
 		md5_process_bytes(buf, stream_read, &control->ctx);
 
-	free(buf);
+	dealloc(buf);
 	return stream_read;
 }
 
@@ -194,7 +194,7 @@ static i64 read_fdhist(rzip_control *control, void *buf, i64 len)
 static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, int chunk_bytes)
 {
 	i64 offset, n, total, cur_pos;
-	uchar *buf, *off_buf;
+	uchar *buf;
 
 	if (unlikely(len < 0))
 		failure_return(("len %lld is negative in unzip_match!\n",len), -1);
@@ -212,34 +212,39 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum,
 		fatal_return(("Seek failed by %d from %d on history file in unzip_match\n",
 		      offset, cur_pos), -1);
 
-	buf = (uchar *)malloc(len);
+	n = MIN(len, offset);
+	if (unlikely(n < 1))
+		fatal_return(("Failed fd history in unzip_match due to corrupt archive\n"), -1);
+
+	buf = (uchar *)malloc(n);
 	if (unlikely(!buf))
 		fatal_return(("Failed to malloc match buffer of size %lld\n", len), -1);
-	off_buf = buf;
+
+	if (unlikely(read_fdhist(control, buf, (size_t)n) != (ssize_t)n)) {
+		dealloc(buf);
+		fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1);
+	}
 
 	while (len) {
 		n = MIN(len, offset);
+		if (unlikely(n < 1))
+			fatal_return(("Failed fd history in unzip_match due to corrupt archive\n"), -1);
 
-		if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) {
-			free(buf);
-			fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1);
-		}
-		if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) {
-			free(buf);
+		if (unlikely(write_1g(control, buf, (size_t)n) != (ssize_t)n)) {
+			dealloc(buf);
 			fatal_return(("Failed to write %d bytes in unzip_match\n", n), -1);
 		}
 
 		if (!HAS_MD5)
-			*cksum = CrcUpdate(*cksum, off_buf, n);
+			*cksum = CrcUpdate(*cksum, buf, n);
 		if (!NO_MD5)
-			md5_process_bytes(off_buf, n, &control->ctx);
+			md5_process_bytes(buf, n, &control->ctx);
 
 		len -= n;
-		off_buf += n;
 		total += n;
 	}
 
-	free(buf);
+	dealloc(buf);
 
 	return total;
 }
@@ -367,7 +372,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64
 /* Decompress an open file. Call fatal_return(() on error
    return the number of bytes that have been retrieved
  */
-i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size)
+i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size)
 {
 	uchar md5_stored[MD5_DIGEST_SIZE];
 	struct timeval start,end;
@@ -380,22 +385,18 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp
 
 	do {
 		u = runzip_chunk(control, fd_in, expected_size, total);
-		if (unlikely(u == -1)) {
+		if (u < 1) {
+			if (u < 0 || total < expected_size) {
 				print_err("Failed to runzip_chunk in runzip_fd\n");
 				return -1;
 			}
+		}
 		total += u;
-		if (TMP_OUTBUF) {
-			if (unlikely(!flush_tmpoutbuf(control))) {
-				print_err("Failed to flush_tmpoutbuf in runzip_fd\n");
+		if (unlikely(!flush_tmpout(control))) {
+			print_err("Failed to flush_tmpout in runzip_fd\n");
 				return -1;
 		}
-		} else if (STDOUT) {
-			if (unlikely(!dump_tmpoutfile(control, fd_out))) {
-				print_err("Failed to dump_tmpoutfile in runzip_fd\n");
-				return -1;
-			}
-		}
+
 		if (TMP_INBUF)
 			clear_tmpinbuf(control);
 		else if (STDIN && !DECOMPRESS) {
@@ -411,7 +412,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp
 		tdiff = end.tv_sec - start.tv_sec;
 		if (!tdiff)
 			tdiff = 1;
-		print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n",
+		print_output("\nAverage DeCompression Speed: %6.3fMB/s\n",
 			       (total / 1024 / 1024) / tdiff);
 	}
 

runzip.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2011 Con Kolivas
+   Copyright (C) 2006-2011,2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -22,6 +22,6 @@
 
 #include "lrzip_private.h"
 
-i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size);
+i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size);
 
 #endif

rzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2018,2022 Con Kolivas
    Copyright (C) 1998 Andrew Tridgell
 
    Modified to use flat hash, memory limit and variable hash culling
@@ -588,7 +588,7 @@ static void *cksumthread(void *data)
 	*control->checksum.cksum = CrcUpdate(*control->checksum.cksum, control->checksum.buf, control->checksum.len);
 	if (!NO_MD5)
 		md5_process_bytes(control->checksum.buf, control->checksum.len, &control->ctx);
-	free(control->checksum.buf);
+	dealloc(control->checksum.buf);
 	cksem_post(control, &control->cksumsem);
 	return NULL;
 }
@@ -718,8 +718,8 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st,
 				failure("Failed to malloc ckbuf in hash_search\n");
 			control->do_mcpy(control, control->checksum.buf, cksum_limit, control->checksum.len);
 			control->checksum.cksum = &st->cksum;
-			cksum_update(control);
 			cksum_limit += control->checksum.len;
+			cksum_update(control);
 		}
 	}
 
@@ -765,7 +765,7 @@ static inline void hash_search(rzip_control *control, struct rzip_state *st,
 		st->cksum = CrcUpdate(st->cksum, control->checksum.buf, cksum_remains);
 		if (!NO_MD5)
 			md5_process_bytes(control->checksum.buf, cksum_remains, &control->ctx);
-		free(control->checksum.buf);
+		dealloc(control->checksum.buf);
 		cksem_post(control, &control->cksumsem);
 	} else {
 		cksem_wait(control, &control->cksumsem);
@@ -872,6 +872,17 @@ init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in,
 	sb->fd = fd_in;
 }
 
+static void add_to_sslist(rzip_control *control, struct rzip_state *st)
+{
+	struct node *node = calloc(sizeof(struct node), 1);
+
+	if (unlikely(!node))
+		failure("Failed to calloc struct node in add_to_sslist\n");
+	node->data = st->ss;
+	node->prev = st->sslist;
+	st->head = node;
+}
+
 /* compress a chunk of an open file. Assumes that the file is able to
    be mmap'd and is seekable */
 static inline void
@@ -903,6 +914,23 @@ rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out,
 
 	if (unlikely(close_stream_out(control, st->ss)))
 		failure("Failed to flush/close streams in rzip_chunk\n");
+
+	/* Save the sinfo data to a list to be safely released after all
+	 * threads have been shut down. */
+	add_to_sslist(control, st);
+}
+
+static void clear_sslist(struct rzip_state *st)
+{
+	while (st->head) {
+		struct node *node = st->head;
+		struct stream_info *sinfo = node->data;
+
+		dealloc(sinfo->s);
+		dealloc(sinfo);
+		st->head = node->prev;
+		dealloc(node);
+	}
 }
 
 /* compress a whole file chunks at a time */
@@ -937,13 +965,13 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out)
 
 	if (LZO_COMPRESS) {
 		if (unlikely(lzo_init() != LZO_E_OK)) {
-			free(st);
+			dealloc(st);
 			failure("lzo_init() failed\n");
 		}
 	}
 
 	if (unlikely(fstat(fd_in, &s))) {
-		free(st);
+		dealloc(st);
 		failure("Failed to stat fd_in in rzip_fd\n");
 	}
 
@@ -958,15 +986,15 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out)
 		* compressed file, based on the compressed file being as large as the
 		* uncompressed file. */
 		if (unlikely(fstatvfs(fd_out, &fbuf))) {
-			free(st);
+			dealloc(st);
 			failure("Failed to fstatvfs in compress_file\n");
 		}
 		free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail;
 		if (free_space < control->st_size) {
 			if (FORCE_REPLACE)
-				print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n");
+				print_output("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n");
 			else {
-				free(st);
+				dealloc(st);
 				failure("Possibly inadequate free space to compress file, use -f to override.\n");
 			}
 		}
@@ -1041,16 +1069,16 @@ retry:
 			if (sb->buf_low == MAP_FAILED) {
 				if (unlikely(errno != ENOMEM)) {
 					close_streamout_threads(control);
-					free(st->hash_table);
-					free(st);
+					dealloc(st->hash_table);
+					dealloc(st);
 					failure("Failed to mmap %s\n", control->infile);
 				}
 				st->mmap_size = st->mmap_size / 10 * 9;
 				round_to_page(&st->mmap_size);
 				if (unlikely(!st->mmap_size)) {
 					close_streamout_threads(control);
-					free(st->hash_table);
-					free(st);
+					dealloc(st->hash_table);
+					dealloc(st);
 					failure("Unable to mmap any ram\n");
 				}
 				goto retry;
@@ -1063,16 +1091,16 @@ retry:
 			if (sb->buf_low == MAP_FAILED) {
 				if (unlikely(errno != ENOMEM)) {
 					close_streamout_threads(control);
-					free(st->hash_table);
-					free(st);
+					dealloc(st->hash_table);
+					dealloc(st);
 					failure("Failed to mmap %s\n", control->infile);
 				}
 				st->mmap_size = st->mmap_size / 10 * 9;
 				round_to_page(&st->mmap_size);
 				if (unlikely(!st->mmap_size)) {
 					close_streamout_threads(control);
-					free(st->hash_table);
-					free(st);
+					dealloc(st->hash_table);
+					dealloc(st);
 					failure("Unable to mmap any ram\n");
 				}
 				goto retry;
@@ -1159,16 +1187,16 @@ retry:
 		len -= st->chunk_size;
 		if (unlikely(len > 0 && control->eof)) {
 			close_streamout_threads(control);
-			free(st->hash_table);
-			free(st);
+			dealloc(st->hash_table);
+			dealloc(st);
 			failure("Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n");
 		}
 	}
 
 	if (likely(st->hash_table))
-		free(st->hash_table);
+		dealloc(st->hash_table);
 	if (unlikely(!close_streamout_threads(control))) {
-		free(st);
+		dealloc(st);
 		failure("Failed to close_streamout_threads in rzip_fd\n");
 	}
 
@@ -1184,20 +1212,18 @@ retry:
 		/* When encrypting data, we encrypt the MD5 value as well */
 		if (ENCRYPT)
 			if (unlikely(!lrz_encrypt(control, control->md5_resblock, MD5_DIGEST_SIZE, control->salt_pass))) {
-				free(st);
+				dealloc(st);
 				failure("Failed to lrz_encrypt in rzip_fd\n");
 			}
 		if (unlikely(write_1g(control, control->md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) {
-			free(st);
+			dealloc(st);
 			failure("Failed to write md5 in rzip_fd\n");
 		}
 	}
 
-	if (TMP_OUTBUF) {
-		if (unlikely(!flush_tmpoutbuf(control))) {
-			free(st);
-			failure("Failed to flush_tmpoutbuf in rzip_fd\n");
-		}
+	if (unlikely(!flush_tmpout(control))) {
+			dealloc(st);
+			failure("Failed to flush_tmpout in rzip_fd\n");
 	}
 
 	gettimeofday(&current, NULL);
@@ -1221,28 +1247,10 @@ retry:
 	       (1.0 + st->stats.match_bytes) / st->stats.literal_bytes);
 
 	if (!STDIN)
-		print_progress("%s - ", control->infile);
-	print_progress("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n",
+		print_output("%s - ", control->infile);
+	print_output("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n",
 		       1.0 * s.st_size / s2.st_size, chunkmbs);
 
-	free(st);
-}
-
-void rzip_control_free(rzip_control *control)
-{
-	size_t x;
-	if (!control)
-		return;
-
-	free(control->tmpdir);
-	free(control->outname);
-	free(control->outdir);
-	if (control->suffix && control->suffix[0]) free(control->suffix);
-
-	for (x = 0; x < control->sinfo_idx; x++) {
-		free(control->sinfo_queue[x]->s);
-		free(control->sinfo_queue[x]);
-	}
-	free(control->sinfo_queue);
-	free(control);
+	clear_sslist(st);
+	dealloc(st);
 }

rzip.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2022 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell
 
@@ -22,6 +22,5 @@
 #include "lrzip_private.h"
 
 void rzip_fd(rzip_control *control, int fd_in, int fd_out);
-void rzip_control_free(rzip_control *control);
 
 #endif

util.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016 Con Kolivas
+   Copyright (C) 2006-2016,2021-2022 Con Kolivas
    Copyright (C) 2011 Serge Belyshev
    Copyright (C) 2008, 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell
@@ -100,6 +100,13 @@ void fatal_exit(rzip_control *control)
 	tcsetattr(fileno(stdin), 0, &termios_p);
 
 	unlink_files(control);
+	if (!STDOUT && !TEST_ONLY && control->outfile) {
+		if (!KEEP_BROKEN) {
+			print_verbose("Deleting broken file %s\n", control->outfile);
+			unlink(control->outfile);
+		} else
+			print_verbose("Keeping broken file %s as requested\n", control->outfile);
+	}
 	fprintf(control->outputfile, "Fatal error - exiting\n");
 	fflush(control->outputfile);
 	exit(1);
@@ -117,7 +124,9 @@ void setup_overhead(rzip_control *control)
 		i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) :
 				(level == 6 ? (1 << 25) : (1 << 26)));
 
-		control->overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024);
+		control->overhead = (dictsize * 23 / 2) + (6 * 1024 * 1024) + 16384;
+		/* LZMA spec shows memory requirements as 6MB, not 4MB and state size
+		 * where default is 16KB */
 	} else if (ZPAQ_COMPRESS)
 		control->overhead = 112 * 1024 * 1024;
 }
@@ -205,7 +214,7 @@ bool read_config(rzip_control *control)
 			fprintf(control->msgout, "Using configuration file /etc/lrzip/lrzip.conf\n");
 	}
 	if (fp == NULL)
-		return true;
+		return false;
 
 	/* if we get here, we have a file. read until no more. */
 

util/gitdesc.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+# Peter Hyman, pete@peterhyman.com
+# December 2020
+
+# This program will return commit references based on Tags and Annotated Tags from git describe
+
+usage() {
+cat >&2  <<EOF
+$(basename $0) command [-r]
+all - entire git describe
+commit - commit, omitting v
+tagrev - tag revision count
+major - major release version
+ninor - minor release version
+micro - micro release version
+version - M.mic + [tag release count-HEAD commit]
+-r -- get release tag only
+EOF
+exit 1
+}
+
+# showw message and usage
+die() {
+	echo "$1"
+	usage
+}
+
+# return variables
+# everything, with leading `v' and leading `g' for commits
+describe_tag=
+# abbreviated commit
+commit=
+# count of commits from last tag
+tagrev=
+# major version
+major=
+# minor version
+minor=
+# micro version
+micro=
+# get release or tag?
+tagopt="--tags"
+
+# get whole commit and parse
+# if tagrev > 0 then add it and commit to micro version
+# Expected format is:
+# v#.###-g#######
+init() {
+	describe_tag=$(git describe $tagopt --long --abbrev=7)
+	describe_tag=${describe_tag/v/}
+	describe_tag=${describe_tag/g/}
+	commit=$(echo $describe_tag | cut -d- -f3)
+	tagrev=$(echo $describe_tag | cut -d- -f2)
+	version=$(echo $describe_tag | cut -d- -f1)
+	micro=${version: -2}
+	[ $tagrev -gt 0 ] && micro=$micro-$tagrev-$commit
+	minor=${version: -3:1}
+	major=$(echo $version | cut -d. -f1)
+}
+
+[ ! $(which git) ] && die "Something very wrong: git not found."
+
+[ $# -eq 0 ] && die "Must provide a command and optional argument."
+
+# are we getting a release only?
+if [ $# -eq 2 ]; then
+	if [ $2 = "-r" ]; then
+		tagopt=""
+	else
+		die "Invalid option. Must be -r or nothing."
+	fi
+fi
+
+init
+
+case "$1" in
+	"all" )
+		retval=$describe_tag
+		;;
+	"commit" )
+		retval=$commit
+		;;
+	"tagrev" )
+		retval=$tagrev
+		;;
+	"version" )
+		retval=$version
+		;;
+	"major" )
+		retval=$major
+		;;
+	"minor" )
+		retval=$minor
+		;;
+	"micro" )
+		retval=$micro
+		;;
+	* )
+		die "Invalid command."
+		;;
+esac
+
+echo $retval
+
+exit 0