Dockerfile

@@ -1,14 +0,0 @@
-FROM alpine as builder
-
-RUN apk add --update git autoconf automake libtool gcc musl-dev zlib-dev bzip2-dev lzo-dev coreutils make g++ lz4-dev && \
-    git clone https://github.com/ckolivas/lrzip.git && \
-    cd /lrzip && ./autogen.sh && ./configure && make -j `nproc` && make install
-
-FROM alpine
-
-RUN apk add --update --no-cache lzo libbz2 libstdc++ lz4-dev && \
-    rm -rf /tmp/* /var/tmp/*
-
-COPY --from=builder /usr/local/bin/lrzip /usr/local/bin/lrzip
-
-CMD ["/bin/sh"]

Lrzip.h

@@ -0,0 +1,767 @@
+/*
+   Copyright (C) 2006-2011 Con Kolivas
+   Copyright (C) 2011 Peter Hyman
+   Copyright (C) 1998-2003 Andrew Tridgell
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef LIBLRZIP_H
+#define LIBLRZIP_H
+
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdio.h>
+#ifdef _WIN32
+# include <stddef.h>
+#else
+# include <inttypes.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ @brief LRZIP library
+
+ @mainpage lrzip
+ @version 1.0
+ @date 2011
+
+ @section intro What is LRZIP?
+ 
+ LRZIP is a compression program optimised for large files. The larger the file
+ and the more memory you have, the better the compression advantage this will
+ provide, especially once the files are larger than 100MB. The advantage can
+ be chosen to be either size (much smaller than bzip2) or speed (much faster
+ than bzip2).
+ * @link Lrzip.h LRZIP API @endlink
+ */
+
+/** @file Lrzip.h */
+
+/**
+ * @typedef Lrzip
+ * @brief The overall struct for managing all operations
+ */
+typedef struct Lrzip Lrzip;
+
+/**
+ * @typedef Lrzip_Log_Level
+ * @brief The amount of information to display using logging functions
+ * This enum is used when setting or getting the log level of an #Lrzip
+ * struct. It determines how much information is shown about the current operation,
+ * either in stdout/stderr or using logging callbacks.
+ * @see lrzip_log_level_set()
+ * @see lrzip_log_level_get()
+ */
+typedef enum {
+	LRZIP_LOG_LEVEL_ERROR = 0, /**< Only display errors */
+	LRZIP_LOG_LEVEL_INFO, /**< Display information and errors */
+	LRZIP_LOG_LEVEL_PROGRESS, /**< Display progress updates, information, and errors */
+	LRZIP_LOG_LEVEL_VERBOSE, /**< Display verbose progress updates, information, and errors */
+	LRZIP_LOG_LEVEL_DEBUG /**< Display all possible information */
+} Lrzip_Log_Level;
+
+/**
+ * @typedef Lrzip_Mode
+ * @brief The mode of operation for an #Lrzip struct
+ * This enum is used when setting or getting the operation mode of an #Lrzip
+ * struct. It determines what will happen when lrzip_run() is called.
+ * @see lrzip_mode_set()
+ * @see lrzip_mode_get()
+ */
+typedef enum {
+	LRZIP_MODE_NONE = 0, /**< No operation set */
+	LRZIP_MODE_INFO, /**< Retrieve info about an archive */
+	LRZIP_MODE_TEST, /**< Test an archive's integrity */
+	LRZIP_MODE_DECOMPRESS, /**< Decompress an archive */
+	LRZIP_MODE_COMPRESS_NONE, /**< RZIP preprocess only */
+	LRZIP_MODE_COMPRESS_LZO, /**< Use LZO compression */
+	LRZIP_MODE_COMPRESS_ZLIB, /**< Use ZLIB (GZIP) compression */
+	LRZIP_MODE_COMPRESS_BZIP2, /**< Use BZIP2 compression */
+	LRZIP_MODE_COMPRESS_LZMA, /**< Use LZMA compression */
+	LRZIP_MODE_COMPRESS_ZPAQ /**< Use ZPAQ compression */
+} Lrzip_Mode;
+
+/**
+ * @typedef Lrzip_Flag
+ * @brief The extra params for an #Lrzip struct's operations
+ * This enum is used when setting or getting the flags of an #Lrzip
+ * struct. It determines some of the miscellaneous extra abilities of LRZIP.
+ * @see lrzip_flags_set()
+ * @see lrzip_flags_get()
+ */
+typedef enum {
+	LRZIP_FLAG_REMOVE_SOURCE = (1 << 0), /**< Remove the input file after the operation completes */
+	LRZIP_FLAG_REMOVE_DESTINATION = (1 << 1), /**< Remove matching destination file if it exists */
+	LRZIP_FLAG_KEEP_BROKEN = (1 << 2), /**< Do not remove broken files */
+	LRZIP_FLAG_VERIFY = (1 << 3), /**< Only verify the archive, do not perform any compression/decompression */
+	LRZIP_FLAG_DISABLE_LZO_CHECK = (1 << 4), /**< Disable test to determine if LZO compression will be useful */
+	LRZIP_FLAG_UNLIMITED_RAM = (1 << 5), /**< Use unlimited ram window size for compression */
+	LRZIP_FLAG_ENCRYPT = (1 << 6) /**< Encrypt archive during compression; @see lrzip_pass_cb_set() */
+} Lrzip_Flag;
+
+/**
+ * @typedef Lrzip_Info_Cb
+ * @brief The callback to call when an operation's progress changes
+ * @param data The data param passed in lrzip_info_cb_set()
+ * @param pct The overall operation progress as a percent
+ * @param chunk_pct The current chunk's operation progress as a percent
+ */
+typedef void (*Lrzip_Info_Cb)(void *data, int pct, int chunk_pct);
+/**
+ * @typedef Lrzip_Log_Cb
+ * @brief The callback to call when a log message is to be shown
+ * @param data The data param passed in lrzip_log_cb_set()
+ * @param level The Lrzip_Log_Level of the message
+ * @param line The line in LRZIP code where the message originated
+ * @param file The file in LRZIP code where the message originated
+ * @param func The function in LRZIP code where the message originated
+ * @param format The printf-style format of the message
+ * @param args The matching va_list for @p format
+ */
+typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *func, const char *format, va_list args);
+/**
+ * @typedef Lrzip_Password_Cb
+ * @brief The callback to call for operations requiring a password
+ * @param data The data param passed in lrzip_pass_cb_set()
+ * @param buffer The pre-allocated buffer to write the password into
+ * @param buf_size The size, in bytes, of @p buffer
+ */
+typedef void (*Lrzip_Password_Cb)(void *data, char *buffer, size_t buf_size);
+
+/**
+ * @brief Initialize liblrzip
+ * This function must be called prior to running any other liblrzip
+ * functions to initialize compression algorithms. It does not allocate.
+ * @return true on success, false on failure
+ */
+bool lrzip_init(void);
+
+/**
+ * @brief Create a new #Lrzip struct
+ * Use this function to allocate a new struct for immediate or later use,
+ * optionally setting flags and changing modes at a later time.
+ * @param mode The optional Lrzip_Mode to set, or LRZIP_MODE_NONE to allow
+ * setting a mode later.
+ * @return The new #Lrzip struct, or NULL on failure
+ * @see lrzip_mode_set()
+ */
+Lrzip *lrzip_new(Lrzip_Mode mode);
+
+/**
+ * @brief Free an #Lrzip struct
+ * Use this function to free all memory associated with an existing struct.
+ * @param lr The struct to free
+ */
+void lrzip_free(Lrzip *lr);
+
+/**
+ * @brief Set up an #Lrzip struct using environment settings
+ * Use this function to acquire and utilize settings already existing in
+ * either environment variables or configuration files for LRZIP. For more detailed
+ * information, see the LRZIP manual.
+ * @param lr The struct to configure
+ * @note This function cannot fail.
+ */
+void lrzip_config_env(Lrzip *lr);
+
+/**
+ * @brief Retrieve the operation mode of an #Lrzip struct
+ * @param lr The struct to query
+ * @return The Lrzip_Mode of @p lr, or LRZIP_MODE_NONE on failure
+ */
+Lrzip_Mode lrzip_mode_get(Lrzip *lr);
+
+/**
+ * @brief Set the operation mode of an #Lrzip struct
+ * @param lr The struct to change the mode for
+ * @param mode The Lrzip_Mode to set for @p lr
+ * @return true on success, false on failure
+ */
+bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode);
+
+/**
+ * @brief Set the compression level of an #Lrzip struct
+ * @param lr The struct to change the compression level for
+ * @param level The value, 1-9, to use as the compression level for operations with @p lr
+ * @return true on success, false on failure
+ * @note This function is only valid for compression operations
+ */
+bool lrzip_compression_level_set(Lrzip *lr, unsigned int level);
+
+/**
+ * @brief Get the compression level of an #Lrzip struct
+ * @param lr The struct to get the compression level of
+ * @return The value, 1-9, used as the compression level for operations with @p lr,
+ * or 0 on failure
+ * @note This function is only valid for compression operations
+ */
+unsigned int lrzip_compression_level_get(Lrzip *lr);
+
+/**
+ * @brief Set the operation specific parameters
+ * @param lr The struct to set parameters for
+ * @param flags A bitwise ORed set of Lrzip_Flags
+ * @note This function does not perform any error checking. Any errors in flags
+ * will be determined when lrzip_run() is called.
+ */
+void lrzip_flags_set(Lrzip *lr, unsigned int flags);
+
+/**
+ * @brief Get the operation specific parameters
+ * @param lr The struct to get parameters of
+ * @return A bitwise ORed set of Lrzip_Flags
+ */
+unsigned int lrzip_flags_get(Lrzip *lr);
+
+/**
+ * @brief Set the nice level for operations in a struct
+ * @param lr The struct to set the nice level for
+ * @param nice The value to use when nicing during operations
+ */
+void lrzip_nice_set(Lrzip *lr, int nice);
+
+/**
+ * @brief Get the nice level for operations in a struct
+ * @param lr The struct to get the nice level of
+ * @return The value to use when nicing during operations
+ */
+int lrzip_nice_get(Lrzip *lr);
+
+/**
+ * @brief Explicitly set the number of threads to use during operations
+ * @param lr The struct to set the threads for
+ * @param threads The number of threads to use for operations
+ * @note LRZIP will automatically determine the optimal number of threads to use,
+ * so this function should only be used to specify FEWER than optimal threads.
+ */
+void lrzip_threads_set(Lrzip *lr, unsigned int threads);
+
+/**
+ * @brief Get the number of threads used during operations
+ * @param lr The struct to query
+ * @return The number of threads to use for operations
+ */
+unsigned int lrzip_threads_get(Lrzip *lr);
+
+/**
+ * @brief Set the maximum compression window for operations
+ * @param lr The struct to set the maximum compression window for
+ * @param size The size (in hundreds of MB) to use for the maximum size of compression
+ * chunks.
+ * @note LRZIP will automatically determine the optimal maximum compression window to use,
+ * so this function should only be used to specify a LOWER value.
+ */
+void lrzip_compression_window_max_set(Lrzip *lr, int64_t size);
+
+/**
+ * @brief Get the maximum compression window for operations
+ * @param lr The struct to query
+ * @return The size (in hundreds of MB) to use for the maximum size of compression
+ * chunks.
+ */
+int64_t lrzip_compression_window_max_get(Lrzip *lr);
+
+/**
+ * @brief Return the size of the stream queue in a struct
+ * This function returns the current count of streams added for processing
+ * using lrzip_file_add. It always returns instantly.
+ * @param lr The struct to query
+ * @return The current number of streams in the queue
+ */
+unsigned int lrzip_files_count(Lrzip *lr);
+
+/**
+ * @brief Return the size of the file queue in a struct
+ * This function returns the current count of files added for processing
+ * using lrzip_filename_add. It always returns instantly.
+ * @param lr The struct to query
+ * @return The current number of files in the queue
+ */
+unsigned int lrzip_filenames_count(Lrzip *lr);
+
+/**
+ * @brief Return the array of the stream queue in a struct
+ * This function returns the current queue of streams added for processing
+ * using lrzip_file_add. It always returns instantly.
+ * @param lr The struct to query
+ * @return The current stream queue
+ */
+FILE **lrzip_files_get(Lrzip *lr);
+
+/**
+ * @brief Return the array of the filename queue in a struct
+ * This function returns the current queue of files added for processing
+ * using lrzip_filename_add. It always returns instantly.
+ * @param lr The struct to query
+ * @return The current filename queue
+ */
+char **lrzip_filenames_get(Lrzip *lr);
+
+/**
+ * @brief Add a stream (FILE) to the operation queue
+ * This function adds a stream to the input queue. Each time lrzip_run()
+ * is called, it will run the current operation (specified by the Lrzip_Mode)
+ * on either a stream or file in the queue.
+ * @param lr The struct
+ * @param file The stream descriptor to queue
+ * @return true on success, false on failure
+ * @note The file queue will be fully processed prior to beginning processing
+ * the stream queue.
+ * @warning Any streams added to this queue MUST NOT be closed until they have
+ * either been processed or removed from the queue!
+ */
+bool lrzip_file_add(Lrzip *lr, FILE *file);
+
+/**
+ * @brief Remove a stream from the operation queue
+ * This function removes a previously added stream from the operation queue by
+ * iterating through the queue and removing the stream if found.
+ * @param lr The struct
+ * @param file The stream to remove
+ * @return true only on successful removal, else false
+ */
+bool lrzip_file_del(Lrzip *lr, FILE *file);
+
+/**
+ * @brief Pop the current head of the stream queue
+ * This function is used to remove the current head of the stream queue. It can be called
+ * immediately following any lrzip_run() stream operation to remove the just-processed stream. This
+ * function modifies the stream queue array, reordering and updating the index count.
+ * @param lr The struct to pop the stream queue of
+ * @return The stream removed from the queue, or NULL on failure
+ */
+FILE *lrzip_file_pop(Lrzip *lr);
+
+/**
+ * @brief Clear the stream queue
+ * This function is used to free and reset the stream queue. The streams
+ * themselves are untouched.
+ * @param lr The struct
+ */
+void lrzip_files_clear(Lrzip *lr);
+
+/**
+ * @brief Add a file to the operation queue
+ * This function adds a file to the input queue. Each time lrzip_run()
+ * is called, it will run the current operation (specified by the Lrzip_Mode)
+ * on either a stream or file in the queue.
+ * @param lr The struct
+ * @param file The file (by absolute path) to queue
+ * @return true on success, false on failure
+ * @note The file queue will be fully processed prior to beginning processing
+ * the stream queue.
+ */
+bool lrzip_filename_add(Lrzip *lr, const char *file);
+
+/**
+ * @brief Remove a filename from the operation queue
+ * This function removes a previously added filename from the operation queue by
+ * iterating through the queue and removing the filename if found.
+ * @param lr The struct
+ * @param file The file to remove
+ * @return true only on successful removal, else false
+ */
+bool lrzip_filename_del(Lrzip *lr, const char *file);
+
+/**
+ * @brief Pop the current head of the file queue
+ * This function is used to remove the current head of the file queue. It can be called
+ * immediately following any lrzip_run() file operation to remove the just-processed file. This
+ * function modifies the file queue array, reordering and updating the index count.
+ * @param lr The struct to pop the filename queue of
+ * @return The filename removed from the queue, or NULL on failure
+ */
+const char *lrzip_filename_pop(Lrzip *lr);
+
+/**
+ * @brief Clear the file queue
+ * This function is used to free and reset the file queue.
+ * @param lr The struct
+ */
+void lrzip_filenames_clear(Lrzip *lr);
+
+/**
+ * @brief Set the default suffix for LRZIP compression operations
+ * This function is used to change the default ".lrz" suffix for operations
+ * to @p suffix.
+ * @param lr The struct
+ * @param suffix The suffix to use for compression operations
+ */
+void lrzip_suffix_set(Lrzip *lr, const char *suffix);
+
+/**
+ * @brief Get the default suffix for LRZIP compression operations
+ * @param lr The struct
+ * @return The suffix to use for compression operations, or NULL on failure
+ */
+const char *lrzip_suffix_get(Lrzip *lr);
+
+/**
+ * @brief Set the output directory for operations
+ * This function can be used to set the output directory for operations.
+ * Files will be stored according to their basename and lrzip suffix where
+ * applicable.
+ * @param lr The struct
+ * @param dir The absolute path of the output directory
+ */
+void lrzip_outdir_set(Lrzip *lr, const char *dir);
+
+/**
+ * @brief Get the output directory for operations
+ * @param lr The struct
+ * @return The previously set output directory
+ */
+const char *lrzip_outdir_get(Lrzip *lr);
+
+/**
+ * @brief Set the output stream for operations
+ * This function can be used to set the output stream for operations.
+ * Raw data will be written to this stream for the duration of lrzip_run().
+ * @param lr The struct
+ * @param file The stream to write to
+ * @warning @p file is NOT created by this library and must be opened by the user!
+ */
+void lrzip_outfile_set(Lrzip *lr, FILE *file);
+
+/**
+ * @brief Get the output stream for operations
+ * @param lr The struct
+ * @return The previously set output stream
+ */
+FILE *lrzip_outfile_get(Lrzip *lr);
+
+/**
+ * @brief Set the output file for operations
+ * This function can be used to set the output file for operations.
+ * Raw data will be written to the file with this name for the duration of lrzip_run().
+ * @param lr The struct
+ * @param file The name of the file to write to
+ */
+void lrzip_outfilename_set(Lrzip *lr, const char *file);
+
+/**
+ * @brief Get the output filename for operations
+ * @param lr The struct
+ * @return The previously set output filename
+ */
+const char *lrzip_outfilename_get(Lrzip *lr);
+
+/**
+ * @brief Retrieve the MD5 digest of an LRZIP file
+ * Use this function after calling lrzip_run() to retrieve the digest of
+ * the processed archive.
+ * @param lr The struct having run an operation
+ * @return The MD5 digest of the operation's associated archive
+ * @note The return value of this function will change after each operation
+ */
+const unsigned char *lrzip_md5digest_get(Lrzip *lr);
+
+/**
+ * @brief Run the current operation
+ * This function is called when all necessary parameters have been set for an operation.
+ * The calling thread will then block until the operation has fully completed, writing
+ * output using logging and progress callbacks and calling password callbacks as required.
+ * @param lr The struct to run an operation with
+ * @return true if the operation successfully completed, else false
+ */
+bool lrzip_run(Lrzip *lr);
+
+/**
+ * @brief Set the logging level
+ * @param lr The struct
+ * @param level The #Lrzip_Log_Level to use
+ */
+void lrzip_log_level_set(Lrzip *lr, int level);
+
+/**
+ * @brief Get the logging level
+ * @param lr The struct to query
+ * @return The #Lrzip_Log_Level of @p lr
+ */
+int lrzip_log_level_get(Lrzip *lr);
+
+/**
+ * @brief Set a logging callback for use with all operations
+ * This function sets an Lrzip_Log_Cb which will be called any time logging
+ * output is to be displayed. The callback will be called as many times as the #Lrzip_Log_Level
+ * requires.
+ * @param lr The struct
+ * @param cb The callback
+ * @param log_data The data param to use in the logging callback
+ */
+void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data);
+
+/**
+ * @brief Redirect stdout log messages to another stream
+ * This function sends any logging messages which would normally go to stdout into another stream.
+ * Useful for when stdout is the target set by lrzip_outfile_set().
+ * @param lr The struct
+ * @param out The stream to use instead of stdout
+ */
+void lrzip_log_stdout_set(Lrzip *lr, FILE *out);
+
+/**
+ * @brief Return the stream currently used as stdout
+ * @param lr The struct to query
+ * @return A stream where stdout messages will be sent, NULL on failure
+ */
+FILE *lrzip_log_stdout_get(Lrzip *lr);
+
+/**
+ * @brief Redirect stderr log messages to another stream
+ * This function sends any logging messages which would normally go to stderr into another stream.
+ * @param lr The struct
+ * @param err The stream to use instead of stderr
+ */
+void lrzip_log_stderr_set(Lrzip *lr, FILE *err);
+
+/**
+ * @brief Return the stream currently used as stderr
+ * @param lr The struct to query
+ * @return A stream where stderr messages will be sent, NULL on failure
+ */
+FILE *lrzip_log_stderr_get(Lrzip *lr);
+
+/**
+ * @brief Set a password callback for use with all operations
+ * This function sets an Lrzip_Password_Cb which will be used when working with encrypted
+ * LRZIP archives. It will be called both when compressing and decompressing archives.
+ * @param lr The struct
+ * @param cb The callback to set
+ * @param data The data param to use in the password callback
+ */
+void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data);
+
+/**
+ * @brief Set an info callback for use with all operations
+ * This function sets an Lrzip_Info_Cb which will be called any time there is a
+ * progress update in an operation.
+ * @param lr The struct
+ * @param cb The callback to set
+ * @param data The data param to use in the info callback
+ */
+void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data);
+
+/**
+ * @brief Quick setup for performing a decompression
+ * This function performs all the required allocations and sets necessary parameters
+ * to decompress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len);
+
+/**
+ * @brief Quick setup for performing a compression
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param mode The compression mode to use
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level);
+
+/**
+ * @brief Quick setup for performing a compression using LZMA
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_compress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, 7); }
+
+/**
+ * @brief Quick setup for performing a compression using LZO
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_lcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, 7); }
+
+/**
+ * @brief Quick setup for performing a compression using ZLIB (GZIP)
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_gcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, 7); }
+
+/**
+ * @brief Quick setup for performing a compression using ZPAQ
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_zcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, 7); }
+
+/**
+ * @brief Quick setup for performing a compression using BZIP
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_bcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, 7); }
+
+/**
+ * @brief Quick setup for performing RZIP preprocessing
+ * This function performs all the required allocations and sets necessary parameters
+ * to preprocess @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @return true on success, else false
+ */
+static inline bool lrzip_rcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, 7); }
+
+/**
+ * @brief Quick setup for performing a compression using LZMA and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_compress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, compress_level); }
+
+/**
+ * @brief Quick setup for performing a compression using LZO and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_lcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, compress_level); }
+
+/**
+ * @brief Quick setup for performing a compression using ZLIB (GZIP) and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_gcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, compress_level); }
+
+/**
+ * @brief Quick setup for performing a compression using ZPAQ and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_zcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, compress_level); }
+
+/**
+ * @brief Quick setup for performing a compression using BZIP and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to compress @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_bcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, compress_level); }
+
+/**
+ * @brief Quick setup for performing RZIP preprocessing and a user-defined compression level
+ * This function performs all the required allocations and sets necessary parameters
+ * to preprocess @p source to @p dest. No extra functions are necessary to call, and
+ * this function will block until it completes.
+ * @param dest A pointer to the LRZIP-allocated destination buffer
+ * @param dest_len A pointer to the length of @p dest
+ * @param source The allocated source buffer to read from
+ * @param source_len The length of @p source
+ * @param compress_level The value, 1-9, to use as a compression level
+ * @return true on success, else false
+ */
+static inline bool lrzip_rcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level)
+{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, compress_level); }
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+

Makefile.am

@@ -34,6 +34,7 @@ lrztar_SCRIPTS = lrztar
 noinst_LTLIBRARIES = libtmplrzip.la
 libtmplrzip_la_SOURCES = \
   lrzip_private.h \
+  liblrzip_private.h \
   lrzip.c \
   lrzip_core.h \
   rzip.h \
@@ -56,6 +57,13 @@ libtmplrzip_la_SOURCES = \
 libtmplrzip_la_LIBADD = lzma/C/liblzma.la
 
 
+lib_LTLIBRARIES = liblrzip.la
+liblrzip_la_SOURCES = \
+  liblrzip.c \
+  liblrzip_private.h
+nodist_EXTRA_liblrzip_la_SOURCES = dummy.cxx
+liblrzip_la_LIBADD = libtmplrzip.la
+
 bin_PROGRAMS = lrzip
 lrzip_SOURCES = \
   main.c
@@ -66,6 +74,13 @@ if STATIC
   lrzip_LDFLAGS = -all-static
 endif
 
+noinst_PROGRAMS = decompress_demo liblrzip_demo
+decompress_demo_SOURCES = decompress_demo.c
+decompress_demo_LDADD = liblrzip.la
+
+liblrzip_demo_SOURCES = liblrzip_demo.c
+liblrzip_demo_LDADD = liblrzip.la
+
 dist_doc_DATA = \
   AUTHORS \
   BUGS \
@@ -76,9 +91,14 @@ dist_doc_DATA = \
   TODO \
   WHATS-NEW
 
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = lrzip.pc
+
+lrzip_HEADERS = Lrzip.h
 lrzipdir = $(includedir)
 
 EXTRA_DIST = \
+		lrzip.pc.in \
 		lrztar \
 		description-pak \
 		autogen.sh \

README.md

@@ -61,7 +61,7 @@ Two different ways of doing this:
 
 Stable: Packaged tarball that is known to work:
 
-Go to <https://github.com/ckolivas/lrzip/releases> and download the `tar.gz`
+Go to <https://github.com/ckolivas/lrzip/releases> and downlaod the `tar.gz`
 file from the top. `cd` to the directory you downloaded, and use `tar xvzf lrzip-X.X.tar.gz`
 to extract the files (don't forget to replace `X.X` with the correct version). Finally, cd
 into the directory you just extracted.

WHATS-NEW

@@ -1,31 +1,7 @@
-lrzip-0.651
-
-Remove redundant files
-Revert locale dependent output
-Add warnings for low memory and threads
+Changelog will be moved to git entirely from this point forward.
 
 lrzip-0.650
 
-Minor optimisations.
-Exit status fixes.
-Update and beautify information output.
-Fix Android build.
-Enable MD5 on Apple build.
-Deprecate and remove liblrzip which was unused and at risk of bitrot.
-Fix failures with compressing to STDOUT with inadequate memory.
-Fix possible race conditions.
-Fix memory leaks.
-Fix -q to only hide progress.
-Add -Q option for very quiet.
-
-
-lrzip-0.641
-
-Critical bugfix for broken lz4 testing which would prevent secondary
-compression from being enabled.
-
-lrzip-0.640
-
 Numerous bugfixes and build fixes.
 lz4 now used for compressibility testing (only) making lz4-dev a build
 requirement.

configure.ac

@@ -2,7 +2,7 @@
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_define([v_maj], [0])
 m4_define([v_min], [6])
-m4_define([v_mic], [51])
+m4_define([v_mic], [40])
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_define([v_v], m4_join([], v_min, v_mic))
 m4_define([v_ver], [v_maj.v_v])
@@ -15,7 +15,7 @@ m4_define([lt_age], v_min)
 
 dnl Process this file with autoconf to produce a configure script.
 AC_INIT([lrzip],[v_ver],[kernel@kolivas.org])
-AC_PREREQ([2.71])
+AC_PREREQ([2.59])
 AC_CONFIG_SRCDIR([configure.ac])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
@@ -24,7 +24,7 @@ AM_INIT_AUTOMAKE([1.6 dist-bzip2 foreign subdir-objects])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_USE_SYSTEM_EXTENSIONS
 
-LT_INIT
+AC_PROG_LIBTOOL
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 ##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##--##
 m4_ifdef([v_rev], , [m4_define([v_rev], [0])])
@@ -53,14 +53,17 @@ AC_SUBST(SHELL)
 AC_SYS_LARGEFILE
 AC_FUNC_FSEEKO
 AC_FUNC_ALLOCA
-
+AC_PROG_CC_C99
+AS_IF([test x"$ac_cv_prog_cc_c99" = x"no"],
+	AC_MSG_ERROR([C compiler does not support C99], 1))
 AC_CHECK_PROG([HAVE_POD2MAN], [pod2man], [yes])
 AS_IF([test "$HAVE_POD2MAN" != "yes"],
 	AC_MSG_FAILURE([pod2man is needed to generate manual from POD]))
 
+
 AC_ARG_ENABLE(
 		asm,
-		[AS_HELP_STRING([--enable-asm],[Enable native Assembly code])],
+		[AC_HELP_STRING([--enable-asm],[Enable native Assembly code])],
 		ASM=$enableval,
 		ASM=yes
              )
@@ -74,7 +77,7 @@ fi
 
 static=no
 AC_ARG_ENABLE([static-bin],
-	[AS_HELP_STRING([--enable-static-bin],[Build statically linked binary @<:@default=no@:>@])],
+	[AC_HELP_STRING([--enable-static-bin],[Build statically linked binary @<:@default=no@:>@])],
 	[static=$enableval]
 	)
 AM_CONDITIONAL([STATIC], [test x"$static" = x"yes"])
@@ -147,6 +150,7 @@ EFL_CHECK_DOXYGEN([build_doc="yes"], [build_doc="no"])
 
 AC_CONFIG_FILES([
 Makefile
+lrzip.pc
 lzma/Makefile
 lzma/C/Makefile
 lzma/ASM/x86/Makefile

doc/README.benchmarks

@@ -45,7 +45,7 @@ purpose compressor at the moment:
 
 These are benchmarks performed on a 2.53Ghz dual core Intel Core2 with 4GB ram
 using lrzip v0.5.1. Note that it was running with a 32 bit userspace so only
-2GB addressing was possible. However the benchmark was run with the -U option
+2GB addressing was posible. However the benchmark was run with the -U option
 allowing the whole file to be treated as one large compression window.
 
 Tarball of 6 consecutive kernel trees.

doc/lrzip.conf.example

@@ -13,7 +13,7 @@
 # Use -U setting, Unlimited ram. Yes or No
 # UNLIMITED = NO
 # Compression Method, rzip, gzip, bzip2, lzo, or lzma (default), or zpaq. (-n -g -b -l --lzma -z)
-# May be overridden by command line compression choice.
+# May be overriden by command line compression choice.
 # COMPRESSIONMETHOD = lzma
 # Perform LZO Test. Default = YES (-T )
 # LZOTEST = NO

liblrzip.c

@@ -0,0 +1,742 @@
+/*
+   Copyright (C) 2012-2016,2018 Con Kolivas
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <liblrzip_private.h>
+
+#ifdef HAVE_SYS_TIME_H
+# include <sys/time.h>
+#endif
+#ifdef HAVE_SYS_RESOURCE_H
+# include <sys/resource.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+
+/* needed for CRC routines */
+#include "lzma/C/7zCrc.h"
+#include "util.h"
+#include "lrzip_core.h"
+#include "rzip.h"
+
+#if defined(__APPLE__) || defined(__FreeBSD__)
+# define fmemopen(s, len, modes) fake_fmemopen((s), (len), (modes))
+static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode)
+{
+	FILE *in;
+
+	in = tmpfile();
+	if (!in)
+		return NULL;
+	if (fwrite(buf, buflen, 1, in) != 1) {
+		fclose(in);
+		return NULL;
+	}
+	rewind(in);
+	return in;
+}
+#endif
+
+static void liblrzip_index_update(size_t x, size_t *idx, void **queue)
+{
+	for (; x < *idx; x++)
+		queue[x] = queue[x + 1];
+	(*idx)--;
+}
+
+static bool liblrzip_setup_flags(Lrzip *lr)
+{
+	if (!lr)
+		return false;
+#define MODE_CHECK(X) \
+	case LRZIP_MODE_COMPRESS_##X: \
+	lr->control->flags ^= FLAG_NOT_LZMA; \
+	lr->control->flags |= FLAG_##X##_COMPRESS; \
+	break
+
+
+	switch (lr->mode) {
+	case LRZIP_MODE_DECOMPRESS:
+		lr->control->flags |= FLAG_DECOMPRESS;
+		break;
+	case LRZIP_MODE_TEST:
+		lr->control->flags |= FLAG_TEST_ONLY;
+		break;
+	case LRZIP_MODE_INFO:
+		lr->control->flags |= FLAG_INFO;
+		break;
+	case LRZIP_MODE_COMPRESS_NONE:
+		lr->control->flags ^= FLAG_NOT_LZMA;
+		lr->control->flags |= FLAG_NO_COMPRESS;
+		break;
+	case LRZIP_MODE_COMPRESS_LZMA:
+		lr->control->flags ^= FLAG_NOT_LZMA;
+		break;
+	MODE_CHECK(LZO);
+	MODE_CHECK(BZIP2);
+	MODE_CHECK(ZLIB);
+	MODE_CHECK(ZPAQ);
+#undef MODE_CHECK
+	default:
+		return false;
+	}
+	setup_overhead(lr->control);
+	if (lr->flags & LRZIP_FLAG_VERIFY) {
+		lr->control->flags |= FLAG_CHECK;
+		lr->control->flags |= FLAG_HASH;
+	}
+	if (lr->flags & LRZIP_FLAG_REMOVE_DESTINATION)
+		lr->control->flags |= FLAG_FORCE_REPLACE;
+	if (lr->flags & LRZIP_FLAG_REMOVE_SOURCE)
+		lr->control->flags &= ~FLAG_KEEP_FILES;
+	if (lr->flags & LRZIP_FLAG_KEEP_BROKEN)
+		lr->control->flags |= FLAG_KEEP_BROKEN;
+	if (lr->flags & LRZIP_FLAG_DISABLE_LZO_CHECK)
+		lr->control->flags &= ~FLAG_THRESHOLD;
+	if (lr->flags & LRZIP_FLAG_UNLIMITED_RAM)
+		lr->control->flags |= FLAG_UNLIMITED;
+	if (lr->flags & LRZIP_FLAG_ENCRYPT)
+		lr->control->flags |= FLAG_ENCRYPT;
+	if (lr->control->log_level > 0) {
+		lr->control->flags |= FLAG_SHOW_PROGRESS;
+		if (lr->control->log_level > 1) {
+			lr->control->flags |= FLAG_VERBOSITY;
+			if (lr->control->log_level > 2)
+				lr->control->flags |= FLAG_VERBOSITY_MAX;
+		}
+	} else lr->control->flags ^= (FLAG_VERBOSE | FLAG_SHOW_PROGRESS);
+	return true;
+}
+
+
+bool lrzip_init(void)
+{
+	/* generate crc table */
+	CrcGenerateTable();
+	return true;
+}
+
+void lrzip_config_env(Lrzip *lr)
+{
+	const char *eptr;
+	/* Get Preloaded Defaults from lrzip.conf
+	 * Look in ., $HOME/.lrzip/, /etc/lrzip.
+	 * If LRZIP=NOCONFIG is set, then ignore config
+	 */
+	eptr = getenv("LRZIP");
+	if (!eptr)
+		read_config(lr->control);
+	else if (!strstr(eptr,"NOCONFIG"))
+		read_config(lr->control);
+}
+
+void lrzip_free(Lrzip *lr)
+{
+	size_t x;
+
+	if ((!lr) || (!lr->infilename_buckets))
+		return;
+	rzip_control_free(lr->control);
+	for (x = 0; x < lr->infilename_idx; x++)
+		dealloc(lr->infilenames[x]);
+	dealloc(lr->infilenames);
+	dealloc(lr->infiles);
+	dealloc(lr);
+}
+
+Lrzip *lrzip_new(Lrzip_Mode mode)
+{
+	Lrzip *lr;
+
+	lr = calloc(1, sizeof(Lrzip));
+	if (!lr)
+		return NULL;
+	lr->control = calloc(1, sizeof(rzip_control));
+	if (!lr->control)
+		goto error;
+	if (!initialize_control(lr->control))
+		goto error;
+	lr->mode = mode;
+	lr->control->library_mode = 1;
+	return lr;
+error:
+	lrzip_free(lr);
+	return NULL;
+}
+
+Lrzip_Mode lrzip_mode_get(Lrzip *lr)
+{
+	if (!lr)
+		return LRZIP_MODE_NONE;
+	return lr->mode;
+}
+
+bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode)
+{
+	if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ))
+		return false;
+	lr->mode = mode;
+	return true;
+}
+
+bool lrzip_compression_level_set(Lrzip *lr, unsigned int level)
+{
+	if ((!lr) || (!level) || (level > 9))
+		return false;
+	lr->control->compression_level = level;
+	return true;
+}
+
+unsigned int lrzip_compression_level_get(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->control->compression_level;
+}
+
+void lrzip_flags_set(Lrzip *lr, unsigned int flags)
+{
+	if (!lr)
+		return;
+	lr->flags = flags;
+}
+
+unsigned int lrzip_flags_get(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->flags;
+}
+
+void lrzip_nice_set(Lrzip *lr, int nice)
+{
+	if ((!lr) || (nice < -19) || (nice > 20))
+		return;
+	lr->control->nice_val = nice;
+}
+
+int lrzip_nice_get(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->control->nice_val;
+}
+
+void lrzip_threads_set(Lrzip *lr, unsigned int threads)
+{
+	if ((!lr) || (!threads))
+		return;
+	lr->control->threads = threads;
+}
+
+unsigned int lrzip_threads_get(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->control->threads;
+}
+
+void lrzip_compression_window_max_set(Lrzip *lr, int64_t size)
+{
+	if (!lr)
+		return;
+	lr->control->window = size;
+}
+
+int64_t lrzip_compression_window_max_get(Lrzip *lr)
+{
+	if (!lr)
+		return -1;
+	return lr->control->window;
+}
+
+unsigned int lrzip_files_count(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->infile_idx;
+}
+
+unsigned int lrzip_filenames_count(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->infilename_idx;
+}
+
+FILE **lrzip_files_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->infiles;
+}
+
+char **lrzip_filenames_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->infilenames;
+}
+
+bool lrzip_file_add(Lrzip *lr, FILE *file)
+{
+	if ((!lr) || (!file))
+		return false;
+	if (lr->infilenames)
+		return false;
+	if (!lr->infile_buckets) {
+		/* no files added */
+		lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*));
+		lr->infile_buckets++;
+	} else if (lr->infile_idx == INFILE_BUCKET_SIZE * lr->infile_buckets + 1) {
+		/* all buckets full, create new bucket */
+		FILE **tmp;
+
+		tmp = realloc(lr->infiles, (++lr->infile_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*));
+		if (!tmp)
+			return false;
+		lr->infiles = tmp;
+	}
+
+	lr->infiles[lr->infile_idx++] = file;
+	return true;
+}
+
+bool lrzip_file_del(Lrzip *lr, FILE *file)
+{
+	size_t x;
+
+	if ((!lr) || (!file))
+		return false;
+	if (!lr->infile_buckets)
+		return true;
+
+	for (x = 0; x <= lr->infile_idx + 1; x++) {
+		if (!lr->infiles[x])
+			return true; /* not found */
+		if (lr->infiles[x] != file)
+			continue; /* not a match */
+		break;
+	}
+	/* update index */
+	liblrzip_index_update(x, &lr->infile_idx, (void**)lr->infiles);
+	return true;
+}
+
+FILE *lrzip_file_pop(Lrzip *lr)
+{
+	FILE *ret;
+	if ((!lr) || (!lr->infile_buckets))
+		return NULL;
+	ret = lr->infiles[0];
+	lrzip_file_del(lr, ret);
+	return ret;
+}
+
+void lrzip_files_clear(Lrzip *lr)
+{
+	if ((!lr) || (!lr->infile_buckets))
+		return;
+	dealloc(lr->infiles);
+	lr->infiles = NULL;
+}
+
+bool lrzip_filename_add(Lrzip *lr, const char *file)
+{
+	struct stat st;
+
+	if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-")))
+		return false;
+	if (lr->infiles)
+		return false;
+	if (stat(file, &st))
+		return false;
+	if (S_ISDIR(st.st_mode))
+		return false;
+
+	if (!lr->infilename_buckets) {
+		/* no files added */
+		lr->infilenames = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*));
+		lr->infilename_buckets++;
+	} else if (lr->infilename_idx == INFILE_BUCKET_SIZE * lr->infilename_buckets + 1) {
+		/* all buckets full, create new bucket */
+		char **tmp;
+
+		tmp = realloc(lr->infilenames, (++lr->infilename_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*));
+		if (!tmp)
+			return false;
+		lr->infilenames = tmp;
+	}
+
+	lr->infilenames[lr->infilename_idx++] = strdup(file);
+	return true;
+}
+
+bool lrzip_filename_del(Lrzip *lr, const char *file)
+{
+	size_t x;
+
+	if ((!lr) || (!file) || (!file[0]))
+		return false;
+	if (!lr->infilename_buckets)
+		return true;
+
+	for (x = 0; x <= lr->infilename_idx + 1; x++) {
+		if (!lr->infilenames[x])
+			return true; /* not found */
+		if (strcmp(lr->infilenames[x], file))
+			continue; /* not a match */
+		dealloc(lr->infilenames[x]);
+		break;
+	}
+	/* update index */
+	liblrzip_index_update(x, &lr->infilename_idx, (void**)lr->infilenames);
+	return true;
+}
+
+const char *lrzip_filename_pop(Lrzip *lr)
+{
+	static char buf[4096];
+	if ((!lr) || (!lr->infilename_buckets))
+		return NULL;
+	strcat(buf, lr->infilenames[0]);
+	lrzip_filename_del(lr, buf);
+	return &buf[0];
+}
+
+void lrzip_filenames_clear(Lrzip *lr)
+{
+	size_t x;
+	if ((!lr) || (!lr->infilename_buckets))
+		return;
+	for (x = 0; x < lr->infilename_idx; x++)
+		dealloc(lr->infilenames[x]);
+	dealloc(lr->infilenames);
+	lr->infilenames = NULL;
+}
+
+void lrzip_suffix_set(Lrzip *lr, const char *suffix)
+{
+	if ((!lr) || (!suffix) || (!suffix[0]))
+		return;
+	dealloc(lr->control->suffix);
+	lr->control->suffix = strdup(suffix);
+}
+
+const char *lrzip_suffix_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->suffix;
+}
+
+void lrzip_outdir_set(Lrzip *lr, const char *dir)
+{
+	const char *slash;
+	char *buf;
+	size_t len;
+	if ((!lr) || (!dir) || (!dir[0]))
+		return;
+	dealloc(lr->control->outdir);
+	slash = strrchr(dir, '/');
+	if (slash && (slash[1] == 0)) {
+		lr->control->outdir = strdup(dir);
+		return;
+	}
+	len = strlen(dir);
+	buf = malloc(len + 2);
+	if (!buf)
+		return;
+	memcpy(buf, dir, len);
+	buf[len] = '/';
+	buf[len + 1] = 0;
+	lr->control->outdir = buf;
+}
+
+const char *lrzip_outdir_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->outdir;
+}
+
+void lrzip_outfile_set(Lrzip *lr, FILE *file)
+{
+	if ((!lr) || (file && (file == stderr)))
+		return;
+	if (lr->control->outname)
+		return;
+	lr->control->outFILE = file;
+}
+
+FILE *lrzip_outfile_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->outFILE;
+}
+
+void lrzip_outfilename_set(Lrzip *lr, const char *file)
+{
+	if ((!lr) || (file && (!file[0])))
+		return;
+	if (lr->control->outFILE)
+		return;
+	if (lr->control->outname && file && (!strcmp(lr->control->outname, file)))
+		return;
+	dealloc(lr->control->outname);
+	lr->control->outname = file ? strdup(file) : NULL;
+}
+
+const char *lrzip_outfilename_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->outname;
+}
+
+const unsigned char *lrzip_md5digest_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->md5_resblock;
+}
+
+bool lrzip_run(Lrzip *lr)
+{
+	struct timeval start_time, end_time;
+	rzip_control *control;
+	double seconds,total_time; // for timers
+	int hours,minutes;
+
+	if (!liblrzip_setup_flags(lr))
+		return false;
+	control = lr->control;
+
+	if ((!lr->infile_idx) && (!lr->infilename_idx))
+		return false;
+	if (lr->control->outFILE) {
+		if (lr->control->outFILE == lr->control->msgout)
+			lr->control->msgout = stderr;
+		lr->control->flags |= FLAG_STDOUT;
+		register_outputfile(lr->control, lr->control->msgout);
+	}
+
+	if (lr->infilenames)
+		lr->control->infile = lr->infilenames[0];
+	else {
+		lr->control->inFILE = lr->infiles[0];
+        if ( lr->infiles[0] == stdin )
+            control->flags |= FLAG_STDIN;
+	}
+
+	if ((!STDOUT) && (!lr->control->msgout)) lr->control->msgout = stdout;
+	register_outputfile(lr->control, lr->control->msgout);
+
+	setup_ram(lr->control);
+
+	gettimeofday(&start_time, NULL);
+
+	if (ENCRYPT && (!lr->control->pass_cb)) {
+		print_err("No password callback set!\n");
+		return false;
+	}
+
+	if (DECOMPRESS || TEST_ONLY) {
+		if (!decompress_file(lr->control))
+			return false;
+	} else if (INFO) {
+		if (!get_fileinfo(lr->control))
+			return false;
+	} else if (!compress_file(lr->control))
+		return false;
+
+	/* compute total time */
+	gettimeofday(&end_time, NULL);
+	total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) -
+	             (start_time.tv_sec + (double)start_time.tv_usec / 1000000);
+	hours = (int)total_time / 3600;
+	minutes = (int)(total_time / 60) % 60;
+	seconds = total_time - hours * 3600 - minutes * 60;
+	if (!INFO)
+		print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
+
+	return true;
+}
+
+void lrzip_log_level_set(Lrzip *lr, int level)
+{
+	if (!lr)
+		return;
+	lr->control->log_level = level;
+}
+
+int lrzip_log_level_get(Lrzip *lr)
+{
+	if (!lr)
+		return 0;
+	return lr->control->log_level;
+}
+
+void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data)
+{
+	if (!lr)
+		return;
+	lr->control->log_cb = cb;
+	lr->control->log_data = log_data;
+}
+
+void lrzip_log_stdout_set(Lrzip *lr, FILE *out)
+{
+	if (!lr)
+		return;
+	lr->control->msgout = out;
+}
+
+FILE *lrzip_log_stdout_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->msgout;
+}
+
+void lrzip_log_stderr_set(Lrzip *lr, FILE *err)
+{
+	if (!lr)
+		return;
+	lr->control->msgerr = err;
+}
+
+FILE *lrzip_log_stderr_get(Lrzip *lr)
+{
+	if (!lr)
+		return NULL;
+	return lr->control->msgerr;
+}
+
+void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data)
+{
+	if (!lr)
+		return;
+	lr->control->pass_cb = cb;
+	lr->control->pass_data = data;
+}
+
+void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data)
+{
+	if (!lr)
+		return;
+	lr->control->info_cb = cb;
+	lr->control->info_data = data;
+}
+
+bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level)
+{
+	FILE *s = NULL, *d = NULL;
+	Lrzip *lr = NULL;
+	bool ret = false;
+	struct stat st;
+	int fd;
+
+	if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE))
+		goto out;
+
+	lrzip_init();
+	if (!mode) mode = LRZIP_MODE_COMPRESS_LZMA;
+	lr = lrzip_new(mode);
+	if (!lr)
+		goto out;
+	lrzip_config_env(lr);
+
+	s = fmemopen((void*)source, source_len, "r");
+	d = tmpfile();
+	if ((!s) || (!d))
+		goto out;
+
+	if (!lrzip_file_add(lr, s))
+		goto out;
+	lrzip_outfile_set(lr, d);
+	if (!lrzip_compression_level_set(lr, compress_level))
+		goto out;
+	if (!lrzip_run(lr))
+		goto out;
+
+	fd = fileno(d);
+	if (fstat(fd, &st))
+		goto out;
+	*dest_len = st.st_size;
+	if (unlikely((i64)fread(dest, sizeof(char), st.st_size, d) != st.st_size))
+		goto out;
+	if (unlikely(ferror(d)))
+		goto out;
+	ret = true;
+
+out:
+	if (s) fclose(s);
+	if (d) fclose(d);
+	lrzip_free(lr);
+	return ret;
+}
+
+
+bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len)
+{
+	FILE *s = NULL, *d = NULL;
+	Lrzip *lr = NULL;
+	bool ret = false;
+	struct stat st;
+	int fd;
+
+	if ((!dest) || (!dest_len) || (!source) || (!source_len))
+		goto out;
+
+	lrzip_init();
+	lr = lrzip_new(LRZIP_MODE_DECOMPRESS);
+	if (!lr)
+		goto out;
+	lrzip_config_env(lr);
+
+	s = fmemopen((void*)source, source_len, "r");
+	d = tmpfile();
+	if ((!s) || (!d))
+		goto out;
+
+	if (!lrzip_file_add(lr, s))
+		goto out;
+	lrzip_outfile_set(lr, d);
+	if (!lrzip_run(lr))
+		goto out;
+
+	fd = fileno(d);
+	if (fstat(fd, &st))
+		goto out;
+	*dest_len = st.st_size;
+	if (unlikely((i64)fread(dest, sizeof(char), st.st_size, d) != st.st_size))
+		goto out;
+	if (unlikely(ferror(d)))
+		goto out;
+	ret = true;
+
+out:
+	if (s) fclose(s);
+	if (d) fclose(d);
+	lrzip_free(lr);
+	return ret;
+}

liblrzip_demo.c

@@ -0,0 +1,346 @@
+/*
+   Copyright (C) 2012 Con Kolivas
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+#undef NDEBUG
+#include <stdlib.h>
+#include <stdio.h>
+#ifdef HAVE_STRING_H
+# include <string.h>
+#endif
+#include <assert.h>
+#ifdef HAVE_ERRNO_H
+# include <errno.h>
+#else
+extern int errno;
+#endif
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
+#include <termios.h>
+#include <Lrzip.h>
+
+#define failure(...) do { \
+	fprintf(stderr, __VA_ARGS__); \
+	exit(1); \
+} while (0)
+
+static void usage(void)
+{
+	printf("lrzip version %s\n", PACKAGE_VERSION);
+	printf("Copyright (C) Con Kolivas 2006-2011\n");
+	printf("Based on rzip ");
+	printf("Copyright (C) Andrew Tridgell 1998-2003\n\n");
+	printf("Usage: lrzip [options] <file...>\n");
+	printf("General options:\n");
+	printf("     -c            check integrity of file written on decompression\n");
+	printf("     -d            decompress\n");
+	printf("     -e            password protected sha512/aes128 encryption on compression\n");
+	printf("     -h|-?         show help\n");
+	printf("     -H            display md5 hash integrity information\n");
+	printf("     -i            show compressed file information\n");
+	printf("     -q            don't show compression progress\n");
+	printf("     -t            test compressed file integrity\n");
+	printf("     -v[v]         Increase verbosity\n");
+	printf("     -V            show version\n");
+	printf("Options affecting output:\n");
+	printf("     -D            delete existing files\n");
+	printf("     -f            force overwrite of any existing files\n");
+	printf("     -k            keep broken or damaged output files\n");
+	printf("     -o filename   specify the output file name and/or path\n");
+	printf("     -O directory  specify the output directory when -o is not used\n");
+	printf("     -S suffix     specify compressed suffix (default '.lrz')\n");
+	printf("Options affecting compression:\n");
+	printf("     -b            bzip2 compression\n");
+	printf("     -g            gzip compression using zlib\n");
+	printf("     -l            lzo compression (ultra fast)\n");
+	printf("     -n            no backend compression - prepare for other compressor\n");
+	printf("     -z            zpaq compression (best, extreme compression, extremely slow)\n");
+	printf("Low level options:\n");
+	printf("     -L level      set lzma/bzip2/gzip compression level (1-9, default 7)\n");
+	printf("     -N value      Set nice value to value (default 19)\n");
+	printf("     -p value      Set processor count to override number of threads\n");
+	printf("     -T            Disable LZO compressibility testing\n");
+	printf("     -U            Use unlimited window size beyond ramsize (potentially much slower)\n");
+	printf("     -w size       maximum compression window in hundreds of MB\n");
+	printf("                   default chosen by heuristic dependent on ram and chosen compression\n");
+	printf("\nLRZIP=NOCONFIG environment variable setting can be used to bypass lrzip.conf.\n");
+	printf("TMP environment variable will be used for storage of temporary files when needed.\n");
+	printf("TMPDIR may also be stored in lrzip.conf file.\n");
+	printf("\nIf no filenames or \"-\" is specified, stdin/out will be used.\n");
+}
+
+static int get_pass(char *s, size_t slen)
+{
+	int len;
+
+	memset(s, 0, slen);
+	if (!fgets(s, slen, stdin)) {
+		fprintf(stderr, "Failed to retrieve passphrase\n");
+		return -1;
+	}
+	len = strlen(s);
+	if (len > 0 && ('\r' ==  s[len - 1] || '\n' == s[len - 1]))
+		s[len - 1] = '\0';
+	if (len > 1 && ('\r' ==  s[len - 2] || '\n' == s[len - 2]))
+		s[len - 2] = '\0';
+	len = strlen(s);
+	if (!len) {
+		fprintf(stderr, "Empty passphrase\n");
+		return -1;
+	}
+	return len;
+}
+
+static void pass_cb(void *data __UNUSED__, char *pass_string, size_t pass_len)
+{
+	int len;
+	struct termios termios_p;
+	/* Disable stdin echo to screen */
+	tcgetattr(fileno(stdin), &termios_p);
+	termios_p.c_lflag &= ~ECHO;
+	tcsetattr(fileno(stdin), 0, &termios_p);
+
+	printf("Enter passphrase: ");
+	len = get_pass(pass_string, pass_len);
+	printf("\n");
+
+	if (len < 1) exit(1);
+
+	termios_p.c_lflag |= ECHO;
+	tcsetattr(fileno(stdin), 0, &termios_p);
+}
+
+static void mode_check(Lrzip *lr, Lrzip_Mode mode)
+{
+	Lrzip_Mode current = lrzip_mode_get(lr);
+	if (current && (current != mode))
+		failure("Can only use one of -l, -b, -g, -z or -n\n");
+	lrzip_mode_set(lr, mode);
+}
+
+int main(int argc, char *argv[])
+{
+	Lrzip *lr;
+	extern int optind;
+	extern char *optarg;
+	int64_t x;
+	int c;
+	bool get_hash = false;
+
+	lrzip_init();
+	lr = lrzip_new(LRZIP_MODE_NONE);
+	assert(lr);
+	lrzip_config_env(lr);
+	lrzip_log_level_set(lr, LRZIP_LOG_LEVEL_PROGRESS);
+	while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) {
+		switch (c) {
+		case 'b':
+			mode_check(lr, LRZIP_MODE_COMPRESS_BZIP2);
+			break;
+		case 'c':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_VERIFY);
+			break;
+		case 'd':
+			mode_check(lr, LRZIP_MODE_DECOMPRESS);
+			break;
+		case 'D':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_SOURCE);
+			break;
+		case 'e':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_ENCRYPT);
+			break;
+		case 'f':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_DESTINATION);
+			break;
+		case 'g':
+			mode_check(lr, LRZIP_MODE_COMPRESS_ZLIB);
+			break;
+		case 'h':
+		case '?':
+			usage();
+			return -1;
+		case 'H':
+			get_hash = true;
+			break;
+		case 'i':
+			mode_check(lr, LRZIP_MODE_INFO);
+			break;
+		case 'k':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_KEEP_BROKEN);
+			break;
+		case 'l':
+			mode_check(lr, LRZIP_MODE_COMPRESS_LZO);
+			break;
+		case 'L':
+			errno = 0;
+			x = strtol(optarg, NULL, 10);
+			if (errno || ((x < 1) || (x > 9)))
+				failure("Invalid compression level (must be 1-9)\n");
+			lrzip_compression_level_set(lr, (unsigned int)x);
+			break;
+		case 'n':
+			mode_check(lr, LRZIP_MODE_COMPRESS_NONE);
+			break;
+		case 'N':
+			errno = 0;
+			x = strtol(optarg, NULL, 10);
+			if (errno || (x < -20 || x > 19))
+				failure("Invalid nice value (must be -20..19)\n");
+			lrzip_nice_set(lr, x);
+			break;
+		case 'o':
+			if (lrzip_outdir_get(lr))
+				failure("Cannot have -o and -O together\n");
+			if (!strcmp(optarg, "-"))
+				lrzip_outfile_set(lr, stdout);
+			else
+				lrzip_outfilename_set(lr, optarg);
+			break;
+		case 'O':
+			if (lrzip_outfilename_get(lr))	/* can't mix -o and -O */
+				failure("Cannot have options -o and -O together\n");
+			if (lrzip_outfile_get(lr))
+				failure("Cannot specify an output directory when outputting to stdout\n");
+			lrzip_outdir_set(lr, optarg);
+			break;
+		case 'p':
+			errno = 0;
+			x = strtol(optarg, NULL, 10);
+			if (errno || (x < 1))
+				failure("Must have at least one thread\n");
+			lrzip_threads_set(lr, (unsigned int)x);
+			break;
+		case 'q':
+			lrzip_log_level_set(lr, lrzip_log_level_get(lr) - 1);
+			break;
+		case 'S':
+			if (lrzip_outfilename_get(lr))
+				failure("Specified output filename already, can't specify an extension.\n");
+			if (lrzip_outfile_get(lr))
+				failure("Cannot specify a filename suffix when outputting to stdout\n");
+			lrzip_suffix_set(lr, optarg);
+			break;
+		case 't':
+			if (lrzip_outfilename_get(lr))
+				failure("Cannot specify an output file name when just testing.\n");
+			if (lrzip_flags_get(lr) & LRZIP_FLAG_REMOVE_SOURCE)
+				failure("Doubt that you want to delete a file when just testing.\n");
+			mode_check(lr, LRZIP_MODE_TEST);
+			break;
+		case 'T':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_DISABLE_LZO_CHECK);
+			break;
+		case 'U':
+			lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_UNLIMITED_RAM);
+			break;
+		case 'v':
+			lrzip_log_level_set(lr, lrzip_log_level_get(lr) + 1);
+			break;
+		case 'V':
+			printf("lrzip version %s\n", PACKAGE_VERSION);
+			exit(0);
+			break;
+		case 'w':
+			errno = 0;
+			x = strtoll(optarg, NULL, 10);
+			if (errno || (x < 1))
+				failure("Invalid compression window '%s'!\n", optarg);
+			lrzip_compression_window_max_set(lr, x);
+			break;
+		case 'z':
+			mode_check(lr, LRZIP_MODE_COMPRESS_ZPAQ);
+			break;
+		}
+	}
+	/* LZMA is the default */
+	if (!lrzip_mode_get(lr)) lrzip_mode_set(lr, LRZIP_MODE_COMPRESS_LZMA);
+	argc -= optind, argv += optind;
+
+	if (lrzip_outfilename_get(lr) && (argc > 1))
+		failure("Cannot specify output filename with more than 1 file\n");
+
+	if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_compression_window_max_get(lr)) {
+		fprintf(stderr, "If -U used, cannot specify a window size with -w.\n");
+		lrzip_compression_window_max_set(lr, 0);
+	}
+
+	if (argc < 1) lrzip_file_add(lr, stdin);
+
+	if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_files_count(lr)) {
+		fprintf(stderr, "Cannot have -U and stdin, unlimited mode disabled.\n");
+		lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_UNLIMITED_RAM);
+	}
+
+	/* If no output filename is specified, and we're using stdin,
+	 * use stdout */
+	if (lrzip_files_count(lr) && (!lrzip_outfilename_get(lr)))
+		lrzip_outfile_set(lr, stdout);
+
+	if (lrzip_flags_get(lr) & LRZIP_FLAG_VERIFY) {
+		if (lrzip_mode_get(lr) != LRZIP_MODE_DECOMPRESS) {
+			fprintf(stderr, "Can only check file written on decompression.\n");
+			lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY);
+		} else if (lrzip_outfile_get(lr)) {
+			fprintf(stderr, "Can't check file written when writing to stdout. Checking disabled.\n");
+			lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY);
+		}
+	}
+
+	for (x = 0; x < argc; x++) {
+		if (argv[x][0] != '-') {
+			assert(lrzip_filename_add(lr, argv[x]));
+			continue;
+		}
+		if (argv[x][1] == 0) {
+			assert(lrzip_file_add(lr, stdin));
+			continue;
+		}
+	}
+	if (argc == 1) {
+		if (!lrzip_files_count(lr)) lrzip_file_add(lr, stdin);
+		if (lrzip_filenames_count(lr)) {
+			if (!lrzip_outfilename_get(lr)) {
+				char *buf;
+				const char *infile;
+				size_t len;
+
+				infile = lrzip_filenames_get(lr)[0];
+				len = strlen(infile);
+				buf = alloca(len + 8);
+				if (!strcmp(infile + len - 4, ".lrz"))
+					strcat(buf, infile);
+				else
+					sprintf(buf, "%s.out", infile);
+				lrzip_outfilename_set(lr, buf);
+			}
+		} else if (!lrzip_outfile_get(lr)) lrzip_outfile_set(lr, stdout);
+	}
+	lrzip_log_stdout_set(lr, stdout);
+	lrzip_log_stderr_set(lr, stderr);
+	lrzip_pass_cb_set(lr, pass_cb, NULL);
+	if (!lrzip_run(lr)) exit(1);
+	if (get_hash) {
+		const unsigned char *digest = lrzip_md5digest_get(lr);
+		for (x = 0; x < 16; x++)
+			fprintf(stdout, "%02x", digest[x] & 0xFF);
+	}
+	lrzip_free(lr);
+	return 0;
+}

liblrzip_private.h

@@ -0,0 +1,22 @@
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+#include <lrzip_private.h>
+#include <Lrzip.h>
+
+#define INFILE_BUCKET_SIZE 10
+
+struct Lrzip
+{
+	Lrzip_Mode mode;
+	unsigned int flags;
+	rzip_control *control;
+
+	/* bucket allocation is used here to avoid frequent calls to realloc */
+	char **infilenames;
+	size_t infilename_idx;
+	size_t infilename_buckets;
+	FILE **infiles;
+	size_t infile_idx;
+	size_t infile_buckets;
+};

lrzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -56,7 +56,6 @@
 #include "stream.h"
 
 #define MAGIC_LEN (24)
-#define STDIO_TMPFILE_BUFFER_SIZE (65536) // used in read_tmpinfile and dump_tmpoutfile
 
 static void release_hashes(rzip_control *control);
 
@@ -91,32 +90,6 @@ i64 get_ram(rzip_control *control)
 
 	return ramsize;
 }
-#elif defined(__OpenBSD__)
-# include <sys/resource.h>
-i64 get_ram(rzip_control *control)
-{
-	struct rlimit rl;
-	i64 ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE;
-
-	/* Raise limits all the way to the max */
-
-	if (getrlimit(RLIMIT_DATA, &rl) == -1)
-		fatal_return(("Failed to get limits in get_ram\n"), -1);
-
-	rl.rlim_cur = rl.rlim_max;
-	if (setrlimit(RLIMIT_DATA, &rl) == -1)
-		fatal_return(("Failed to set limits in get_ram\n"), -1);
-
-	/* Declare detected RAM to be either the max RAM available from
-	physical memory or the max RAM allowed by RLIMIT_DATA, whatever
-	is smaller, to prevent the heuristics from selecting
-	compression windows which cause lrzip to go into deep swap */
-
-	if (rl.rlim_max < ramsize)
-		return rl.rlim_max;
-
-	return ramsize;
-}
 #else /* __APPLE__ */
 i64 get_ram(rzip_control *control)
 {
@@ -252,7 +225,7 @@ static bool get_magic(rzip_control *control, char *magic)
 
 	/* Whether this archive contains md5 data at the end or not */
 	md5 = magic[21];
-	if (md5) {
+	if (md5 && MD5_RELIABLE) {
 		if (md5 == 1)
 			control->flags |= FLAG_MD5;
 		else
@@ -341,11 +314,10 @@ int open_tmpoutfile(rzip_control *control)
 
 	fd_out = mkstemp(control->outfile);
 	if (fd_out == -1) {
-		print_output("WARNING: Failed to create out tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
+		print_progress("WARNING: Failed to create out tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
 			       control->outfile, DECOMPRESS ? "de" : "");
 	} else
 		register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN);
-	print_maxverbose("Created temporary outfile %s\n", control->outfile);
 	return fd_out;
 }
 
@@ -359,8 +331,8 @@ static bool fwrite_stdout(rzip_control *control, void *buf, i64 len)
 	while (len > 0) {
 		ssize_t wrote;
 
-		if (BITS32)
-			ret = MIN(len, one_g);
+		if (len > one_g)
+			ret = one_g;
 		else
 			ret = len;
 		wrote = fwrite(offset_buf, 1, ret, control->outFILE);
@@ -380,10 +352,7 @@ bool write_fdout(rzip_control *control, void *buf, i64 len)
 	ssize_t ret;
 
 	while (len > 0) {
-		if (BITS32)
-			ret = MIN(len, one_g);
-		else
-			ret = len;
+		ret = MIN(len, one_g);
 		ret = write(control->fd_out, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			fatal_return(("Failed to write to fd_out in write_fdout\n"), false);
@@ -393,7 +362,7 @@ bool write_fdout(rzip_control *control, void *buf, i64 len)
 	return true;
 }
 
-static bool flush_tmpoutbuf(rzip_control *control)
+bool flush_tmpoutbuf(rzip_control *control)
 {
 	if (!TEST_ONLY) {
 		print_maxverbose("Dumping buffer to physical file.\n");
@@ -411,10 +380,10 @@ static bool flush_tmpoutbuf(rzip_control *control)
 }
 
 /* Dump temporary outputfile to perform stdout */
-static bool dump_tmpoutfile(rzip_control *control)
+bool dump_tmpoutfile(rzip_control *control, int fd_out)
 {
-	int fd_out = control->fd_out;
 	FILE *tmpoutfp;
+	int tmpchar;
 
 	if (unlikely(fd_out == -1))
 		fatal_return(("Failed: No temporary outfile created, unable to do in ram\n"), false);
@@ -426,35 +395,9 @@ static bool dump_tmpoutfile(rzip_control *control)
 	rewind(tmpoutfp);
 
 	if (!TEST_ONLY) {
-		char* buf;
-
 		print_verbose("Dumping temporary file to control->outFILE.\n");
-		fflush(control->outFILE);
-
-		buf = malloc(STDIO_TMPFILE_BUFFER_SIZE);
-		if (unlikely(!buf))
-			fatal_return(("Failed to allocate buffer in dump_tmpoutfile\n"), false);
-
-		while (1) {
-			ssize_t num_read, num_written;
-			num_read = fread(buf, 1, STDIO_TMPFILE_BUFFER_SIZE, tmpoutfp);
-			if (unlikely(num_read == 0)) {
-				if (ferror(tmpoutfp)) {
-					dealloc(buf);
-					fatal_return(("Failed read in dump_tmpoutfile\n"), false);
-				} else {
-					break; // must be at EOF
-				}
-			}
-
-			num_written = fwrite(buf, 1, num_read, control->outFILE);
-			if (unlikely(num_written != num_read)) {
-				dealloc(buf);
-				fatal_return(("Failed write in dump_tmpoutfile\n"), false);
-			}
-		}
-
-		dealloc(buf);
+		while ((tmpchar = fgetc(tmpoutfp)) != EOF)
+			putchar(tmpchar);
 		fflush(control->outFILE);
 		rewind(tmpoutfp);
 	}
@@ -464,15 +407,6 @@ static bool dump_tmpoutfile(rzip_control *control)
 	return true;
 }
 
-bool flush_tmpout(rzip_control *control)
-{
-	if (!STDOUT)
-		return true;
-	if (TMP_OUTBUF)
-		return flush_tmpoutbuf(control);
-	return dump_tmpoutfile(control);
-}
-
 /* Used if we're unable to read STDIN into the temporary buffer, shunts data
  * to temporary file */
 bool write_fdin(rzip_control *control)
@@ -482,10 +416,7 @@ bool write_fdin(rzip_control *control)
 	ssize_t ret;
 
 	while (len > 0) {
-		if (BITS32)
-			ret = MIN(len, one_g);
-		else
-			ret = len;
+		ret = MIN(len, one_g);
 		ret = write(control->fd_in, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			fatal_return(("Failed to write to fd_in in write_fdin\n"), false);
@@ -531,7 +462,7 @@ int open_tmpinfile(rzip_control *control)
 	}
 
 	if (fd_in == -1) {
-		print_output("WARNING: Failed to create in tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
+		print_progress("WARNING: Failed to create in tmpfile: %s, will fail if cannot perform %scompression entirely in ram\n",
 			       control->infile, DECOMPRESS ? "de" : "");
 	} else {
 		register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN);
@@ -566,7 +497,6 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
 {
 	FILE *tmpinfp;
 	int tmpchar;
-	char* buf;
 
 	if (fd_in == -1)
 		return false;
@@ -576,30 +506,9 @@ bool read_tmpinfile(rzip_control *control, int fd_in)
 	if (unlikely(tmpinfp == NULL))
 		fatal_return(("Failed to fdopen in tmpfile\n"), false);
 
-	buf = malloc(STDIO_TMPFILE_BUFFER_SIZE);
-	if (unlikely(!buf))
-		fatal_return(("Failed to allocate buffer in read_tmpinfile\n"), false);
+	while ((tmpchar = getchar()) != EOF)
+		fputc(tmpchar, tmpinfp);
 
-	while (1) {
-		ssize_t num_read, num_written;
-		num_read = fread(buf, 1, STDIO_TMPFILE_BUFFER_SIZE, stdin);
-		if (unlikely(num_read == 0)) {
-			if (ferror(stdin)) {
-				dealloc(buf);
-				fatal_return(("Failed read in read_tmpinfile\n"), false);
-			} else {
-				break; // must be at EOF
-			}
-		}
-
-		num_written = fwrite(buf, 1, num_read, tmpinfp);
-		if (unlikely(num_written != num_read)) {
-			dealloc(buf);
-			fatal_return(("Failed write in read_tmpinfile\n"), false);
-		}
-	}
-
-	dealloc(buf);
 	fflush(tmpinfp);
 	rewind(tmpinfp);
 	return true;
@@ -777,21 +686,6 @@ static void release_hashes(rzip_control *control)
 	dealloc(control->hash);
 }
 
-static void clear_rulist(rzip_control *control)
-{
-	while (control->ruhead) {
-		struct runzip_node *node = control->ruhead;
-		struct stream_info *sinfo = node->sinfo;
-
-		dealloc(sinfo->ucthreads);
-		dealloc(node->pthreads);
-		dealloc(sinfo->s);
-		dealloc(sinfo);
-		control->ruhead = node->prev;
-		dealloc(node);
-	}
-}
-
 /*
   decompress one file from the command line
 */
@@ -849,7 +743,7 @@ bool decompress_file(rzip_control *control)
 		}
 
 		if (!STDOUT)
-			print_output("Output filename is: %s\n", control->outfile);
+			print_progress("Output filename is: %s\n", control->outfile);
 	}
 
 	if ( IS_FROM_FILE ) {
@@ -946,27 +840,26 @@ bool decompress_file(rzip_control *control)
 		if (unlikely(!get_hash(control, 0)))
 			return false;
 
-	print_output("Decompressing...\n");
+	print_progress("Decompressing...\n");
 
-	if (unlikely(runzip_fd(control, fd_in, fd_hist, expected_size) < 0)) {
-		clear_rulist(control);
+	if (unlikely(runzip_fd(control, fd_in, fd_out, fd_hist, expected_size) < 0))
 		return false;
-	}
 
-	/* We can now safely delete sinfo and pthread data of all threads
-	 * created. */
-	clear_rulist(control);
+	if (STDOUT && !TMP_OUTBUF) {
+		if (unlikely(!dump_tmpoutfile(control, fd_out)))
+			return false;
+	}
 
 	/* if we get here, no fatal_return(( errors during decompression */
 	print_progress("\r");
 	if (!(STDOUT | TEST_ONLY))
-		print_output("Output filename is: %s: ", control->outfile);
+		print_progress("Output filename is: %s: ", control->outfile);
 	if (!expected_size)
 		expected_size = control->st_size;
 	if (!ENCRYPT)
-		print_output("[OK] - %lld bytes                                \n", expected_size);
+		print_progress("[OK] - %lld bytes                                \n", expected_size);
 	else
-		print_output("[OK]                                             \n");
+		print_progress("[OK]                                             \n");
 
 	if (TMP_OUTBUF)
 		close_tmpoutbuf(control);
@@ -1142,12 +1035,12 @@ next_chunk:
 	stream_head[0] = 0;
 	stream_head[1] = stream_head[0] + header_length;
 
-	print_verbose("Rzip chunk:       %d\n", ++chunk);
+	print_verbose("Rzip chunk %d:\n", ++chunk);
 	if (chunk_byte)
 		print_verbose("Chunk byte width: %d\n", chunk_byte);
 	if (chunk_size) {
 		chunk_total += chunk_size;
-		print_verbose("Chunk size:       %"PRId64"\n", chunk_size);
+		print_verbose("Chunk size: %lld\n", chunk_size);
 	}
 	if (unlikely(chunk_byte && (chunk_byte > 8 || chunk_size < 0)))
 		failure("Invalid chunk data\n");
@@ -1161,14 +1054,12 @@ next_chunk:
 			return false;
 
 		print_verbose("Stream: %d\n", stream);
-		print_maxverbose("Offset: %"PRId64"\n", stream_head[stream] + ofs);
-		print_verbose("%s\t%s\t%s\t%16s / %14s", "Block","Comp","Percent","Comp Size", "UComp Size");
-		print_maxverbose("%18s : %14s", "Offset", "Head");
-		print_verbose("\n");
+		print_maxverbose("Offset: %lld\n", ofs);
+		print_verbose("Block\tComp\tPercent\tSize\n");
 		do {
 			i64 head_off;
 
-			if (unlikely(last_head && last_head <= second_last))
+			if (unlikely(last_head && last_head < second_last))
 				failure_goto(("Invalid earlier last_head position, corrupt archive.\n"), error);
 			second_last = last_head;
 			if (unlikely(last_head + ofs > infile_size))
@@ -1202,8 +1093,8 @@ next_chunk:
 						     * the future */
 			utotal += u_len;
 			ctotal += c_len;
-			print_verbose("\t%5.1f%%\t%16"PRId64" / %14"PRId64"", percentage(c_len, u_len), c_len, u_len);
-			print_maxverbose("%18"PRId64" : %14"PRId64"", head_off, last_head);
+			print_verbose("\t%.1f%%\t%lld / %lld", percentage(c_len, u_len), c_len, u_len);
+			print_maxverbose("\tOffset: %lld\tHead: %lld", head_off, last_head);
 			print_verbose("\n");
 			block++;
 		} while (last_head);
@@ -1236,36 +1127,23 @@ next_chunk:
 	}
 	goto next_chunk;
 done:
-	cratio = (long double)expected_size / (long double)infile_size;
 	if (unlikely(ofs > infile_size))
 		failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error);
-	print_output("\nSummary\n=======\n");
-	print_output("File: %s\nlrzip version: %d.%d \n\n", infilecopy,
-				control->major_version, control->minor_version);
+	print_verbose("Rzip compression: %.1f%% %lld / %lld\n",
+			percentage (utotal, expected_size),
+			utotal, expected_size);
+	print_verbose("Back end compression: %.1f%% %lld / %lld\n",
+			percentage(ctotal, utotal),
+			ctotal, utotal);
+	print_verbose("Overall compression: %.1f%% %lld / %lld\n",
+			percentage(ctotal, expected_size),
+			ctotal, expected_size);
 
-	if (!expected_size)
-		print_output("Due to %s, expected decompression size not available\n", "Compression to STDOUT");
-	print_verbose("  Stats         Percent       Compressed /   Uncompressed\n  -------------------------------------------------------\n");
-		/* If we can't show expected size, tailor output for it */
-	if (expected_size) {
-		print_verbose("  Rzip:         %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
-					percentage (utotal, expected_size),
-					utotal, expected_size);
-		print_verbose("  Back end:     %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
-					percentage(ctotal, utotal),
-					ctotal, utotal);
-		print_verbose("  Overall:      %5.1f%%\t%16"PRId64" / %14"PRId64"\n",
-					percentage(ctotal, expected_size),
-					ctotal, expected_size);
-	} else {
-		print_verbose("  Rzip:         Unavailable\n");
-		print_verbose("  Back end:     %5.1f%%\t%16"PRId64" / %14"PRId64"\n", percentage(ctotal, utotal), ctotal, utotal);
-		print_verbose("  Overall:      Unavailable\n");
-	}
-	print_verbose("\n");
+	cratio = (long double)expected_size / (long double)infile_size;
 
-	print_output("  Compression Method: ");
+	print_output("%s:\nlrzip version: %d.%d file\n", infilecopy, control->major_version, control->minor_version);
 
+	print_output("Compression: ");
 	if (save_ctype == CTYPE_NONE)
 		print_output("rzip alone\n");
 	else if (save_ctype == CTYPE_BZIP2)
@@ -1280,32 +1158,25 @@ done:
 		print_output("rzip + zpaq\n");
 	else
 		print_output("Dunno wtf\n");
+	print_output("Decompressed file size: %llu\n", expected_size);
+	print_output("Compressed file size: %llu\n", infile_size);
+	print_output("Compression ratio: %.3Lf\n", cratio);
 
-	print_output("\n");
-
-	if (expected_size) {
-		print_output("  Decompressed file size: %14"PRIu64"\n", expected_size);
-		print_output("  Compressed file size:   %14"PRIu64"\n", infile_size);
-		print_output("  Compression ratio:      %14.3Lfx\n", cratio);
-	} else {
-		print_output("  Decompressed file size:    Unavailable\n");
-		print_output("  Compressed file size:   %14"PRIu64"\n", infile_size);
-		print_output("  Compression ratio:         Unavailable\n");
-	}
 	if (HAS_MD5) {
 		char md5_stored[MD5_DIGEST_SIZE];
 		int i;
 
+		print_output("MD5 used for integrity testing\n");
 		if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END) == -1))
 			fatal_goto(("Failed to seek to md5 data in runzip_fd\n"), error);
 		if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE))
 			fatal_goto(("Failed to read md5 data in runzip_fd\n"), error);
-		print_output("\n  MD5 Checksum: ");
+		print_output("MD5: ");
 		for (i = 0; i < MD5_DIGEST_SIZE; i++)
 			print_output("%02x", md5_stored[i] & 0xFF);
 		print_output("\n");
 	} else
-		print_output("\n  CRC32 used for integrity testing\n");
+		print_output("CRC32 used for integrity testing\n");
 	if ( !IS_FROM_FILE )
 		if (unlikely(close(fd_in)))
 			fatal_return(("Failed to close fd_in in get_fileinfo\n"), false);
@@ -1329,7 +1200,8 @@ bool compress_file(rzip_control *control)
 	int fd_in = -1, fd_out = -1;
 	char header[MAGIC_LEN];
 
-	control->flags |= FLAG_MD5;
+	if (MD5_RELIABLE)
+		control->flags |= FLAG_MD5;
 	if (ENCRYPT)
 		if (unlikely(!get_hash(control, 1)))
 			return false;
@@ -1385,7 +1257,7 @@ bool compress_file(rzip_control *control)
 			} else
 				strcpy(control->outfile, tmpinfile);
 			strcat(control->outfile, control->suffix);
-			print_output("Output filename is: %s\n", control->outfile);
+			print_progress("Output filename is: %s\n", control->outfile);
 		}
 
 		fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666);
@@ -1406,12 +1278,6 @@ bool compress_file(rzip_control *control)
 				goto error;
 		}
 	} else {
-		control->fd_out = fd_out = open_tmpoutfile(control);
-		if (likely(fd_out != -1)) {
-			/* Unlink temporary file as soon as possible */
-			if (unlikely(unlink(control->outfile)))
-				fatal_return(("Failed to unlink tmpfile: %s\n", control->outfile), false);
-		}
 		if (unlikely(!open_tmpoutbuf(control)))
 			goto error;
 	}
@@ -1422,7 +1288,7 @@ bool compress_file(rzip_control *control)
 
 	rzip_fd(control, fd_in, fd_out);
 
-	/* Write magic at end b/c lzma does not tell us properties until it is done */
+	/* Wwrite magic at end b/c lzma does not tell us properties until it is done */
 	if (!STDOUT) {
 		if (unlikely(!write_magic(control)))
 			goto error;

lrzip.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: lrzip
+Description: lrzip compression library
+Version: @VERSION@
+Libs: -L${libdir} -llrzip
+Libs.private: @LIBS@

lrzip_core.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2022 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -27,7 +27,7 @@ bool write_magic(rzip_control *control);
 bool read_magic(rzip_control *control, int fd_in, i64 *expected_size);
 bool preserve_perms(rzip_control *control, int fd_in, int fd_out);
 int open_tmpoutfile(rzip_control *control);
-bool flush_tmpout(rzip_control *control);
+bool dump_tmpoutfile(rzip_control *control, int fd_out);
 int open_tmpinfile(rzip_control *control);
 bool read_tmpinfile(rzip_control *control, int fd_in);
 bool decompress_file(rzip_control *control);
@@ -36,6 +36,7 @@ bool get_fileinfo(rzip_control *control);
 bool compress_file(rzip_control *control);
 bool write_fdout(rzip_control *control, void *buf, i64 len);
 bool write_fdin(rzip_control *control);
+bool flush_tmpoutbuf(rzip_control *control);
 void close_tmpoutbuf(rzip_control *control);
 void clear_tmpinbuf(rzip_control *control);
 bool clear_tmpinfile(rzip_control *control);

lrzip_private.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -144,7 +144,7 @@ extern int errno;
 #define unlikely(x)	__builtin_expect(!!(x), 0)
 #define __maybe_unused	__attribute__((unused))
 
-#if defined(__MINGW32__) || defined(__CYGWIN__) || defined(__ANDROID__) || defined(__APPLE__) || defined(__OpenBSD__)
+#if defined(__MINGW32__) || defined(__CYGWIN__) || defined(__ANDROID__) || defined(__APPLE__)
 # define ffsll __builtin_ffsll
 #endif
 
@@ -170,6 +170,12 @@ typedef sem_t cksem_t;
  #define mremap fake_mremap
 #endif
 
+#if defined(__APPLE__)
+# define MD5_RELIABLE (0)
+#else
+# define MD5_RELIABLE (1)
+#endif
+
 #define bswap_32(x) \
      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) |		      \
       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
@@ -229,7 +235,6 @@ typedef sem_t cksem_t;
 #define FLAG_TMP_OUTBUF		(1 << 21)
 #define FLAG_TMP_INBUF		(1 << 22)
 #define FLAG_ENCRYPT		(1 << 23)
-#define FLAG_OUTPUT		(1 << 24)
 
 #define NO_MD5		(!(HASH_CHECK) && !(HAS_MD5))
 
@@ -255,12 +260,10 @@ typedef sem_t cksem_t;
 # define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
 #endif
 
-#ifndef PAGE_SIZE
-# ifdef _SC_PAGE_SIZE
-#  define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
-# else
-#  define PAGE_SIZE (4096)
-# endif
+#ifdef _SC_PAGE_SIZE
+# define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
+#else
+# define PAGE_SIZE (4096)
 #endif
 
 #define dealloc(ptr) do { \
@@ -309,7 +312,6 @@ typedef sem_t cksem_t;
 #define TMP_OUTBUF	(control->flags & FLAG_TMP_OUTBUF)
 #define TMP_INBUF	(control->flags & FLAG_TMP_INBUF)
 #define ENCRYPT		(control->flags & FLAG_ENCRYPT)
-#define SHOW_OUTPUT	(control->flags & FLAG_OUTPUT)
 
 #define IS_FROM_FILE ( !!(control->inFILE) && !STDIN )
 
@@ -472,6 +474,7 @@ struct rzip_control {
 	i64 (*match_len)(rzip_control *, struct rzip_state *, i64, i64, i64, i64 *);
 
 	pthread_t *pthreads;
+	struct runzip_node *rulist;
 	struct runzip_node *ruhead;
 };
 
@@ -549,7 +552,6 @@ static inline void print_err(const rzip_control *control, unsigned int line, con
 } while (0)
 
 #define print_output(...)	do {\
-	if (SHOW_OUTPUT)	\
 	print_stuff(1, __VA_ARGS__); \
 } while (0)
 

lzma/ASM/x86/7zCrcOpt_asm.asm

@@ -138,10 +138,4 @@ MY_ENDP
 %ifidn __OUTPUT_FORMAT__,elf
 section .note.GNU-stack noalloc noexec nowrite progbits
 %endif
-%ifidn __OUTPUT_FORMAT__,elf32
-section .note.GNU-stack noalloc noexec nowrite progbits
-%endif
-%ifidn __OUTPUT_FORMAT__,elf64
-section .note.GNU-stack noalloc noexec nowrite progbits
-%endif
 

lzma/C/Makefile.am

@@ -53,12 +53,11 @@ liblzma_la_LIBADD = $(ASM_7z).lo
 \n\# Generated by libtool -- hack to allow asm linking\
 \n\# Peter Hyman\
 \npic_object='.libs/$(ASM_7z).o'\
-\nnon_pic_object='$(ASM_7z).o'\
-\n
+\nnon_pic_object='$(ASM_7z).o'
 
 $(ASM_7z).lo: $(ASM_S)
 	$(ASM_PROG) $(ASM_OPT) -o $(ASM_7z).o $(ASM_S)
 	mkdir -p .libs
 	cp $(ASM_7z).o .libs/
-	@printf "$(7ZIPASMLOFILE)" > $(ASM_7z).lo
+	@echo -e "$(7ZIPASMLOFILE)" > $(ASM_7z).lo
 endif

m4/ac_attribute.m4

@@ -18,7 +18,8 @@ AC_DEFUN([AC_C___ATTRIBUTE__],
 AC_MSG_CHECKING([for __attribute__])
 
 AC_CACHE_VAL([ac_cv___attribute__],
-   [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+   [AC_TRY_COMPILE(
+       [
 #include <stdlib.h>
 
 int func(int x);
@@ -26,8 +27,11 @@ int foo(int x __attribute__ ((unused)))
 {
    exit(1);
 }
-       ]], [[]])],[ac_cv___attribute__="yes"],[ac_cv___attribute__="no"
-    ])])
+       ],
+       [],
+       [ac_cv___attribute__="yes"],
+       [ac_cv___attribute__="no"]
+    )])
 
 AC_MSG_RESULT($ac_cv___attribute__)
 

m4/ax_pthread.m4

@@ -0,0 +1,486 @@
+# ===========================================================================
+#        https://www.gnu.org/software/autoconf-archive/ax_pthread.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+#   This macro figures out how to build C programs using POSIX threads. It
+#   sets the PTHREAD_LIBS output variable to the threads library and linker
+#   flags, and the PTHREAD_CFLAGS output variable to any special C compiler
+#   flags that are needed. (The user can also force certain compiler
+#   flags/libs to be tested by setting these environment variables.)
+#
+#   Also sets PTHREAD_CC to any special C compiler that is needed for
+#   multi-threaded programs (defaults to the value of CC otherwise). (This
+#   is necessary on AIX to use the special cc_r compiler alias.)
+#
+#   NOTE: You are assumed to not only compile your program with these flags,
+#   but also to link with them as well. For example, you might link with
+#   $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
+#
+#   If you are only building threaded programs, you may wish to use these
+#   variables in your default LIBS, CFLAGS, and CC:
+#
+#     LIBS="$PTHREAD_LIBS $LIBS"
+#     CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+#     CC="$PTHREAD_CC"
+#
+#   In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant
+#   has a nonstandard name, this macro defines PTHREAD_CREATE_JOINABLE to
+#   that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#
+#   Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the
+#   PTHREAD_PRIO_INHERIT symbol is defined when compiling with
+#   PTHREAD_CFLAGS.
+#
+#   ACTION-IF-FOUND is a list of shell commands to run if a threads library
+#   is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it
+#   is not found. If ACTION-IF-FOUND is not specified, the default action
+#   will define HAVE_PTHREAD.
+#
+#   Please let the authors know if this macro fails on any platform, or if
+#   you have any other suggestions or comments. This macro was based on work
+#   by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help
+#   from M. Frigo), as well as ac_pthread and hb_pthread macros posted by
+#   Alejandro Forero Cuervo to the autoconf macro repository. We are also
+#   grateful for the helpful feedback of numerous users.
+#
+#   Updated for Autoconf 2.68 by Daniel Richard G.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
+#   Copyright (c) 2011 Daniel Richard G. <skunk@iSKUNK.ORG>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 25
+
+AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD])
+AC_DEFUN([AX_PTHREAD], [
+AC_REQUIRE([AC_CANONICAL_HOST])
+AC_REQUIRE([AC_PROG_CC])
+AC_REQUIRE([AC_PROG_SED])
+AC_LANG_PUSH([C])
+ax_pthread_ok=no
+
+# We used to check for pthread.h first, but this fails if pthread.h
+# requires special compiler flags (e.g. on Tru64 or Sequent).
+# It gets checked for in the link test anyway.
+
+# First of all, check if the user has set any of the PTHREAD_LIBS,
+# etcetera environment variables, and if threads linking works using
+# them:
+if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then
+        ax_pthread_save_CC="$CC"
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        AS_IF([test "x$PTHREAD_CC" != "x"], [CC="$PTHREAD_CC"])
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+        AC_MSG_CHECKING([for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS])
+        AC_LINK_IFELSE([AC_LANG_CALL([], [pthread_join])], [ax_pthread_ok=yes])
+        AC_MSG_RESULT([$ax_pthread_ok])
+        if test "x$ax_pthread_ok" = "xno"; then
+                PTHREAD_LIBS=""
+                PTHREAD_CFLAGS=""
+        fi
+        CC="$ax_pthread_save_CC"
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
+fi
+
+# We must check for the threads library under a number of different
+# names; the ordering is very important because some systems
+# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
+# libraries is broken (non-POSIX).
+
+# Create a list of thread flags to try.  Items starting with a "-" are
+# C compiler flags, and other items are library names, except for "none"
+# which indicates that we try without any flags at all, and "pthread-config"
+# which is a program returning the flags for the Pth emulation library.
+
+ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+
+# The ordering *is* (sometimes) important.  Some notes on the
+# individual items follow:
+
+# pthreads: AIX (must check this before -lpthread)
+# none: in case threads are in libc; should be tried before -Kthread and
+#       other compiler flags to prevent continual compiler warnings
+# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads), Tru64
+#           (Note: HP C rejects this with "bad form for `-t' option")
+# -pthreads: Solaris/gcc (Note: HP C also rejects)
+# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
+#      doesn't hurt to check since this sometimes defines pthreads and
+#      -D_REENTRANT too), HP C (must be checked before -lpthread, which
+#      is present but should not be used directly; and before -mthreads,
+#      because the compiler interprets this as "-mt" + "-hreads")
+# -mthreads: Mingw32/gcc, Lynx/gcc
+# pthread: Linux, etcetera
+# --thread-safe: KAI C++
+# pthread-config: use pthread-config program (for GNU Pth library)
+
+case $host_os in
+
+        freebsd*)
+
+        # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+        # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+
+        ax_pthread_flags="-kthread lthread $ax_pthread_flags"
+        ;;
+
+        hpux*)
+
+        # From the cc(1) man page: "[-mt] Sets various -D flags to enable
+        # multi-threading and also sets -lpthread."
+
+        ax_pthread_flags="-mt -pthread pthread $ax_pthread_flags"
+        ;;
+
+        openedition*)
+
+        # IBM z/OS requires a feature-test macro to be defined in order to
+        # enable POSIX threads at all, so give the user a hint if this is
+        # not set. (We don't define these ourselves, as they can affect
+        # other portions of the system API in unpredictable ways.)
+
+        AC_EGREP_CPP([AX_PTHREAD_ZOS_MISSING],
+            [
+#            if !defined(_OPEN_THREADS) && !defined(_UNIX03_THREADS)
+             AX_PTHREAD_ZOS_MISSING
+#            endif
+            ],
+            [AC_MSG_WARN([IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support.])])
+        ;;
+
+        solaris*)
+
+        # On Solaris (at least, for some versions), libc contains stubbed
+        # (non-functional) versions of the pthreads routines, so link-based
+        # tests will erroneously succeed. (N.B.: The stubs are missing
+        # pthread_cleanup_push, or rather a function called by this macro,
+        # so we could check for that, but who knows whether they'll stub
+        # that too in a future libc.)  So we'll check first for the
+        # standard Solaris way of linking pthreads (-mt -lpthread).
+
+        ax_pthread_flags="-mt,pthread pthread $ax_pthread_flags"
+        ;;
+esac
+
+# GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC)
+
+AS_IF([test "x$GCC" = "xyes"],
+      [ax_pthread_flags="-pthread -pthreads $ax_pthread_flags"])
+
+# The presence of a feature test macro requesting re-entrant function
+# definitions is, on some systems, a strong hint that pthreads support is
+# correctly enabled
+
+case $host_os in
+        darwin* | hpux* | linux* | osf* | solaris*)
+        ax_pthread_check_macro="_REENTRANT"
+        ;;
+
+        aix*)
+        ax_pthread_check_macro="_THREAD_SAFE"
+        ;;
+
+        *)
+        ax_pthread_check_macro="--"
+        ;;
+esac
+AS_IF([test "x$ax_pthread_check_macro" = "x--"],
+      [ax_pthread_check_cond=0],
+      [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"])
+
+# Are we compiling with Clang?
+
+AC_CACHE_CHECK([whether $CC is Clang],
+    [ax_cv_PTHREAD_CLANG],
+    [ax_cv_PTHREAD_CLANG=no
+     # Note that Autoconf sets GCC=yes for Clang as well as GCC
+     if test "x$GCC" = "xyes"; then
+        AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG],
+            [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */
+#            if defined(__clang__) && defined(__llvm__)
+             AX_PTHREAD_CC_IS_CLANG
+#            endif
+            ],
+            [ax_cv_PTHREAD_CLANG=yes])
+     fi
+    ])
+ax_pthread_clang="$ax_cv_PTHREAD_CLANG"
+
+ax_pthread_clang_warning=no
+
+# Clang needs special handling, because older versions handle the -pthread
+# option in a rather... idiosyncratic way
+
+if test "x$ax_pthread_clang" = "xyes"; then
+
+        # Clang takes -pthread; it has never supported any other flag
+
+        # (Note 1: This will need to be revisited if a system that Clang
+        # supports has POSIX threads in a separate library.  This tends not
+        # to be the way of modern systems, but it's conceivable.)
+
+        # (Note 2: On some systems, notably Darwin, -pthread is not needed
+        # to get POSIX threads support; the API is always present and
+        # active.  We could reasonably leave PTHREAD_CFLAGS empty.  But
+        # -pthread does define _REENTRANT, and while the Darwin headers
+        # ignore this macro, third-party headers might not.)
+
+        PTHREAD_CFLAGS="-pthread"
+        PTHREAD_LIBS=
+
+        ax_pthread_ok=yes
+
+        # However, older versions of Clang make a point of warning the user
+        # that, in an invocation where only linking and no compilation is
+        # taking place, the -pthread option has no effect ("argument unused
+        # during compilation").  They expect -pthread to be passed in only
+        # when source code is being compiled.
+        #
+        # Problem is, this is at odds with the way Automake and most other
+        # C build frameworks function, which is that the same flags used in
+        # compilation (CFLAGS) are also used in linking.  Many systems
+        # supported by AX_PTHREAD require exactly this for POSIX threads
+        # support, and in fact it is often not straightforward to specify a
+        # flag that is used only in the compilation phase and not in
+        # linking.  Such a scenario is extremely rare in practice.
+        #
+        # Even though use of the -pthread flag in linking would only print
+        # a warning, this can be a nuisance for well-run software projects
+        # that build with -Werror.  So if the active version of Clang has
+        # this misfeature, we search for an option to squash it.
+
+        AC_CACHE_CHECK([whether Clang needs flag to prevent "argument unused" warning when linking with -pthread],
+            [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG],
+            [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG=unknown
+             # Create an alternate version of $ac_link that compiles and
+             # links in two steps (.c -> .o, .o -> exe) instead of one
+             # (.c -> exe), because the warning occurs only in the second
+             # step
+             ax_pthread_save_ac_link="$ac_link"
+             ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g'
+             ax_pthread_link_step=`$as_echo "$ac_link" | sed "$ax_pthread_sed"`
+             ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)"
+             ax_pthread_save_CFLAGS="$CFLAGS"
+             for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do
+                AS_IF([test "x$ax_pthread_try" = "xunknown"], [break])
+                CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS"
+                ac_link="$ax_pthread_save_ac_link"
+                AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+                    [ac_link="$ax_pthread_2step_ac_link"
+                     AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+                         [break])
+                    ])
+             done
+             ac_link="$ax_pthread_save_ac_link"
+             CFLAGS="$ax_pthread_save_CFLAGS"
+             AS_IF([test "x$ax_pthread_try" = "x"], [ax_pthread_try=no])
+             ax_cv_PTHREAD_CLANG_NO_WARN_FLAG="$ax_pthread_try"
+            ])
+
+        case "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" in
+                no | unknown) ;;
+                *) PTHREAD_CFLAGS="$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG $PTHREAD_CFLAGS" ;;
+        esac
+
+fi # $ax_pthread_clang = yes
+
+if test "x$ax_pthread_ok" = "xno"; then
+for ax_pthread_try_flag in $ax_pthread_flags; do
+
+        case $ax_pthread_try_flag in
+                none)
+                AC_MSG_CHECKING([whether pthreads work without any flags])
+                ;;
+
+                -mt,pthread)
+                AC_MSG_CHECKING([whether pthreads work with -mt -lpthread])
+                PTHREAD_CFLAGS="-mt"
+                PTHREAD_LIBS="-lpthread"
+                ;;
+
+                -*)
+                AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag])
+                PTHREAD_CFLAGS="$ax_pthread_try_flag"
+                ;;
+
+                pthread-config)
+                AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no])
+                AS_IF([test "x$ax_pthread_config" = "xno"], [continue])
+                PTHREAD_CFLAGS="`pthread-config --cflags`"
+                PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+                ;;
+
+                *)
+                AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag])
+                PTHREAD_LIBS="-l$ax_pthread_try_flag"
+                ;;
+        esac
+
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+
+        # Check for various functions.  We must include pthread.h,
+        # since some functions may be macros.  (On the Sequent, we
+        # need a special flag -Kthread to make this header compile.)
+        # We check for pthread_join because it is in -lpthread on IRIX
+        # while pthread_create is in libc.  We check for pthread_attr_init
+        # due to DEC craziness with -lpthreads.  We check for
+        # pthread_cleanup_push because it is one of the few pthread
+        # functions on Solaris that doesn't have a non-functional libc stub.
+        # We try pthread_create on general principles.
+
+        AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>
+#                       if $ax_pthread_check_cond
+#                        error "$ax_pthread_check_macro must be defined"
+#                       endif
+                        static void routine(void *a) { a = 0; }
+                        static void *start_routine(void *a) { return a; }],
+                       [pthread_t th; pthread_attr_t attr;
+                        pthread_create(&th, 0, start_routine, 0);
+                        pthread_join(th, 0);
+                        pthread_attr_init(&attr);
+                        pthread_cleanup_push(routine, 0);
+                        pthread_cleanup_pop(0) /* ; */])],
+            [ax_pthread_ok=yes],
+            [])
+
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
+
+        AC_MSG_RESULT([$ax_pthread_ok])
+        AS_IF([test "x$ax_pthread_ok" = "xyes"], [break])
+
+        PTHREAD_LIBS=""
+        PTHREAD_CFLAGS=""
+done
+fi
+
+# Various other checks:
+if test "x$ax_pthread_ok" = "xyes"; then
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+
+        # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
+        AC_CACHE_CHECK([for joinable pthread attribute],
+            [ax_cv_PTHREAD_JOINABLE_ATTR],
+            [ax_cv_PTHREAD_JOINABLE_ATTR=unknown
+             for ax_pthread_attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+                 AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>],
+                                                 [int attr = $ax_pthread_attr; return attr /* ; */])],
+                                [ax_cv_PTHREAD_JOINABLE_ATTR=$ax_pthread_attr; break],
+                                [])
+             done
+            ])
+        AS_IF([test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xunknown" && \
+               test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xPTHREAD_CREATE_JOINABLE" && \
+               test "x$ax_pthread_joinable_attr_defined" != "xyes"],
+              [AC_DEFINE_UNQUOTED([PTHREAD_CREATE_JOINABLE],
+                                  [$ax_cv_PTHREAD_JOINABLE_ATTR],
+                                  [Define to necessary symbol if this constant
+                                   uses a non-standard name on your system.])
+               ax_pthread_joinable_attr_defined=yes
+              ])
+
+        AC_CACHE_CHECK([whether more special flags are required for pthreads],
+            [ax_cv_PTHREAD_SPECIAL_FLAGS],
+            [ax_cv_PTHREAD_SPECIAL_FLAGS=no
+             case $host_os in
+             solaris*)
+             ax_cv_PTHREAD_SPECIAL_FLAGS="-D_POSIX_PTHREAD_SEMANTICS"
+             ;;
+             esac
+            ])
+        AS_IF([test "x$ax_cv_PTHREAD_SPECIAL_FLAGS" != "xno" && \
+               test "x$ax_pthread_special_flags_added" != "xyes"],
+              [PTHREAD_CFLAGS="$ax_cv_PTHREAD_SPECIAL_FLAGS $PTHREAD_CFLAGS"
+               ax_pthread_special_flags_added=yes])
+
+        AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT],
+            [ax_cv_PTHREAD_PRIO_INHERIT],
+            [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]],
+                                             [[int i = PTHREAD_PRIO_INHERIT;
+                                               return i;]])],
+                            [ax_cv_PTHREAD_PRIO_INHERIT=yes],
+                            [ax_cv_PTHREAD_PRIO_INHERIT=no])
+            ])
+        AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes" && \
+               test "x$ax_pthread_prio_inherit_defined" != "xyes"],
+              [AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], [1], [Have PTHREAD_PRIO_INHERIT.])
+               ax_pthread_prio_inherit_defined=yes
+              ])
+
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
+
+        # More AIX lossage: compile with *_r variant
+        if test "x$GCC" != "xyes"; then
+            case $host_os in
+                aix*)
+                AS_CASE(["x/$CC"],
+                    [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6],
+                    [#handle absolute path differently from PATH based program lookup
+                     AS_CASE(["x$CC"],
+                         [x/*],
+                         [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])],
+                         [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])])
+                ;;
+            esac
+        fi
+fi
+
+test -n "$PTHREAD_CC" || PTHREAD_CC="$CC"
+
+AC_SUBST([PTHREAD_LIBS])
+AC_SUBST([PTHREAD_CFLAGS])
+AC_SUBST([PTHREAD_CC])
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test "x$ax_pthread_ok" = "xyes"; then
+        ifelse([$1],,[AC_DEFINE([HAVE_PTHREAD],[1],[Define if you have POSIX threads libraries and header files.])],[$1])
+        :
+else
+        ax_pthread_ok=no
+        $2
+fi
+AC_LANG_POP
+])dnl AX_PTHREAD

m4/efl_doxygen.m4

@@ -15,7 +15,9 @@ dnl
 dnl Disable the build of the documentation
 dnl
 AC_ARG_ENABLE([doc],
-   [AS_HELP_STRING([--disable-doc],[Disable documentation build @<:@default=enabled@:>@])],
+   [AC_HELP_STRING(
+       [--disable-doc],
+       [Disable documentation build @<:@default=enabled@:>@])],
    [
     if test "x${enableval}" = "xyes" ; then
        efl_enable_doc="yes"
@@ -37,7 +39,9 @@ dnl
    efl_doxygen="doxygen"
 
    AC_ARG_WITH([doxygen],
-      [AS_HELP_STRING([--with-doxygen=FILE],[doxygen program to use @<:@default=doxygen@:>@])],
+      [AC_HELP_STRING(
+          [--with-doxygen=FILE],
+          [doxygen program to use @<:@default=doxygen@:>@])],
 dnl
 dnl Check the given doxygen program.
 dnl

main.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2021 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -68,7 +68,7 @@ static rzip_control base_control, local_control, *control;
 static void usage(bool compat)
 {
 	print_output("lrz%s version %s\n", compat ? "" : "ip", PACKAGE_VERSION);
-	print_output("Copyright (C) Con Kolivas 2006-2022\n");
+	print_output("Copyright (C) Con Kolivas 2006-2021\n");
 	print_output("Based on rzip ");
 	print_output("Copyright (C) Andrew Tridgell 1998-2003\n\n");
 	print_output("Usage: lrz%s [options] <file...>\n", compat ? "" : "ip");
@@ -86,10 +86,8 @@ static void usage(bool compat)
 	if (compat) {
 		print_output("	-L, --license		display software version and license\n");
 		print_output("	-P, --progress		show compression progress\n");
-	} else {
+	} else
 		print_output("	-q, --quiet		don't show compression progress\n");
-		print_output("	-Q, --very-quiet	don't show any output\n");
-	}
 	print_output("	-r, --recursive		operate recursively on directories\n");
 	print_output("	-t, --test		test compressed file integrity\n");
 	print_output("	-v[v%s], --verbose	Increase verbosity\n", compat ? "v" : "");
@@ -247,7 +245,6 @@ static struct option long_options[] = {
 	{"threads",	required_argument,	0,	'p'},
 	{"progress",	no_argument,	0,	'P'},
 	{"quiet",	no_argument,	0,	'q'},
-	{"very-quiet",	no_argument,	0,	'Q'},
 	{"recursive",	no_argument,	0,	'r'},
 	{"suffix",	required_argument,	0,	'S'},
 	{"test",	no_argument,	0,	't'},  /* 25 */
@@ -302,7 +299,7 @@ static void recurse_dirlist(char *indir, char **dirlist, int *entries)
 	closedir(dirp);
 }
 
-static const char *loptions = "bcCdDefghHiKlL:nN:o:O:p:PqQrS:tTUm:vVw:z?";
+static const char *loptions = "bcCdDefghHiKlL:nN:o:O:p:PqrS:tTUm:vVw:z?";
 static const char *coptions = "bcCdefghHikKlLnN:o:O:p:PrS:tTUm:vVw:z?123456789";
 
 int main(int argc, char *argv[])
@@ -324,7 +321,6 @@ int main(int argc, char *argv[])
 	initialise_control(control);
 
 	av = basename(argv[0]);
-	control->flags |= FLAG_OUTPUT;
 	if (!strcmp(av, "lrunzip"))
 		control->flags |= FLAG_DECOMPRESS;
 	else if (!strcmp(av, "lrzcat")) {
@@ -413,9 +409,9 @@ int main(int argc, char *argv[])
 			control->flags |= FLAG_FORCE_REPLACE;
 			break;
 		case 'h':
+		case '?':
 			usage(compat);
-			exit(0);
-			break;
+			return -1;
 		case 'H':
 			control->flags |= FLAG_HASH;
 			break;
@@ -489,10 +485,6 @@ int main(int argc, char *argv[])
 		case 'q':
 			control->flags &= ~FLAG_SHOW_PROGRESS;
 			break;
-		case 'Q':
-			control->flags &= ~FLAG_SHOW_PROGRESS;
-			control->flags &= ~FLAG_OUTPUT;
-			break;
 		case 'r':
 			recurse = true;
 			break;
@@ -552,15 +544,9 @@ int main(int argc, char *argv[])
 		case '9':
 			control->compression_level = c - '0';
 			break;
-		default:
-			usage(compat);
-			return 2;
 		}
 	}
 
-	if (compat && !SHOW_PROGRESS)
-		control->flags &= ~FLAG_OUTPUT;
-
 	argc -= optind;
 	argv += optind;
 
@@ -712,7 +698,7 @@ recursion:
 
 		gettimeofday(&start_time, NULL);
 
-		if (!control->passphrase && (unlikely((STDIN || STDOUT) && ENCRYPT)))
+		if (unlikely((STDIN || STDOUT) && ENCRYPT))
 			failure("Unable to work from STDIO while reading password\n");
 
 		memcpy(&local_control, &base_control, sizeof(rzip_control));
@@ -731,7 +717,7 @@ recursion:
 		minutes = (int)(total_time / 60) % 60;
 		seconds = total_time - hours * 3600 - minutes * 60;
 		if (!INFO)
-			print_output("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
+			print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
 		if (recurse)
 			goto recursion;
 	}

man/lrzip.1

@@ -1,4 +1,4 @@
-.TH "lrzip" "1" "February 2022" "" ""
+.TH "lrzip" "1" "February 2021" "" ""
 .SH "NAME"
 lrzip \- a large-file compression program
 .SH "SYNOPSIS"
@@ -43,7 +43,6 @@ General options:
  \-H, \-\-hash              display md5 hash integrity information
  \-i, \-\-info              show compressed file information
  \-q, \-\-quiet             don't show compression progress
- \-Q, \-\-very-quiet        don't show any output
  \-r, \-\-recursive         operate recursively on directories
  \-t, \-\-test              test compressed file integrity
  \-v[v], \-\-verbose        Increase verbosity
@@ -143,10 +142,6 @@ bursts with lzma compression which is the default compression. This means
 that it will progress very rapidly for short periods and then stop for
 long periods.
 .IP
-.IP "\fB-Q\fP"
-If this option is specified then lrzip will not show any output to the console
-except for error messages.
-.IP
 .IP "\fB-r\fP"
 If this option is specified, lrzip will recursively enter the directories
 specified, compressing or decompressing every file individually in the same
@@ -354,12 +349,6 @@ lzop(1),
 rzip(1),
 zip(1)
 
-.PP
-.SH "DIAGNOSTICS"
-.PP
-Exit status is normally 0; if an error occurs, exit status is 1, usage errors
-is 2.
-
 .PP
 .SH "AUTHOR and CREDITS"
 .br
@@ -376,9 +365,8 @@ zpaq was written by Matt Mahoney.
 Peter Hyman added informational output, updated LZMA SDK,
 and added lzma multi-threading capabilities.
 .PP
-If you wish to report a problem, or make a suggestion, then please consult the
-git repository at:
-https://github.com/ckolivas/lrzip
+If you wish to report a problem, or make a suggestion, then please email Con at
+kernel@kolivas.org
 .PP
 lrzip is released under the GNU General Public License version 2.
 Please see the file COPYING for license details.

runzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021 Con Kolivas
    Copyright (C) 1998-2003 Andrew Tridgell
 
    This program is free software; you can redistribute it and/or modify
@@ -194,7 +194,7 @@ static i64 read_fdhist(rzip_control *control, void *buf, i64 len)
 static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, int chunk_bytes)
 {
 	i64 offset, n, total, cur_pos;
-	uchar *buf;
+	uchar *buf, *off_buf;
 
 	if (unlikely(len < 0))
 		failure_return(("len %lld is negative in unzip_match!\n",len), -1);
@@ -212,35 +212,32 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum,
 		fatal_return(("Seek failed by %d from %d on history file in unzip_match\n",
 		      offset, cur_pos), -1);
 
-	n = MIN(len, offset);
-	if (unlikely(n < 1))
-		fatal_return(("Failed fd history in unzip_match due to corrupt archive\n"), -1);
-
-	buf = (uchar *)malloc(n);
+	buf = (uchar *)malloc(len);
 	if (unlikely(!buf))
 		fatal_return(("Failed to malloc match buffer of size %lld\n", len), -1);
-
-	if (unlikely(read_fdhist(control, buf, (size_t)n) != (ssize_t)n)) {
-		dealloc(buf);
-		fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1);
-	}
+	off_buf = buf;
 
 	while (len) {
 		n = MIN(len, offset);
 		if (unlikely(n < 1))
 			fatal_return(("Failed fd history in unzip_match due to corrupt archive\n"), -1);
 
-		if (unlikely(write_1g(control, buf, (size_t)n) != (ssize_t)n)) {
+		if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) {
+			dealloc(buf);
+			fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1);
+		}
+		if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) {
 			dealloc(buf);
 			fatal_return(("Failed to write %d bytes in unzip_match\n", n), -1);
 		}
 
 		if (!HAS_MD5)
-			*cksum = CrcUpdate(*cksum, buf, n);
+			*cksum = CrcUpdate(*cksum, off_buf, n);
 		if (!NO_MD5)
-			md5_process_bytes(buf, n, &control->ctx);
+			md5_process_bytes(off_buf, n, &control->ctx);
 
 		len -= n;
+		off_buf += n;
 		total += n;
 	}
 
@@ -249,6 +246,21 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum,
 	return total;
 }
 
+static void clear_rulist(rzip_control *control)
+{
+	while (control->ruhead) {
+		struct runzip_node *node = control->ruhead;
+		struct stream_info *sinfo = node->sinfo;
+
+		dealloc(sinfo->ucthreads);
+		dealloc(node->pthreads);
+		dealloc(sinfo->s);
+		dealloc(sinfo);
+		control->ruhead = node->prev;
+		dealloc(node);
+	}
+}
+
 /* decompress a section of an open file. Call fatal_return(() on error
    return the number of bytes that have been retrieved
  */
@@ -366,13 +378,17 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64
 	if (unlikely(close_stream_in(control, ss)))
 		fatal("Failed to close stream!\n");
 
+	/* We can now safely delete sinfo and pthread data of all threads
+	 * created. */
+	clear_rulist(control);
+
 	return total;
 }
 
 /* Decompress an open file. Call fatal_return(() on error
    return the number of bytes that have been retrieved
  */
-i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size)
+i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size)
 {
 	uchar md5_stored[MD5_DIGEST_SIZE];
 	struct timeval start,end;
@@ -392,11 +408,17 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size)
 			}
 		}
 		total += u;
-		if (unlikely(!flush_tmpout(control))) {
-			print_err("Failed to flush_tmpout in runzip_fd\n");
+		if (TMP_OUTBUF) {
+			if (unlikely(!flush_tmpoutbuf(control))) {
+				print_err("Failed to flush_tmpoutbuf in runzip_fd\n");
 				return -1;
+			}
+		} else if (STDOUT) {
+			if (unlikely(!dump_tmpoutfile(control, fd_out))) {
+				print_err("Failed to dump_tmpoutfile in runzip_fd\n");
+				return -1;
+			}
 		}
-
 		if (TMP_INBUF)
 			clear_tmpinbuf(control);
 		else if (STDIN && !DECOMPRESS) {
@@ -412,7 +434,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size)
 		tdiff = end.tv_sec - start.tv_sec;
 		if (!tdiff)
 			tdiff = 1;
-		print_output("\nAverage DeCompression Speed: %6.3fMB/s\n",
+		print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n",
 			       (total / 1024 / 1024) / tdiff);
 	}
 

runzip.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2011,2022 Con Kolivas
+   Copyright (C) 2006-2011 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998-2003 Andrew Tridgell
 
@@ -22,6 +22,6 @@
 
 #include "lrzip_private.h"
 
-i64 runzip_fd(rzip_control *control, int fd_in, int fd_hist, i64 expected_size);
+i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size);
 
 #endif

rzip.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2018,2022 Con Kolivas
+   Copyright (C) 2006-2016,2018 Con Kolivas
    Copyright (C) 1998 Andrew Tridgell
 
    Modified to use flat hash, memory limit and variable hash culling
@@ -1221,9 +1221,11 @@ retry:
 		}
 	}
 
-	if (unlikely(!flush_tmpout(control))) {
+	if (TMP_OUTBUF) {
+		if (unlikely(!flush_tmpoutbuf(control))) {
 			dealloc(st);
-			failure("Failed to flush_tmpout in rzip_fd\n");
+			failure("Failed to flush_tmpoutbuf in rzip_fd\n");
+		}
 	}
 
 	gettimeofday(&current, NULL);
@@ -1247,10 +1249,24 @@ retry:
 	       (1.0 + st->stats.match_bytes) / st->stats.literal_bytes);
 
 	if (!STDIN)
-		print_output("%s - ", control->infile);
-	print_output("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n",
+		print_progress("%s - ", control->infile);
+	print_progress("Compression Ratio: %.3f. Average Compression Speed: %6.3fMB/s.\n",
 		       1.0 * s.st_size / s2.st_size, chunkmbs);
 
 	clear_sslist(st);
 	dealloc(st);
 }
+
+void rzip_control_free(rzip_control *control)
+{
+	if (!control)
+		return;
+
+	dealloc(control->tmpdir);
+	dealloc(control->outname);
+	dealloc(control->outdir);
+	if (control->suffix && control->suffix[0])
+		dealloc(control->suffix);
+
+	dealloc(control);
+}

rzip.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2022 Con Kolivas
+   Copyright (C) 2006-2016 Con Kolivas
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell
 
@@ -22,5 +22,6 @@
 #include "lrzip_private.h"
 
 void rzip_fd(rzip_control *control, int fd_in, int fd_out);
+void rzip_control_free(rzip_control *control);
 
 #endif

stream.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2018,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2018,2021 Con Kolivas
    Copyright (C) 2011 Serge Belyshev
    Copyright (C) 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell
@@ -610,22 +610,20 @@ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret)
 	if (unlikely(control->out_ofs + ret > control->out_maxlen)) {
 		/* The data won't fit in a temporary output buffer so we have
 		 * to fall back to temporary files. */
-		print_verbose("Unable to %scompress entirely in ram, will use physical files\n",
-			      DECOMPRESS ? "de" : "");
-		if (unlikely(control->fd_out == -1)) {
-			failure("Was unable to %scompress entirely in ram and no temporary file creation was possible\n",
-				DECOMPRESS ? "de" : "");
-		}
-		/* Copy tmp_outbuf to tmpoutfile before deallocation */
+		print_verbose("Unable to decompress entirely in ram, will use physical files\n");
+		if (unlikely(control->fd_out == -1))
+			failure("Was unable to decompress entirely in ram and no temporary file creation was possible\n");
 		if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) {
 			print_err("Unable to write_fdout tmpoutbuf in put_fdout\n");
 			return -1;
 		}
-		/* Deallocate now unused tmpoutbuf and unset tmp_outbuf flag */
 		close_tmpoutbuf(control);
-		return write(control->fd_out, offset_buf, (size_t)ret);
+		if (unlikely(!write_fdout(control, offset_buf, ret))) {
+			print_err("Unable to write_fdout offset_buf in put_fdout\n");
+			return -1;
+		}
+		return ret;
 	}
-
 	memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret);
 	control->out_ofs += ret;
 	if (likely(control->out_ofs > control->out_len))
@@ -634,7 +632,8 @@ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret)
 }
 
 /* This is a custom version of write() which writes in 1GB chunks to avoid
-   the overflows at the >= 2GB mark thanks to 32bit fuckage. */
+   the overflows at the >= 2GB mark thanks to 32bit fuckage. This should help
+   even on the rare occasion write() fails to write 1GB as well. */
 ssize_t write_1g(rzip_control *control, void *buf, i64 len)
 {
 	uchar *offset_buf = buf;
@@ -643,10 +642,7 @@ ssize_t write_1g(rzip_control *control, void *buf, i64 len)
 
 	total = 0;
 	while (len > 0) {
-		if (BITS32)
-			ret = MIN(len, one_g);
-		else
-			ret = len;
+		ret = MIN(len, one_g);
 		ret = put_fdout(control, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			return ret;
@@ -697,10 +693,8 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len)
 		/* We're decompressing from STDIN */
 		if (unlikely(control->in_ofs + len > control->in_maxlen)) {
 			/* We're unable to fit it all into the temp buffer */
-			if (dump_stdin(control)) {
-				failure_return(("Inadequate ram to %scompress from STDIN and unable to create in tmpfile",
-					DECOMPRESS ? "de" : ""), -1);
-			}
+			if (dump_stdin(control))
+				failure_return(("Inadequate ram to %compress from STDIN and unable to create in tmpfile"), -1);
 			goto read_fd;
 		}
 		if (control->in_ofs + len > control->in_len) {
@@ -723,10 +717,7 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len)
 read_fd:
 	total = 0;
 	while (len > 0) {
-		if (BITS32)
-			ret = MIN(len, one_g);
-		else
-			ret = len;
+		ret = MIN(len, one_g);
 		ret = read(fd, offset_buf, (size_t)ret);
 		if (unlikely(ret <= 0))
 			return ret;
@@ -937,10 +928,9 @@ bool close_streamout_threads(rzip_control *control)
 void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes)
 {
 	struct stream_info *sinfo;
-	unsigned int i, testbufs;
-	bool threadlimit = false;
 	i64 testsize, limit;
 	uchar *testmalloc;
+	unsigned int i, testbufs;
 
 	sinfo = calloc(sizeof(struct stream_info), 1);
 	if (unlikely(!sinfo))
@@ -976,28 +966,20 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li
 	/* If we don't have enough ram for the number of threads, decrease the
 	 * number of threads till we do, or only have one thread. */
 	while (limit < STREAM_BUFSIZE && limit < chunk_limit) {
-		if (control->threads > 1) {
+		if (control->threads > 1)
 			--control->threads;
-			threadlimit = true;
-		} else
+		else
 			break;
 		limit = (control->usable_ram - (control->overhead * control->threads)) / testbufs;
 		limit = MIN(limit, chunk_limit);
 	}
-	if (threadlimit) {
-		print_output("Minimising number of threads to %d to limit memory usage\n",
-			     control->threads);
-	}
 	if (BITS32) {
 		limit = MIN(limit, one_g);
 		if (limit + (control->overhead * control->threads) > one_g)
 			limit = one_g - (control->overhead * control->threads);
 	}
 	/* Use a nominal minimum size should we fail all previous shrinking */
-	if (limit < STREAM_BUFSIZE) {
-		limit = MAX(limit, STREAM_BUFSIZE);
-		print_output("Warning, low memory for chosen compression settings\n");
-	}
+	limit = MAX(limit, STREAM_BUFSIZE);
 	limit = MIN(limit, chunk_limit);
 retest_malloc:
 	testsize = limit + (control->overhead * control->threads);
@@ -1379,11 +1361,16 @@ retry:
 	if (!ctis->chunks++) {
 		int j;
 
-		if (STDOUT) {
+		if (TMP_OUTBUF) {
 			lock_mutex(control, &control->control_lock);
 			if (!control->magic_written)
 				write_magic(control);
 			unlock_mutex(control, &control->control_lock);
+
+			if (unlikely(!flush_tmpoutbuf(control))) {
+				print_err("Failed to flush_tmpoutbuf in compthread\n");
+				goto error;
+			}
 		}
 
 		print_maxverbose("Writing initial chunk bytes value %d at %lld\n",
@@ -1686,7 +1673,7 @@ fill_another:
 	fsync(control->fd_out);
 
 	if (unlikely(u_len > control->maxram))
-		print_output("Warning, attempting to malloc very large buffer for this environment of size %lld\n", u_len);
+		print_progress("Warning, attempting to malloc very large buffer for this environment of size %lld\n", u_len);
 	max_len = MAX(u_len, MIN_SIZE);
 	max_len = MAX(max_len, c_len);
 	s_buf = malloc(max_len);
@@ -1862,11 +1849,8 @@ static void add_to_rulist(rzip_control *control, struct stream_info *sinfo)
 		failure("Failed to calloc struct node in add_rulist\n");
 	node->sinfo = sinfo;
 	node->pthreads = control->pthreads;
-
-	lock_mutex(control, &control->control_lock);
-	node->prev = control->ruhead;
+	node->prev = control->rulist;
 	control->ruhead = node;
-	unlock_mutex(control, &control->control_lock);
 }
 
 /* close down an input stream */
@@ -1898,8 +1882,11 @@ int close_stream_in(rzip_control *control, void *ss)
    so do not compress any block that is incompressible by lz4. */
 static int lz4_compresses(rzip_control *control, uchar *s_buf, i64 s_len)
 {
-	int dlen, test_len;
+	int in_len, test_len = s_len, save_len = s_len;
+	int dlen;
 	char *c_buf = NULL, *test_buf = (char *)s_buf;
+	/* set minimum buffer test size based on the length of the test stream */
+	int buftest_size = (test_len > 5 * STREAM_BUFSIZE ? STREAM_BUFSIZE : STREAM_BUFSIZE / 4096);
 	int ret = 0;
 	int workcounter = 0;	/* count # of passes */
 	int best_dlen = INT_MAX; /* save best compression estimate */
@@ -1907,37 +1894,40 @@ static int lz4_compresses(rzip_control *control, uchar *s_buf, i64 s_len)
 	if (!LZ4_TEST)
 		return 1;
 
-	dlen = MIN(s_len, STREAM_BUFSIZE);
-	test_len = MIN(dlen, STREAM_BUFSIZE >> 8);
+	in_len = MIN(test_len, buftest_size);
+	dlen = STREAM_BUFSIZE + STREAM_BUFSIZE / 16 + 64 + 3;
+
 	c_buf = malloc(dlen);
 	if (unlikely(!c_buf))
 		fatal_return(("Unable to allocate c_buf in lz4_compresses\n"), 0);
 
 	/* Test progressively larger blocks at a time and as soon as anything
 	   compressible is found, jump out as a success */
-	do {
+	while (test_len > 0) {
 		int lz4_ret;
 
 		workcounter++;
 		lz4_ret = LZ4_compress_default((const char *)test_buf, c_buf, test_len, dlen);
-		if (!lz4_ret) // Bigger than dlen
-			lz4_ret = test_len;
+		if (!lz4_ret) // Bigger than dlen, no point going further
+			break;
 		if (lz4_ret < best_dlen)
 			best_dlen = lz4_ret;
 		if (lz4_ret < test_len) {
 			ret = 1;
 			break;
 		}
-		/* expand test length */
-		test_len <<= 1;
-	} while (test_len <= dlen);
-
-	if (!ret)
-		print_maxverbose("lz4 testing FAILED for chunk %ld. %d Passes\n", workcounter);
-	else {
-		print_maxverbose("lz4 testing OK for chunk %ld. Compressed size = %5.2F%% of chunk, %d Passes\n",
-				s_len, 100 * ((double) best_dlen / (double) test_len), workcounter);
+		/* expand and move buffer */
+		test_len -= in_len;
+		if (test_len) {
+			test_buf += (ptrdiff_t)in_len;
+			if (buftest_size < STREAM_BUFSIZE)
+				buftest_size <<= 1;
+			in_len = MIN(test_len, buftest_size);
+		}
 	}
+	print_maxverbose("lz4 testing %s for chunk %ld. Compressed size = %5.2F%% of chunk, %d Passes\n",
+			(ret == 0? "FAILED" : "OK"), save_len,
+			100 * ((double) best_dlen / (double) in_len), workcounter);
 
 	dealloc(c_buf);
 

util.c

@@ -1,5 +1,5 @@
 /*
-   Copyright (C) 2006-2016,2021-2022 Con Kolivas
+   Copyright (C) 2006-2016,2021 Con Kolivas
    Copyright (C) 2011 Serge Belyshev
    Copyright (C) 2008, 2011 Peter Hyman
    Copyright (C) 1998 Andrew Tridgell