mirror of
https://github.com/ckolivas/lrzip.git
synced 2025-12-06 07:12:00 +01:00
5505097b2f
This facilitates a massive speed up on SMP machines proportional to the number of CPUs during the back end compression phase, but does so at some cost to the final size. Limit the number of threads to ensure that each thread at least works on a window of STREAM_BUFSIZE. Disable the lzma threading library as it does not contribute any more to the scalability of this new approach, yet compromises compression. Increase the size of the windows passed to all the compression back end types now as they need more to split them up into multiple threads, and the number of blocks increases the compressed size slightly.
304 lines
7.3 KiB
C
304 lines
7.3 KiB
C
/*
|
|
Copyright (C) Andrew Tridgell 1998,
|
|
Con Kolivas 2006-2010
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
|
*/
|
|
|
|
#define LRZIP_MAJOR_VERSION 0
|
|
#define LRZIP_MINOR_VERSION 5
|
|
#define LRZIP_MINOR_SUBVERSION 20
|
|
|
|
#define NUM_STREAMS 2
|
|
|
|
#include "config.h"
|
|
|
|
#include <sys/types.h>
|
|
#include <unistd.h>
|
|
#include <stdio.h>
|
|
#include <stddef.h>
|
|
#include <stdarg.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <signal.h>
|
|
#include <bzlib.h>
|
|
#include <zlib.h>
|
|
#include <pthread.h>
|
|
#include <semaphore.h>
|
|
#include <sys/resource.h>
|
|
#include <netinet/in.h>
|
|
|
|
#include <sys/time.h>
|
|
|
|
#include <sys/mman.h>
|
|
#include <sys/syscall.h>
|
|
|
|
#include <lzo/lzoconf.h>
|
|
#include <lzo/lzo1x.h>
|
|
|
|
/* LZMA C Wrapper */
|
|
#include "lzma/C/LzmaLib.h"
|
|
|
|
#ifdef HAVE_STRING_H
|
|
#include <string.h>
|
|
#endif
|
|
|
|
#ifdef HAVE_MALLOC_H
|
|
#include <malloc.h>
|
|
#endif
|
|
|
|
#include <fcntl.h>
|
|
#include <sys/stat.h>
|
|
|
|
#ifdef HAVE_CTYPE_H
|
|
#include <ctype.h>
|
|
#endif
|
|
#include <errno.h>
|
|
#include <sys/mman.h>
|
|
|
|
/* needed for CRC routines */
|
|
#include "lzma/C/7zCrc.h"
|
|
|
|
#ifndef uchar
|
|
#define uchar unsigned char
|
|
#endif
|
|
|
|
#ifndef int32
|
|
#if (SIZEOF_INT == 4)
|
|
#define int32 int
|
|
#elif (SIZEOF_LONG == 4)
|
|
#define int32 long
|
|
#elif (SIZEOF_SHORT == 4)
|
|
#define int32 short
|
|
#endif
|
|
#endif
|
|
|
|
#ifndef int16
|
|
#if (SIZEOF_INT == 2)
|
|
#define int16 int
|
|
#elif (SIZEOF_SHORT == 2)
|
|
#define int16 short
|
|
#endif
|
|
#endif
|
|
|
|
#ifndef uint32
|
|
#define uint32 unsigned int32
|
|
#endif
|
|
|
|
#ifndef uint16
|
|
#define uint16 unsigned int16
|
|
#endif
|
|
|
|
#ifndef MIN
|
|
#define MIN(a, b) ((a) < (b)? (a): (b))
|
|
#endif
|
|
|
|
#ifndef MAX
|
|
#define MAX(a, b) ((a) > (b)? (a): (b))
|
|
#endif
|
|
|
|
#if !HAVE_STRERROR
|
|
extern char *sys_errlist[];
|
|
#define strerror(i) sys_errlist[i]
|
|
#endif
|
|
|
|
#ifndef HAVE_ERRNO_DECL
|
|
extern int errno;
|
|
#endif
|
|
|
|
#define likely(x) __builtin_expect(!!(x), 1)
|
|
#define unlikely(x) __builtin_expect(!!(x), 0)
|
|
|
|
typedef long long int i64;
|
|
typedef uint16_t u16;
|
|
typedef uint32_t u32;
|
|
|
|
#ifndef MAP_ANONYMOUS
|
|
#define MAP_ANONYMOUS MAP_ANON
|
|
#endif
|
|
|
|
#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN)
|
|
#define PROCESSORS (1)
|
|
#else
|
|
#define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
|
|
#endif
|
|
|
|
#ifdef _SC_PAGE_SIZE
|
|
#define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
|
|
#else
|
|
#define PAGE_SIZE (4096)
|
|
#endif
|
|
|
|
#ifdef __APPLE__
|
|
#include <sys/sysctl.h>
|
|
#define fmemopen fake_fmemopen
|
|
#define open_memstream fake_open_memstream
|
|
#define memstream_update_buffer fake_open_memstream_update_buffer
|
|
#define mremap fake_mremap
|
|
static inline i64 get_ram(void)
|
|
{
|
|
int mib[2];
|
|
size_t len;
|
|
i64 *p, ramsize;
|
|
|
|
mib[0] = CTL_HW;
|
|
mib[1] = HW_MEMSIZE;
|
|
sysctl(mib, 2, NULL, &len, NULL, 0);
|
|
p = malloc(len);
|
|
sysctl(mib, 2, p, &len, NULL, 0);
|
|
ramsize = *p;
|
|
|
|
return ramsize;
|
|
}
|
|
#else /* __APPLE__ */
|
|
#define memstream_update_buffer(A, B, C) (0)
|
|
static inline i64 get_ram(void)
|
|
{
|
|
return (i64)sysconf(_SC_PHYS_PAGES) * (i64)sysconf(_SC_PAGE_SIZE);
|
|
}
|
|
#endif
|
|
|
|
#define FLAG_SHOW_PROGRESS 2
|
|
#define FLAG_KEEP_FILES 4
|
|
#define FLAG_TEST_ONLY 8
|
|
#define FLAG_FORCE_REPLACE 16
|
|
#define FLAG_DECOMPRESS 32
|
|
#define FLAG_NO_COMPRESS 64
|
|
#define FLAG_LZO_COMPRESS 128
|
|
#define FLAG_BZIP2_COMPRESS 256
|
|
#define FLAG_ZLIB_COMPRESS 512
|
|
#define FLAG_ZPAQ_COMPRESS 1024
|
|
#define FLAG_VERBOSITY 2048
|
|
#define FLAG_VERBOSITY_MAX 4096
|
|
#define FLAG_NO_SET_PERMS 8192
|
|
#define FLAG_STDIN 16384
|
|
#define FLAG_STDOUT 32768
|
|
#define FLAG_INFO 65536
|
|
#define FLAG_MAXRAM 131072
|
|
#define FLAG_UNLIMITED 262144
|
|
|
|
#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
|
|
#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS)
|
|
#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA))
|
|
|
|
#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS)
|
|
#define KEEP_FILES (control.flags & FLAG_KEEP_FILES)
|
|
#define TEST_ONLY (control.flags & FLAG_TEST_ONLY)
|
|
#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE)
|
|
#define DECOMPRESS (control.flags & FLAG_DECOMPRESS)
|
|
#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS)
|
|
#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS)
|
|
#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS)
|
|
#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS)
|
|
#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS)
|
|
#define VERBOSE (control.flags & FLAG_VERBOSE)
|
|
#define VERBOSITY (control.flags & FLAG_VERBOSITY)
|
|
#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX)
|
|
#define NO_SET_PERMS (control.flags & FLAG_NO_SET_PERMS)
|
|
#define STDIN (control.flags & FLAG_STDIN)
|
|
#define STDOUT (control.flags & FLAG_STDOUT)
|
|
#define INFO (control.flags & FLAG_INFO)
|
|
#define MAXRAM (control.flags & FLAG_MAXRAM)
|
|
#define UNLIMITED (control.flags & FLAG_UNLIMITED)
|
|
|
|
#define BITS32 (sizeof(long) == 4)
|
|
|
|
#define CTYPE_NONE 3
|
|
#define CTYPE_BZIP2 4
|
|
#define CTYPE_LZO 5
|
|
#define CTYPE_LZMA 6
|
|
#define CTYPE_GZIP 7
|
|
#define CTYPE_ZPAQ 8
|
|
|
|
struct rzip_control {
|
|
char *infile;
|
|
char *outname;
|
|
char *outfile;
|
|
char *outdir;
|
|
FILE *msgout; //stream for output messages
|
|
const char *suffix;
|
|
int compression_level;
|
|
unsigned char lzma_properties[5]; // lzma properties, encoded
|
|
double threshold;
|
|
unsigned long long window;
|
|
unsigned long flags;
|
|
unsigned long long ramsize;
|
|
int threads;
|
|
int nice_val; // added for consistency
|
|
int major_version;
|
|
int minor_version;
|
|
i64 st_size;
|
|
long page_size;
|
|
} control;
|
|
|
|
struct stream {
|
|
i64 last_head;
|
|
uchar *buf;
|
|
i64 buflen;
|
|
i64 bufp;
|
|
};
|
|
|
|
struct stream_info {
|
|
struct stream *s;
|
|
int num_streams;
|
|
int fd;
|
|
i64 bufsize;
|
|
i64 cur_pos;
|
|
i64 initial_pos;
|
|
i64 total_read;
|
|
};
|
|
|
|
void fatal(const char *format, ...);
|
|
void sighandler();
|
|
i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size);
|
|
void rzip_fd(int fd_in, int fd_out);
|
|
void *open_stream_out(int f, int n, i64 limit);
|
|
void *open_stream_in(int f, int n);
|
|
int write_stream(void *ss, int stream, uchar *p, i64 len);
|
|
i64 read_stream(void *ss, int stream, uchar *p, i64 len);
|
|
int close_stream_out(void *ss);
|
|
int close_stream_in(void *ss);
|
|
int flush_buffer(struct stream_info *sinfo, int stream);
|
|
void read_config(struct rzip_control *s);
|
|
ssize_t write_1g(int fd, void *buf, i64 len);
|
|
ssize_t read_1g(int fd, void *buf, i64 len);
|
|
void zpipe_compress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress);
|
|
void zpipe_decompress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress);
|
|
const i64 two_gig;
|
|
|
|
#define print_err(format, args...) do {\
|
|
fprintf(stderr, format, ##args); \
|
|
} while (0)
|
|
|
|
#define print_output(format, args...) do {\
|
|
fprintf(control.msgout, format, ##args); \
|
|
fflush(control.msgout); \
|
|
} while (0)
|
|
|
|
#define print_progress(format, args...) do {\
|
|
if (SHOW_PROGRESS) \
|
|
print_output(format, ##args); \
|
|
} while (0)
|
|
|
|
#define print_verbose(format, args...) do {\
|
|
if (VERBOSE) \
|
|
print_output(format, ##args); \
|
|
} while (0)
|
|
|
|
#define print_maxverbose(format, args...) do {\
|
|
if (MAX_VERBOSE) \
|
|
print_output(format, ##args); \
|
|
} while (0)
|