Change byte width to be dependant on file size.

This will increase speed of compression and generate a smaller file, but not be backward compatible.
Tweak the way memory is allocated to optimise chances of success and minimise slowdown for the machine.
fsync to empty dirty data before allocating large ram to increase chance of mem allocation and decrease disk thrash of write vs read.
Add lots more information to verbose mode.
Lots of code tidying and minor tweaks.

main.c

@@ -1,6 +1,6 @@
 /*
    Copyright (C) Andrew Tridgell 1998-2003,
-   Con Kolivas 2006-2009
+   Con Kolivas 2006-2010
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

runzip.c

@@ -1,6 +1,6 @@
 /*
    Copyright (C) Andrew Tridgell 1998-2003
-   Con Kolivas 2006-2009
+   Con Kolivas 2006-2010
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -86,22 +86,13 @@ static i64 unzip_literal(void *ss, i64 len, int fd_out, uint32 *cksum)
 	return len;
 }
 
-static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum)
+static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum, int chunk_bytes)
 {
 	i64 offset, n, total, cur_pos;
-	int chunk_bytes = 8;
 
 	if (len < 0)
 		fatal("len %lld is negative in unzip_match!\n",len);
 
-	if (control.major_version == 0) {
-		/* Versions < 0.4 used 4 bytes for all offsets, version 0.4 used 8 bytes.
-		 * Versions 0.5+ used a variable number of bytes depending on block size. */
-		if (control.minor_version < 4)
-			chunk_bytes = 4;
-		else if (control.minor_version == 4)
-			chunk_bytes = 8;
-	}
 	total = 0;
 	cur_pos = lseek(fd_out, 0, SEEK_CUR);
 	if (cur_pos == -1)
@@ -140,16 +131,14 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
 /* decompress a section of an open file. Call fatal() on error
    return the number of bytes that have been retrieved
  */
-static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally)
+static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally, char chunk_bytes)
 {
-	uchar head;
+	i64 len, ofs, total = 0;
-	i64 len;
-	struct stat st;
-	void *ss;
-	i64 ofs;
-	i64 total = 0;
 	uint32 good_cksum, cksum = 0;
 	int l = -1, p = 0;
+	struct stat st;
+	uchar head;
+	void *ss;
 
 	/* for display of progress */
 	char *suffix[] = {"","KB","MB","GB"};
@@ -186,7 +175,7 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
 				break;
 
 			default:
-				total += unzip_match(ss, len, fd_out, fd_hist, &cksum);
+				total += unzip_match(ss, len, fd_out, fd_hist, &cksum, chunk_bytes);
 				break;
 		}
 		p = 100 * ((double)(tally + total) / (double)expected_size);
@@ -217,13 +206,33 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
 
 i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size)
 {
-	i64 total = 0;
 	struct timeval start,end;
+	char chunk_bytes;
+	i64 total = 0;
 
 	gettimeofday(&start,NULL);
 
+	/* Determine the chunk_byte width size. Versions < 0.4 used 4
+	 * bytes for all offsets, version 0.4 used 8 bytes. Versions 0.5+ use
+	 * a variable number of bytes depending on file size.*/
+	if (control.major_version == 0 && control.minor_version < 4)
+		chunk_bytes = 4;
+	else if (control.major_version == 0 && control.minor_version == 4)
+		chunk_bytes = 8;
+	else {
+		int bits = 0;
+
+		while (expected_size >> bits > 0)
+			bits++;
+		chunk_bytes = bits / 8;
+		if (bits % 8)
+			chunk_bytes++;
+	}
+	if (control.flags & FLAG_VERBOSE)
+		fprintf(control.msgout, "Expected size: %lld\nChunk byte width: %d\n", expected_size, chunk_bytes);
+
 	while (total < expected_size)
-		total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total);
+		total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total, chunk_bytes);
 
 	gettimeofday(&end,NULL);
 	if (control.flags & FLAG_SHOW_PROGRESS)

rzip.c

@@ -1,6 +1,6 @@
 /*
    Copyright (C) Andrew Tridgell 1998,
-   Con Kolivas 2006-2009
+   Con Kolivas 2006-2010
 
    Modified to use flat hash, memory limit and variable hash culling
    by Rusty Russell copyright (C) 2003.
@@ -71,6 +71,7 @@ struct rzip_state {
 	i64 tag_clean_ptr;
 	uchar *last_match;
 	i64 chunk_size;
+	char chunk_bytes;
 	uint32_t cksum;
 	int fd_in, fd_out;
 	struct {
@@ -124,7 +125,7 @@ static void put_match(struct rzip_state *st, uchar *p, uchar *buf, i64 offset, i
 
 		ofs = (p - (buf + offset));
 		put_header(st->ss, 1, n);
-		put_vchars(st->ss, 0, ofs, 8);
+		put_vchars(st->ss, 0, ofs, st->chunk_bytes);
 
 		st->stats.matches++;
 		st->stats.match_bytes += n;
@@ -171,6 +172,7 @@ static inline tag increase_mask(tag tag_mask)
 static int minimum_bitness(struct rzip_state *st, tag t)
 {
 	tag better_than_min = increase_mask(st->minimum_tag_mask);
+
 	if ((t & better_than_min) != better_than_min)
 		return 1;
 	return 0;
@@ -273,7 +275,7 @@ again:
 static inline tag next_tag(struct rzip_state *st, uchar *p, tag t)
 {
 	t ^= st->hash_index[p[-1]];
-	t ^= st->hash_index[p[MINIMUM_MATCH-1]];
+	t ^= st->hash_index[p[-1]];
 	return t;
 }
 
@@ -281,6 +283,7 @@ static inline tag full_tag(struct rzip_state *st, uchar *p)
 {
 	tag ret = 0;
 	int i;
+
 	for (i = 0; i < MINIMUM_MATCH; i++)
 		ret ^= st->hash_index[p[i]];
 	return ret;
@@ -327,8 +330,8 @@ static i64 find_best_match(struct rzip_state *st,
 			   i64 *offset, i64 *reverse)
 {
 	i64 length = 0;
-	i64 rev;
 	i64 h, best_h;
+	i64 rev;
 
 	rev = 0;
 	*reverse = 0;
@@ -365,9 +368,9 @@ static i64 find_best_match(struct rzip_state *st,
 
 static void show_distrib(struct rzip_state *st)
 {
-	i64 i;
-	i64 total = 0;
 	i64 primary = 0;
+	i64 total = 0;
+	i64 i;
 
 	for (i = 0; i < (1U << st->hash_bits); i++) {
 		if (empty_hash(st, i))
@@ -387,10 +390,9 @@ static void show_distrib(struct rzip_state *st)
 static void hash_search(struct rzip_state *st, uchar *buf,
 			double pct_base, double pct_multiple)
 {
+	i64 cksum_limit = 0, pct, lastpct=0;
 	uchar *p, *end;
 	tag t = 0;
-	i64 cksum_limit = 0;
-	i64 pct, lastpct=0;
 	struct {
 		uchar *p;
 		i64 ofs;
@@ -531,15 +533,17 @@ static void rzip_chunk(struct rzip_state *st, int fd_in, int fd_out, i64 offset,
 	 * faster than slowly reading in the file and then failing. Filling
 	 * it with zeroes has a defragmenting effect on ram before the real
 	 * read in. */
-	if (control.flags & FLAG_SHOW_PROGRESS)
+	if (control.flags & FLAG_VERBOSE)
-		fprintf(control.msgout, "Allocating ram...\n");
+		fprintf(control.msgout, "Preallocating ram...\n");
 	buf = malloc(st->chunk_size);
 	if (!buf)
 		fatal("Failed to premalloc in rzip_chunk\n");
 	if (!memset(buf, 0, st->chunk_size))
 		fatal("Failed to memset in rzip_chunk\n");
 	free(buf);
-	buf = (uchar *)mmap(buf, st->chunk_size, PROT_READ, MAP_SHARED | MAP_POPULATE, fd_in, offset);
+	if (control.flags & FLAG_VERBOSE)
+		fprintf(control.msgout, "Reading file into mmapped ram...\n");
+	buf = (uchar *)mmap(buf, st->chunk_size, PROT_READ, MAP_SHARED, fd_in, offset);
 	if (buf == (uchar *)-1)
 		fatal("Failed to map buffer in rzip_chunk\n");
 
@@ -572,13 +576,11 @@ void rzip_fd(int fd_in, int fd_out)
 	 * Track elapsed time and estimated time to go
 	 * If file size < compression window, can't do
 	 */
-
 	struct timeval current, start, last;
 	struct stat s, s2;
 	struct rzip_state *st;
-	i64 len, last_chunk = 0;
+	i64 len, chunk_window, last_chunk = 0;
-	i64 chunk_window;
+	int pass = 0, passes, bits = 8;
-	int pass = 0, passes;
 	unsigned int eta_hours, eta_minutes, eta_seconds, elapsed_hours,
 		     elapsed_minutes, elapsed_seconds;
 	double finish_time, elapsed_time, chunkmbs;
@@ -596,10 +598,17 @@ void rzip_fd(int fd_in, int fd_out)
 		fatal("Failed to stat fd_in in rzip_fd - %s\n", strerror(errno));
 
 	len = s.st_size;
+	if (control.flags & FLAG_VERBOSE)
+		fprintf(control.msgout, "File size: %lld\n", len);
+	while (len >> bits > 0)
+		bits++;
+	st->chunk_bytes = bits / 8;
+	if (bits % 8)
+		st->chunk_bytes++;
+	if (control.flags & FLAG_VERBOSE)
+		fprintf(control.msgout, "Byte width: %d\n", st->chunk_bytes);
 
-	/* Windows must be the width of _SC_PAGE_SIZE for offset to work in mmap */
 	chunk_window = control.window * CHUNK_MULTIPLE;
-	round_to_page_size(&chunk_window);
 
 	st->level = &levels[MIN(9, control.window)];
 	st->fd_in = fd_in;
@@ -614,22 +623,27 @@ void rzip_fd(int fd_in, int fd_out)
 	gettimeofday(&start, NULL);
 
 	while (len > 0) {
-		i64 chunk, limit = 0;
 		double pct_base, pct_multiple;
+		i64 chunk, limit = 0;
 
+		/* Flushing the dirty data will decrease our chances of
+		 * running out of memory when we allocate ram again on the
+		 * next chunk. It will also prevent thrashing on-disk due to
+		 * concurrent reads and writes if we're on the same device. */
+		if (last_chunk && control.flags & FLAG_VERBOSE)
+			fprintf(control.msgout, "Flushing data to disk.\n");
+		fsync(fd_out);
 		chunk = chunk_window;
-
+		if (chunk > len)
-		if (chunk > len) {
 			chunk = len;
 		round_to_page_size(&chunk);
 		limit = chunk;
-		}
+		st->chunk_size = chunk;
+		if (control.flags & FLAG_VERBOSE)
+			fprintf(control.msgout, "Chunk size: %lld\n\n", chunk);
 
 		pct_base = (100.0 * (s.st_size - len)) / s.st_size;
 		pct_multiple = ((double)chunk) / s.st_size;
-
-		st->chunk_size = chunk;
-
 		pass++;
 
 		gettimeofday(&current, NULL);

rzip.h

@@ -1,6 +1,6 @@
 /*
    Copyright (C) Andrew Tridgell 1998,
-   Con Kolivas 2006-2009
+   Con Kolivas 2006-2010
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

stream.c

@@ -1,6 +1,6 @@
 /*
    Copyright (C) Andrew Tridgell 1998,
-   Con Kolivas 2006-2009
+   Con Kolivas 2006-2010
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -120,8 +120,8 @@ static int memstream_update_buffer(FILE *fp, uchar **buf, size_t *length)
 
 static void zpaq_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 {
-	uchar *c_buf;
 	FILE *in, *out;
+	uchar *c_buf;
 	size_t dlen;
 
 	if (!lzo_compresses(s))
@@ -156,8 +156,8 @@ static void zpaq_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 
 static void bzip2_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 {
-	uchar *c_buf;
 	u32 dlen = s->buflen;
+	uchar *c_buf;
 
 	if (!lzo_compresses(s))
 		return;
@@ -187,8 +187,8 @@ static void bzip2_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 
 static void gzip_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 {
-	uchar *c_buf;
 	unsigned long dlen = s->buflen;
+	uchar *c_buf;
 
 	c_buf = malloc(dlen);
 	if (!c_buf)
@@ -213,9 +213,9 @@ static void gzip_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 
 static void lzma_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 {
+	size_t prop_size = 5; /* return value for lzma_properties */
 	uchar *c_buf;
 	size_t dlen;
-	size_t prop_size = 5; /* return value for lzma_properties */
 	int lzma_ret;
 
 	if (!lzo_compresses(s))
@@ -279,11 +279,11 @@ out:
 
 static void lzo_compress_buf(struct stream *s, int *c_type, i64 *c_len)
 {
-	uchar *c_buf;
-	lzo_bytep wrkmem;
 	lzo_uint in_len = s->buflen;
 	lzo_uint dlen = in_len + in_len / 16 + 64 + 3;
 	lzo_int return_var;	/* lzo1x_1_compress does not return anything but LZO_OK */
+	lzo_bytep wrkmem;
+	uchar *c_buf;
 
 	wrkmem = (lzo_bytep) malloc(LZO1X_1_MEM_COMPRESS);
 	if (wrkmem == NULL)
@@ -320,8 +320,8 @@ out_free:
 
 static int zpaq_decompress_buf(struct stream *s)
 {
-	uchar *c_buf;
 	FILE *in, *out;
+	uchar *c_buf;
 	size_t dlen;
 
 	in = fmemopen(s->buf, s->buflen, "r");
@@ -355,8 +355,8 @@ static int zpaq_decompress_buf(struct stream *s)
 
 static int bzip2_decompress_buf(struct stream *s, i64 c_len)
 {
-	uchar *c_buf;
 	u32 dlen = s->buflen;
+	uchar *c_buf;
 	int bzerr;
 
 	c_buf = s->buf;
@@ -383,8 +383,8 @@ static int bzip2_decompress_buf(struct stream *s, i64 c_len)
 
 static int gzip_decompress_buf(struct stream *s, i64 c_len)
 {
-	uchar *c_buf;
 	unsigned long dlen = s->buflen;
+	uchar *c_buf;
 	int gzerr;
 
 	c_buf = s->buf;
@@ -411,8 +411,8 @@ static int gzip_decompress_buf(struct stream *s, i64 c_len)
 
 static int lzma_decompress_buf(struct stream *s, size_t c_len)
 {
-	uchar *c_buf;
 	size_t dlen = (size_t)s->buflen;
+	uchar *c_buf;
 	int lzmaerr;
 
 	c_buf = s->buf;
@@ -441,8 +441,8 @@ static int lzma_decompress_buf(struct stream *s, size_t c_len)
 
 static int lzo_decompress_buf(struct stream *s, i64 c_len)
 {
-	uchar *c_buf;
 	lzo_uint dlen = s->buflen;
+	uchar *c_buf;
 	int lzerr;
 
 	c_buf = s->buf;
@@ -476,9 +476,9 @@ const i64 one_g = 1000 * 1024 * 1024;
    even on the rare occasion write() fails to write 1GB as well. */
 ssize_t write_1g(int fd, void *buf, i64 len)
 {
+	uchar *offset_buf = buf;
 	i64 total, offset;
 	ssize_t ret;
-	uchar *offset_buf = buf;
 
 	total = offset = 0;
 	while (len > 0) {
@@ -499,9 +499,9 @@ ssize_t write_1g(int fd, void *buf, i64 len)
 /* Ditto for read */
 ssize_t read_1g(int fd, void *buf, i64 len)
 {
+	uchar *offset_buf = buf;
 	i64 total, offset;
 	ssize_t ret;
-	uchar *offset_buf = buf;
 
 	total = offset = 0;
 	while (len > 0) {
@@ -590,6 +590,7 @@ static int read_i64(int f, i64 *v)
 static int seekto(struct stream_info *sinfo, i64 pos)
 {
 	i64 spos = pos + sinfo->initial_pos;
+
 	if (lseek(sinfo->fd, spos, SEEK_SET) != spos) {
 		err_msg("Failed to seek to %d in stream\n", pos);
 		return -1;
@@ -602,8 +603,8 @@ static int seekto(struct stream_info *sinfo, i64 pos)
 void *open_stream_out(int f, int n, i64 limit)
 {
 	unsigned cwindow = control.window;
-	int i;
 	struct stream_info *sinfo;
+	int i;
 
 	sinfo = malloc(sizeof(*sinfo));
 	if (!sinfo)
@@ -675,9 +676,9 @@ failed:
 /* prepare a set of n streams for reading on file descriptor f */
 void *open_stream_in(int f, int n)
 {
+	struct stream_info *sinfo;
 	i64 header_length;
 	int i;
-	struct stream_info *sinfo;
 
 	sinfo = calloc(sizeof(*sinfo), 1);
 	if (!sinfo)
@@ -759,8 +760,8 @@ failed:
 /* flush out any data in a stream buffer. Return -1 on failure */
 static int flush_buffer(struct stream_info *sinfo, int stream)
 {
-	int c_type = CTYPE_NONE;
 	i64 c_len = sinfo->s[stream].buflen;
+	int c_type = CTYPE_NONE;
 
 	if (seekto(sinfo, sinfo->s[stream].last_head) != 0)
 		return -1;
@@ -809,9 +810,8 @@ static int flush_buffer(struct stream_info *sinfo, int stream)
 /* fill a buffer from a stream - return -1 on failure */
 static int fill_buffer(struct stream_info *sinfo, int stream)
 {
+	i64 header_length, u_len, c_len;
 	uchar c_type;
-	i64 header_length;
-	i64 u_len, c_len;
 
 	if (seekto(sinfo, sinfo->s[stream].last_head) != 0)
 		return -1;