ckolivas/lrzip
1
0
Fork 0
mirror of https://github.com/ckolivas/lrzip.git synced 2025-12-06 07:12:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Change the byte width to be variable depending on chunk size, and write it as a single char describing the next byte width for decompression.

Browse source 
More minor tidying.
This commit is contained in:
Con Kolivas 2010-11-01 13:28:49 +11:00
parent cb27097cb4
commit 1ed2ce423f
2 changed files with 50 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
runzip.c
View file

@ -132,11 +132,12 @@ static i64 unzip_match(void *ss, i64 len, int fd_out, int fd_hist, uint32 *cksum
/* decompress a section of an open file. Call fatal() on error /* decompress a section of an open file. Call fatal() on error
return the number of bytes that have been retrieved return the number of bytes that have been retrieved
*/ */
static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally, char chunk_bytes) static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i64 tally)
{ {
uint32 good_cksum, cksum = 0; uint32 good_cksum, cksum = 0;
i64 len, ofs, total = 0; i64 len, ofs, total = 0;
int l = -1, p = 0; int l = -1, p = 0;
char chunk_bytes;
struct stat st; struct stat st;
uchar head; uchar head;
void *ss; void *ss;
@ -165,6 +166,20 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
if (fstat(fd_in, &st) != 0 || st.st_size-ofs == 0) if (fstat(fd_in, &st) != 0 || st.st_size-ofs == 0)
return 0; return 0;
/* Determine the chunk_byte width size. Versions < 0.4 used 4
* bytes for all offsets, version 0.4 used 8 bytes. Versions 0.5+ use
* a variable number of bytes depending on chunk size.*/
if (control.major_version == 0 && control.minor_version < 4)
chunk_bytes = 4;
else if (control.major_version == 0 && control.minor_version == 4)
chunk_bytes = 8;
else {
/* Read in the stored chunk byte width from the file */
if (read(fd_in, &chunk_bytes, 1) != 1)
fatal("Failed to read chunk_bytes size in runzip_chunk\n");
}
print_maxverbose("\nExpected size: %lld\nChunk byte width: %d\n", expected_size, chunk_bytes);
ss = open_stream_in(fd_in, NUM_STREAMS); ss = open_stream_in(fd_in, NUM_STREAMS);
if (!ss) if (!ss)
fatal("Failed to open_stream_in in runzip_chunk\n"); fatal("Failed to open_stream_in in runzip_chunk\n");
@ -180,16 +195,13 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
break; break;
} }
p = 100 * ((double)(tally + total) / (double)expected_size); p = 100 * ((double)(tally + total) / (double)expected_size);
if (SHOW_PROGRESS) {
if ( p != l ) { if ( p != l ) {
prog_done = (double)(tally + total) / (double)divisor[divisor_index]; prog_done = (double)(tally + total) / (double)divisor[divisor_index];
print_output("%3d%% %9.2f / %9.2f %s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b", print_progress("%3d%% %9.2f / %9.2f %s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b",
p, prog_done, prog_tsize, suffix[divisor_index] ); p, prog_done, prog_tsize, suffix[divisor_index] );
fflush(control.msgout);
l = p; l = p;
} }
} }
}
good_cksum = read_u32(ss, 0); good_cksum = read_u32(ss, 0);
if (good_cksum != cksum) if (good_cksum != cksum)
@ -207,31 +219,12 @@ static i64 runzip_chunk(int fd_in, int fd_out, int fd_hist, i64 expected_size, i
i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size) i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size)
{ {
struct timeval start,end; struct timeval start,end;
char chunk_bytes;
i64 total = 0; i64 total = 0;
gettimeofday(&start,NULL); gettimeofday(&start,NULL);
/* Determine the chunk_byte width size. Versions < 0.4 used 4
* bytes for all offsets, version 0.4 used 8 bytes. Versions 0.5+ use
* a variable number of bytes depending on file size.*/
if (control.major_version == 0 && control.minor_version < 4)
chunk_bytes = 4;
else if (control.major_version == 0 && control.minor_version == 4)
chunk_bytes = 8;
else {
int bits = 0;
while (expected_size >> bits > 0)
bits++;
chunk_bytes = bits / 8;
if (bits % 8)
chunk_bytes++;
}
print_maxverbose("\nExpected size: %lld\nChunk byte width: %d\n", expected_size, chunk_bytes);
while (total < expected_size) while (total < expected_size)
total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total, chunk_bytes); total += runzip_chunk(fd_in, fd_out, fd_hist, expected_size, total);
gettimeofday(&end,NULL); gettimeofday(&end,NULL);
print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n", print_progress("\nAverage DeCompression Speed: %6.3fMB/s\n",

30
rzip.c
View file

@ -482,7 +482,6 @@ static void hash_search(struct rzip_state *st, uchar *buf,
fstat(st->fd_in, &s1); fstat(st->fd_in, &s1);
fstat(st->fd_out, &s2); fstat(st->fd_out, &s2);
print_output("%2lld%%\r", pct); print_output("%2lld%%\r", pct);
fflush(control.msgout);
lastpct = pct; lastpct = pct;
} }
} }
@ -495,7 +494,7 @@ static void hash_search(struct rzip_state *st, uchar *buf,
} }
if (control.flags & FLAG_VERBOSITY_MAX) if (MAX_VERBOSE)
show_distrib(st); show_distrib(st);
if (st->last_match < buf + st->chunk_size) if (st->last_match < buf + st->chunk_size)
@ -579,7 +578,7 @@ void rzip_fd(int fd_in, int fd_out)
struct stat s, s2; struct stat s, s2;
struct rzip_state *st; struct rzip_state *st;
i64 len, chunk_window, last_chunk = 0; i64 len, chunk_window, last_chunk = 0;
int pass = 0, passes, bits = 8; int pass = 0, passes;
unsigned int eta_hours, eta_minutes, eta_seconds, elapsed_hours, unsigned int eta_hours, eta_minutes, eta_seconds, elapsed_hours,
elapsed_minutes, elapsed_seconds; elapsed_minutes, elapsed_seconds;
double finish_time, elapsed_time, chunkmbs; double finish_time, elapsed_time, chunkmbs;
@ -588,7 +587,7 @@ void rzip_fd(int fd_in, int fd_out)
if (!st) if (!st)
fatal("Failed to allocate control state in rzip_fd\n"); fatal("Failed to allocate control state in rzip_fd\n");
if (control.flags & FLAG_LZO_COMPRESS) { if (LZO_COMPRESS) {
if (lzo_init() != LZO_E_OK) if (lzo_init() != LZO_E_OK)
fatal("lzo_init() failed\n"); fatal("lzo_init() failed\n");
} }
@ -598,12 +597,6 @@ void rzip_fd(int fd_in, int fd_out)
len = s.st_size; len = s.st_size;
print_verbose("File size: %lld\n", len); print_verbose("File size: %lld\n", len);
while (len >> bits > 0)
bits++;
st->chunk_bytes = bits / 8;
if (bits % 8)
st->chunk_bytes++;
print_maxverbose("Byte width: %d\n", st->chunk_bytes);
chunk_window = control.window * CHUNK_MULTIPLE; chunk_window = control.window * CHUNK_MULTIPLE;
@ -622,6 +615,7 @@ void rzip_fd(int fd_in, int fd_out)
while (len) { while (len) {
double pct_base, pct_multiple; double pct_base, pct_multiple;
i64 chunk, limit = 0; i64 chunk, limit = 0;
int bits = 8;
/* Flushing the dirty data will decrease our chances of /* Flushing the dirty data will decrease our chances of
* running out of memory when we allocate ram again on the * running out of memory when we allocate ram again on the
@ -637,6 +631,18 @@ void rzip_fd(int fd_in, int fd_out)
st->chunk_size = chunk; st->chunk_size = chunk;
print_maxverbose("Chunk size: %lld\n\n", chunk); print_maxverbose("Chunk size: %lld\n\n", chunk);
/* Determine the chunk byte width and write it to the file
* This allows archives of different chunk sizes to have
* optimal byte width entries. */
while (chunk >> bits > 0)
bits++;
st->chunk_bytes = bits / 8;
if (bits % 8)
st->chunk_bytes++;
print_maxverbose("Byte width: %d\n", st->chunk_bytes);
if (write(fd_out, &st->chunk_bytes, 1) != 1)
fatal("Failed to write chunk_bytes size in rzip_fd\n");
pct_base = (100.0 * (s.st_size - len)) / s.st_size; pct_base = (100.0 * (s.st_size - len)) / s.st_size;
pct_multiple = ((double)chunk) / s.st_size; pct_multiple = ((double)chunk) / s.st_size;
pass++; pass++;
@ -644,7 +650,6 @@ void rzip_fd(int fd_in, int fd_out)
gettimeofday(&current, NULL); gettimeofday(&current, NULL);
/* this will count only when size > window */ /* this will count only when size > window */
if (last.tv_sec > 0) { if (last.tv_sec > 0) {
if (VERBOSE) {
elapsed_time = current.tv_sec - start.tv_sec; elapsed_time = current.tv_sec - start.tv_sec;
finish_time = elapsed_time / (pct_base / 100.0); finish_time = elapsed_time / (pct_base / 100.0);
elapsed_hours = (unsigned int)(elapsed_time) / 3600; elapsed_hours = (unsigned int)(elapsed_time) / 3600;
@ -654,11 +659,10 @@ void rzip_fd(int fd_in, int fd_out)
eta_minutes = (unsigned int)((finish_time - elapsed_time) - eta_hours * 3600) / 60; eta_minutes = (unsigned int)((finish_time - elapsed_time) - eta_hours * 3600) / 60;
eta_seconds = (unsigned int)(finish_time - elapsed_time) - eta_hours * 60 - eta_minutes * 60; eta_seconds = (unsigned int)(finish_time - elapsed_time) - eta_hours * 60 - eta_minutes * 60;
chunkmbs=(last_chunk / 1024 / 1024) / (double)(current.tv_sec-last.tv_sec); chunkmbs=(last_chunk / 1024 / 1024) / (double)(current.tv_sec-last.tv_sec);
print_output("\nPass %d / %d -- Elapsed Time: %02d:%02d:%02d. ETA: %02d:%02d:%02d. Compress Speed: %3.3fMB/s.\n", print_verbose("\nPass %d / %d -- Elapsed Time: %02d:%02d:%02d. ETA: %02d:%02d:%02d. Compress Speed: %3.3fMB/s.\n",
pass, passes, elapsed_hours, elapsed_minutes, elapsed_seconds, pass, passes, elapsed_hours, elapsed_minutes, elapsed_seconds,
eta_hours, eta_minutes, eta_seconds, chunkmbs); eta_hours, eta_minutes, eta_seconds, chunkmbs);
} }
}
last.tv_sec = current.tv_sec; last.tv_sec = current.tv_sec;
last.tv_usec = current.tv_usec; last.tv_usec = current.tv_usec;
last_chunk = chunk; last_chunk = chunk;