Chris@43: /* gzappend -- command to append to a gzip file Chris@43: Chris@43: Copyright (C) 2003, 2012 Mark Adler, all rights reserved Chris@43: version 1.2, 11 Oct 2012 Chris@43: Chris@43: This software is provided 'as-is', without any express or implied Chris@43: warranty. In no event will the author be held liable for any damages Chris@43: arising from the use of this software. Chris@43: Chris@43: Permission is granted to anyone to use this software for any purpose, Chris@43: including commercial applications, and to alter it and redistribute it Chris@43: freely, subject to the following restrictions: Chris@43: Chris@43: 1. The origin of this software must not be misrepresented; you must not Chris@43: claim that you wrote the original software. If you use this software Chris@43: in a product, an acknowledgment in the product documentation would be Chris@43: appreciated but is not required. Chris@43: 2. Altered source versions must be plainly marked as such, and must not be Chris@43: misrepresented as being the original software. Chris@43: 3. This notice may not be removed or altered from any source distribution. Chris@43: Chris@43: Mark Adler madler@alumni.caltech.edu Chris@43: */ Chris@43: Chris@43: /* Chris@43: * Change history: Chris@43: * Chris@43: * 1.0 19 Oct 2003 - First version Chris@43: * 1.1 4 Nov 2003 - Expand and clarify some comments and notes Chris@43: * - Add version and copyright to help Chris@43: * - Send help to stdout instead of stderr Chris@43: * - Add some preemptive typecasts Chris@43: * - Add L to constants in lseek() calls Chris@43: * - Remove some debugging information in error messages Chris@43: * - Use new data_type definition for zlib 1.2.1 Chris@43: * - Simplfy and unify file operations Chris@43: * - Finish off gzip file in gztack() Chris@43: * - Use deflatePrime() instead of adding empty blocks Chris@43: * - Keep gzip file clean on appended file read errors Chris@43: * - Use in-place rotate instead of auxiliary buffer Chris@43: * (Why you ask? Because it was fun to write!) Chris@43: * 1.2 11 Oct 2012 - Fix for proper z_const usage Chris@43: * - Check for input buffer malloc failure Chris@43: */ Chris@43: Chris@43: /* Chris@43: gzappend takes a gzip file and appends to it, compressing files from the Chris@43: command line or data from stdin. The gzip file is written to directly, to Chris@43: avoid copying that file, in case it's large. Note that this results in the Chris@43: unfriendly behavior that if gzappend fails, the gzip file is corrupted. Chris@43: Chris@43: This program was written to illustrate the use of the new Z_BLOCK option of Chris@43: zlib 1.2.x's inflate() function. This option returns from inflate() at each Chris@43: block boundary to facilitate locating and modifying the last block bit at Chris@43: the start of the final deflate block. Also whether using Z_BLOCK or not, Chris@43: another required feature of zlib 1.2.x is that inflate() now provides the Chris@43: number of unusued bits in the last input byte used. gzappend will not work Chris@43: with versions of zlib earlier than 1.2.1. Chris@43: Chris@43: gzappend first decompresses the gzip file internally, discarding all but Chris@43: the last 32K of uncompressed data, and noting the location of the last block Chris@43: bit and the number of unused bits in the last byte of the compressed data. Chris@43: The gzip trailer containing the CRC-32 and length of the uncompressed data Chris@43: is verified. This trailer will be later overwritten. Chris@43: Chris@43: Then the last block bit is cleared by seeking back in the file and rewriting Chris@43: the byte that contains it. Seeking forward, the last byte of the compressed Chris@43: data is saved along with the number of unused bits to initialize deflate. Chris@43: Chris@43: A deflate process is initialized, using the last 32K of the uncompressed Chris@43: data from the gzip file to initialize the dictionary. If the total Chris@43: uncompressed data was less than 32K, then all of it is used to initialize Chris@43: the dictionary. The deflate output bit buffer is also initialized with the Chris@43: last bits from the original deflate stream. From here on, the data to Chris@43: append is simply compressed using deflate, and written to the gzip file. Chris@43: When that is complete, the new CRC-32 and uncompressed length are written Chris@43: as the trailer of the gzip file. Chris@43: */ Chris@43: Chris@43: #include Chris@43: #include Chris@43: #include Chris@43: #include Chris@43: #include Chris@43: #include "zlib.h" Chris@43: Chris@43: #define local static Chris@43: #define LGCHUNK 14 Chris@43: #define CHUNK (1U << LGCHUNK) Chris@43: #define DSIZE 32768U Chris@43: Chris@43: /* print an error message and terminate with extreme prejudice */ Chris@43: local void bye(char *msg1, char *msg2) Chris@43: { Chris@43: fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2); Chris@43: exit(1); Chris@43: } Chris@43: Chris@43: /* return the greatest common divisor of a and b using Euclid's algorithm, Chris@43: modified to be fast when one argument much greater than the other, and Chris@43: coded to avoid unnecessary swapping */ Chris@43: local unsigned gcd(unsigned a, unsigned b) Chris@43: { Chris@43: unsigned c; Chris@43: Chris@43: while (a && b) Chris@43: if (a > b) { Chris@43: c = b; Chris@43: while (a - c >= c) Chris@43: c <<= 1; Chris@43: a -= c; Chris@43: } Chris@43: else { Chris@43: c = a; Chris@43: while (b - c >= c) Chris@43: c <<= 1; Chris@43: b -= c; Chris@43: } Chris@43: return a + b; Chris@43: } Chris@43: Chris@43: /* rotate list[0..len-1] left by rot positions, in place */ Chris@43: local void rotate(unsigned char *list, unsigned len, unsigned rot) Chris@43: { Chris@43: unsigned char tmp; Chris@43: unsigned cycles; Chris@43: unsigned char *start, *last, *to, *from; Chris@43: Chris@43: /* normalize rot and handle degenerate cases */ Chris@43: if (len < 2) return; Chris@43: if (rot >= len) rot %= len; Chris@43: if (rot == 0) return; Chris@43: Chris@43: /* pointer to last entry in list */ Chris@43: last = list + (len - 1); Chris@43: Chris@43: /* do simple left shift by one */ Chris@43: if (rot == 1) { Chris@43: tmp = *list; Chris@43: memcpy(list, list + 1, len - 1); Chris@43: *last = tmp; Chris@43: return; Chris@43: } Chris@43: Chris@43: /* do simple right shift by one */ Chris@43: if (rot == len - 1) { Chris@43: tmp = *last; Chris@43: memmove(list + 1, list, len - 1); Chris@43: *list = tmp; Chris@43: return; Chris@43: } Chris@43: Chris@43: /* otherwise do rotate as a set of cycles in place */ Chris@43: cycles = gcd(len, rot); /* number of cycles */ Chris@43: do { Chris@43: start = from = list + cycles; /* start index is arbitrary */ Chris@43: tmp = *from; /* save entry to be overwritten */ Chris@43: for (;;) { Chris@43: to = from; /* next step in cycle */ Chris@43: from += rot; /* go right rot positions */ Chris@43: if (from > last) from -= len; /* (pointer better not wrap) */ Chris@43: if (from == start) break; /* all but one shifted */ Chris@43: *to = *from; /* shift left */ Chris@43: } Chris@43: *to = tmp; /* complete the circle */ Chris@43: } while (--cycles); Chris@43: } Chris@43: Chris@43: /* structure for gzip file read operations */ Chris@43: typedef struct { Chris@43: int fd; /* file descriptor */ Chris@43: int size; /* 1 << size is bytes in buf */ Chris@43: unsigned left; /* bytes available at next */ Chris@43: unsigned char *buf; /* buffer */ Chris@43: z_const unsigned char *next; /* next byte in buffer */ Chris@43: char *name; /* file name for error messages */ Chris@43: } file; Chris@43: Chris@43: /* reload buffer */ Chris@43: local int readin(file *in) Chris@43: { Chris@43: int len; Chris@43: Chris@43: len = read(in->fd, in->buf, 1 << in->size); Chris@43: if (len == -1) bye("error reading ", in->name); Chris@43: in->left = (unsigned)len; Chris@43: in->next = in->buf; Chris@43: return len; Chris@43: } Chris@43: Chris@43: /* read from file in, exit if end-of-file */ Chris@43: local int readmore(file *in) Chris@43: { Chris@43: if (readin(in) == 0) bye("unexpected end of ", in->name); Chris@43: return 0; Chris@43: } Chris@43: Chris@43: #define read1(in) (in->left == 0 ? readmore(in) : 0, \ Chris@43: in->left--, *(in->next)++) Chris@43: Chris@43: /* skip over n bytes of in */ Chris@43: local void skip(file *in, unsigned n) Chris@43: { Chris@43: unsigned bypass; Chris@43: Chris@43: if (n > in->left) { Chris@43: n -= in->left; Chris@43: bypass = n & ~((1U << in->size) - 1); Chris@43: if (bypass) { Chris@43: if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1) Chris@43: bye("seeking ", in->name); Chris@43: n -= bypass; Chris@43: } Chris@43: readmore(in); Chris@43: if (n > in->left) Chris@43: bye("unexpected end of ", in->name); Chris@43: } Chris@43: in->left -= n; Chris@43: in->next += n; Chris@43: } Chris@43: Chris@43: /* read a four-byte unsigned integer, little-endian, from in */ Chris@43: unsigned long read4(file *in) Chris@43: { Chris@43: unsigned long val; Chris@43: Chris@43: val = read1(in); Chris@43: val += (unsigned)read1(in) << 8; Chris@43: val += (unsigned long)read1(in) << 16; Chris@43: val += (unsigned long)read1(in) << 24; Chris@43: return val; Chris@43: } Chris@43: Chris@43: /* skip over gzip header */ Chris@43: local void gzheader(file *in) Chris@43: { Chris@43: int flags; Chris@43: unsigned n; Chris@43: Chris@43: if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file"); Chris@43: if (read1(in) != 8) bye("unknown compression method in", in->name); Chris@43: flags = read1(in); Chris@43: if (flags & 0xe0) bye("unknown header flags set in", in->name); Chris@43: skip(in, 6); Chris@43: if (flags & 4) { Chris@43: n = read1(in); Chris@43: n += (unsigned)(read1(in)) << 8; Chris@43: skip(in, n); Chris@43: } Chris@43: if (flags & 8) while (read1(in) != 0) ; Chris@43: if (flags & 16) while (read1(in) != 0) ; Chris@43: if (flags & 2) skip(in, 2); Chris@43: } Chris@43: Chris@43: /* decompress gzip file "name", return strm with a deflate stream ready to Chris@43: continue compression of the data in the gzip file, and return a file Chris@43: descriptor pointing to where to write the compressed data -- the deflate Chris@43: stream is initialized to compress using level "level" */ Chris@43: local int gzscan(char *name, z_stream *strm, int level) Chris@43: { Chris@43: int ret, lastbit, left, full; Chris@43: unsigned have; Chris@43: unsigned long crc, tot; Chris@43: unsigned char *window; Chris@43: off_t lastoff, end; Chris@43: file gz; Chris@43: Chris@43: /* open gzip file */ Chris@43: gz.name = name; Chris@43: gz.fd = open(name, O_RDWR, 0); Chris@43: if (gz.fd == -1) bye("cannot open ", name); Chris@43: gz.buf = malloc(CHUNK); Chris@43: if (gz.buf == NULL) bye("out of memory", ""); Chris@43: gz.size = LGCHUNK; Chris@43: gz.left = 0; Chris@43: Chris@43: /* skip gzip header */ Chris@43: gzheader(&gz); Chris@43: Chris@43: /* prepare to decompress */ Chris@43: window = malloc(DSIZE); Chris@43: if (window == NULL) bye("out of memory", ""); Chris@43: strm->zalloc = Z_NULL; Chris@43: strm->zfree = Z_NULL; Chris@43: strm->opaque = Z_NULL; Chris@43: ret = inflateInit2(strm, -15); Chris@43: if (ret != Z_OK) bye("out of memory", " or library mismatch"); Chris@43: Chris@43: /* decompress the deflate stream, saving append information */ Chris@43: lastbit = 0; Chris@43: lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; Chris@43: left = 0; Chris@43: strm->avail_in = gz.left; Chris@43: strm->next_in = gz.next; Chris@43: crc = crc32(0L, Z_NULL, 0); Chris@43: have = full = 0; Chris@43: do { Chris@43: /* if needed, get more input */ Chris@43: if (strm->avail_in == 0) { Chris@43: readmore(&gz); Chris@43: strm->avail_in = gz.left; Chris@43: strm->next_in = gz.next; Chris@43: } Chris@43: Chris@43: /* set up output to next available section of sliding window */ Chris@43: strm->avail_out = DSIZE - have; Chris@43: strm->next_out = window + have; Chris@43: Chris@43: /* inflate and check for errors */ Chris@43: ret = inflate(strm, Z_BLOCK); Chris@43: if (ret == Z_STREAM_ERROR) bye("internal stream error!", ""); Chris@43: if (ret == Z_MEM_ERROR) bye("out of memory", ""); Chris@43: if (ret == Z_DATA_ERROR) Chris@43: bye("invalid compressed data--format violated in", name); Chris@43: Chris@43: /* update crc and sliding window pointer */ Chris@43: crc = crc32(crc, window + have, DSIZE - have - strm->avail_out); Chris@43: if (strm->avail_out) Chris@43: have = DSIZE - strm->avail_out; Chris@43: else { Chris@43: have = 0; Chris@43: full = 1; Chris@43: } Chris@43: Chris@43: /* process end of block */ Chris@43: if (strm->data_type & 128) { Chris@43: if (strm->data_type & 64) Chris@43: left = strm->data_type & 0x1f; Chris@43: else { Chris@43: lastbit = strm->data_type & 0x1f; Chris@43: lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in; Chris@43: } Chris@43: } Chris@43: } while (ret != Z_STREAM_END); Chris@43: inflateEnd(strm); Chris@43: gz.left = strm->avail_in; Chris@43: gz.next = strm->next_in; Chris@43: Chris@43: /* save the location of the end of the compressed data */ Chris@43: end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; Chris@43: Chris@43: /* check gzip trailer and save total for deflate */ Chris@43: if (crc != read4(&gz)) Chris@43: bye("invalid compressed data--crc mismatch in ", name); Chris@43: tot = strm->total_out; Chris@43: if ((tot & 0xffffffffUL) != read4(&gz)) Chris@43: bye("invalid compressed data--length mismatch in", name); Chris@43: Chris@43: /* if not at end of file, warn */ Chris@43: if (gz.left || readin(&gz)) Chris@43: fprintf(stderr, Chris@43: "gzappend warning: junk at end of gzip file overwritten\n"); Chris@43: Chris@43: /* clear last block bit */ Chris@43: lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET); Chris@43: if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); Chris@43: *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7))); Chris@43: lseek(gz.fd, -1L, SEEK_CUR); Chris@43: if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name); Chris@43: Chris@43: /* if window wrapped, build dictionary from window by rotating */ Chris@43: if (full) { Chris@43: rotate(window, DSIZE, have); Chris@43: have = DSIZE; Chris@43: } Chris@43: Chris@43: /* set up deflate stream with window, crc, total_in, and leftover bits */ Chris@43: ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); Chris@43: if (ret != Z_OK) bye("out of memory", ""); Chris@43: deflateSetDictionary(strm, window, have); Chris@43: strm->adler = crc; Chris@43: strm->total_in = tot; Chris@43: if (left) { Chris@43: lseek(gz.fd, --end, SEEK_SET); Chris@43: if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); Chris@43: deflatePrime(strm, 8 - left, *gz.buf); Chris@43: } Chris@43: lseek(gz.fd, end, SEEK_SET); Chris@43: Chris@43: /* clean up and return */ Chris@43: free(window); Chris@43: free(gz.buf); Chris@43: return gz.fd; Chris@43: } Chris@43: Chris@43: /* append file "name" to gzip file gd using deflate stream strm -- if last Chris@43: is true, then finish off the deflate stream at the end */ Chris@43: local void gztack(char *name, int gd, z_stream *strm, int last) Chris@43: { Chris@43: int fd, len, ret; Chris@43: unsigned left; Chris@43: unsigned char *in, *out; Chris@43: Chris@43: /* open file to compress and append */ Chris@43: fd = 0; Chris@43: if (name != NULL) { Chris@43: fd = open(name, O_RDONLY, 0); Chris@43: if (fd == -1) Chris@43: fprintf(stderr, "gzappend warning: %s not found, skipping ...\n", Chris@43: name); Chris@43: } Chris@43: Chris@43: /* allocate buffers */ Chris@43: in = malloc(CHUNK); Chris@43: out = malloc(CHUNK); Chris@43: if (in == NULL || out == NULL) bye("out of memory", ""); Chris@43: Chris@43: /* compress input file and append to gzip file */ Chris@43: do { Chris@43: /* get more input */ Chris@43: len = read(fd, in, CHUNK); Chris@43: if (len == -1) { Chris@43: fprintf(stderr, Chris@43: "gzappend warning: error reading %s, skipping rest ...\n", Chris@43: name); Chris@43: len = 0; Chris@43: } Chris@43: strm->avail_in = (unsigned)len; Chris@43: strm->next_in = in; Chris@43: if (len) strm->adler = crc32(strm->adler, in, (unsigned)len); Chris@43: Chris@43: /* compress and write all available output */ Chris@43: do { Chris@43: strm->avail_out = CHUNK; Chris@43: strm->next_out = out; Chris@43: ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH); Chris@43: left = CHUNK - strm->avail_out; Chris@43: while (left) { Chris@43: len = write(gd, out + CHUNK - strm->avail_out - left, left); Chris@43: if (len == -1) bye("writing gzip file", ""); Chris@43: left -= (unsigned)len; Chris@43: } Chris@43: } while (strm->avail_out == 0 && ret != Z_STREAM_END); Chris@43: } while (len != 0); Chris@43: Chris@43: /* write trailer after last entry */ Chris@43: if (last) { Chris@43: deflateEnd(strm); Chris@43: out[0] = (unsigned char)(strm->adler); Chris@43: out[1] = (unsigned char)(strm->adler >> 8); Chris@43: out[2] = (unsigned char)(strm->adler >> 16); Chris@43: out[3] = (unsigned char)(strm->adler >> 24); Chris@43: out[4] = (unsigned char)(strm->total_in); Chris@43: out[5] = (unsigned char)(strm->total_in >> 8); Chris@43: out[6] = (unsigned char)(strm->total_in >> 16); Chris@43: out[7] = (unsigned char)(strm->total_in >> 24); Chris@43: len = 8; Chris@43: do { Chris@43: ret = write(gd, out + 8 - len, len); Chris@43: if (ret == -1) bye("writing gzip file", ""); Chris@43: len -= ret; Chris@43: } while (len); Chris@43: close(gd); Chris@43: } Chris@43: Chris@43: /* clean up and return */ Chris@43: free(out); Chris@43: free(in); Chris@43: if (fd > 0) close(fd); Chris@43: } Chris@43: Chris@43: /* process the compression level option if present, scan the gzip file, and Chris@43: append the specified files, or append the data from stdin if no other file Chris@43: names are provided on the command line -- the gzip file must be writable Chris@43: and seekable */ Chris@43: int main(int argc, char **argv) Chris@43: { Chris@43: int gd, level; Chris@43: z_stream strm; Chris@43: Chris@43: /* ignore command name */ Chris@43: argc--; argv++; Chris@43: Chris@43: /* provide usage if no arguments */ Chris@43: if (*argv == NULL) { Chris@43: printf( Chris@43: "gzappend 1.2 (11 Oct 2012) Copyright (C) 2003, 2012 Mark Adler\n" Chris@43: ); Chris@43: printf( Chris@43: "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n"); Chris@43: return 0; Chris@43: } Chris@43: Chris@43: /* set compression level */ Chris@43: level = Z_DEFAULT_COMPRESSION; Chris@43: if (argv[0][0] == '-') { Chris@43: if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0) Chris@43: bye("invalid compression level", ""); Chris@43: level = argv[0][1] - '0'; Chris@43: if (*++argv == NULL) bye("no gzip file name after options", ""); Chris@43: } Chris@43: Chris@43: /* prepare to append to gzip file */ Chris@43: gd = gzscan(*argv++, &strm, level); Chris@43: Chris@43: /* append files on command line, or from stdin if none */ Chris@43: if (*argv == NULL) Chris@43: gztack(NULL, gd, &strm, 1); Chris@43: else Chris@43: do { Chris@43: gztack(*argv, gd, &strm, argv[1] == NULL); Chris@43: } while (*++argv != NULL); Chris@43: return 0; Chris@43: }