cannam@128: /* gzappend -- command to append to a gzip file
cannam@128: 
cannam@128:   Copyright (C) 2003, 2012 Mark Adler, all rights reserved
cannam@128:   version 1.2, 11 Oct 2012
cannam@128: 
cannam@128:   This software is provided 'as-is', without any express or implied
cannam@128:   warranty.  In no event will the author be held liable for any damages
cannam@128:   arising from the use of this software.
cannam@128: 
cannam@128:   Permission is granted to anyone to use this software for any purpose,
cannam@128:   including commercial applications, and to alter it and redistribute it
cannam@128:   freely, subject to the following restrictions:
cannam@128: 
cannam@128:   1. The origin of this software must not be misrepresented; you must not
cannam@128:      claim that you wrote the original software. If you use this software
cannam@128:      in a product, an acknowledgment in the product documentation would be
cannam@128:      appreciated but is not required.
cannam@128:   2. Altered source versions must be plainly marked as such, and must not be
cannam@128:      misrepresented as being the original software.
cannam@128:   3. This notice may not be removed or altered from any source distribution.
cannam@128: 
cannam@128:   Mark Adler    madler@alumni.caltech.edu
cannam@128:  */
cannam@128: 
cannam@128: /*
cannam@128:  * Change history:
cannam@128:  *
cannam@128:  * 1.0  19 Oct 2003     - First version
cannam@128:  * 1.1   4 Nov 2003     - Expand and clarify some comments and notes
cannam@128:  *                      - Add version and copyright to help
cannam@128:  *                      - Send help to stdout instead of stderr
cannam@128:  *                      - Add some preemptive typecasts
cannam@128:  *                      - Add L to constants in lseek() calls
cannam@128:  *                      - Remove some debugging information in error messages
cannam@128:  *                      - Use new data_type definition for zlib 1.2.1
cannam@128:  *                      - Simplfy and unify file operations
cannam@128:  *                      - Finish off gzip file in gztack()
cannam@128:  *                      - Use deflatePrime() instead of adding empty blocks
cannam@128:  *                      - Keep gzip file clean on appended file read errors
cannam@128:  *                      - Use in-place rotate instead of auxiliary buffer
cannam@128:  *                        (Why you ask?  Because it was fun to write!)
cannam@128:  * 1.2  11 Oct 2012     - Fix for proper z_const usage
cannam@128:  *                      - Check for input buffer malloc failure
cannam@128:  */
cannam@128: 
cannam@128: /*
cannam@128:    gzappend takes a gzip file and appends to it, compressing files from the
cannam@128:    command line or data from stdin.  The gzip file is written to directly, to
cannam@128:    avoid copying that file, in case it's large.  Note that this results in the
cannam@128:    unfriendly behavior that if gzappend fails, the gzip file is corrupted.
cannam@128: 
cannam@128:    This program was written to illustrate the use of the new Z_BLOCK option of
cannam@128:    zlib 1.2.x's inflate() function.  This option returns from inflate() at each
cannam@128:    block boundary to facilitate locating and modifying the last block bit at
cannam@128:    the start of the final deflate block.  Also whether using Z_BLOCK or not,
cannam@128:    another required feature of zlib 1.2.x is that inflate() now provides the
cannam@128:    number of unusued bits in the last input byte used.  gzappend will not work
cannam@128:    with versions of zlib earlier than 1.2.1.
cannam@128: 
cannam@128:    gzappend first decompresses the gzip file internally, discarding all but
cannam@128:    the last 32K of uncompressed data, and noting the location of the last block
cannam@128:    bit and the number of unused bits in the last byte of the compressed data.
cannam@128:    The gzip trailer containing the CRC-32 and length of the uncompressed data
cannam@128:    is verified.  This trailer will be later overwritten.
cannam@128: 
cannam@128:    Then the last block bit is cleared by seeking back in the file and rewriting
cannam@128:    the byte that contains it.  Seeking forward, the last byte of the compressed
cannam@128:    data is saved along with the number of unused bits to initialize deflate.
cannam@128: 
cannam@128:    A deflate process is initialized, using the last 32K of the uncompressed
cannam@128:    data from the gzip file to initialize the dictionary.  If the total
cannam@128:    uncompressed data was less than 32K, then all of it is used to initialize
cannam@128:    the dictionary.  The deflate output bit buffer is also initialized with the
cannam@128:    last bits from the original deflate stream.  From here on, the data to
cannam@128:    append is simply compressed using deflate, and written to the gzip file.
cannam@128:    When that is complete, the new CRC-32 and uncompressed length are written
cannam@128:    as the trailer of the gzip file.
cannam@128:  */
cannam@128: 
cannam@128: #include <stdio.h>
cannam@128: #include <stdlib.h>
cannam@128: #include <string.h>
cannam@128: #include <fcntl.h>
cannam@128: #include <unistd.h>
cannam@128: #include "zlib.h"
cannam@128: 
cannam@128: #define local static
cannam@128: #define LGCHUNK 14
cannam@128: #define CHUNK (1U << LGCHUNK)
cannam@128: #define DSIZE 32768U
cannam@128: 
cannam@128: /* print an error message and terminate with extreme prejudice */
cannam@128: local void bye(char *msg1, char *msg2)
cannam@128: {
cannam@128:     fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2);
cannam@128:     exit(1);
cannam@128: }
cannam@128: 
cannam@128: /* return the greatest common divisor of a and b using Euclid's algorithm,
cannam@128:    modified to be fast when one argument much greater than the other, and
cannam@128:    coded to avoid unnecessary swapping */
cannam@128: local unsigned gcd(unsigned a, unsigned b)
cannam@128: {
cannam@128:     unsigned c;
cannam@128: 
cannam@128:     while (a && b)
cannam@128:         if (a > b) {
cannam@128:             c = b;
cannam@128:             while (a - c >= c)
cannam@128:                 c <<= 1;
cannam@128:             a -= c;
cannam@128:         }
cannam@128:         else {
cannam@128:             c = a;
cannam@128:             while (b - c >= c)
cannam@128:                 c <<= 1;
cannam@128:             b -= c;
cannam@128:         }
cannam@128:     return a + b;
cannam@128: }
cannam@128: 
cannam@128: /* rotate list[0..len-1] left by rot positions, in place */
cannam@128: local void rotate(unsigned char *list, unsigned len, unsigned rot)
cannam@128: {
cannam@128:     unsigned char tmp;
cannam@128:     unsigned cycles;
cannam@128:     unsigned char *start, *last, *to, *from;
cannam@128: 
cannam@128:     /* normalize rot and handle degenerate cases */
cannam@128:     if (len < 2) return;
cannam@128:     if (rot >= len) rot %= len;
cannam@128:     if (rot == 0) return;
cannam@128: 
cannam@128:     /* pointer to last entry in list */
cannam@128:     last = list + (len - 1);
cannam@128: 
cannam@128:     /* do simple left shift by one */
cannam@128:     if (rot == 1) {
cannam@128:         tmp = *list;
cannam@128:         memcpy(list, list + 1, len - 1);
cannam@128:         *last = tmp;
cannam@128:         return;
cannam@128:     }
cannam@128: 
cannam@128:     /* do simple right shift by one */
cannam@128:     if (rot == len - 1) {
cannam@128:         tmp = *last;
cannam@128:         memmove(list + 1, list, len - 1);
cannam@128:         *list = tmp;
cannam@128:         return;
cannam@128:     }
cannam@128: 
cannam@128:     /* otherwise do rotate as a set of cycles in place */
cannam@128:     cycles = gcd(len, rot);             /* number of cycles */
cannam@128:     do {
cannam@128:         start = from = list + cycles;   /* start index is arbitrary */
cannam@128:         tmp = *from;                    /* save entry to be overwritten */
cannam@128:         for (;;) {
cannam@128:             to = from;                  /* next step in cycle */
cannam@128:             from += rot;                /* go right rot positions */
cannam@128:             if (from > last) from -= len;   /* (pointer better not wrap) */
cannam@128:             if (from == start) break;   /* all but one shifted */
cannam@128:             *to = *from;                /* shift left */
cannam@128:         }
cannam@128:         *to = tmp;                      /* complete the circle */
cannam@128:     } while (--cycles);
cannam@128: }
cannam@128: 
cannam@128: /* structure for gzip file read operations */
cannam@128: typedef struct {
cannam@128:     int fd;                     /* file descriptor */
cannam@128:     int size;                   /* 1 << size is bytes in buf */
cannam@128:     unsigned left;              /* bytes available at next */
cannam@128:     unsigned char *buf;         /* buffer */
cannam@128:     z_const unsigned char *next;    /* next byte in buffer */
cannam@128:     char *name;                 /* file name for error messages */
cannam@128: } file;
cannam@128: 
cannam@128: /* reload buffer */
cannam@128: local int readin(file *in)
cannam@128: {
cannam@128:     int len;
cannam@128: 
cannam@128:     len = read(in->fd, in->buf, 1 << in->size);
cannam@128:     if (len == -1) bye("error reading ", in->name);
cannam@128:     in->left = (unsigned)len;
cannam@128:     in->next = in->buf;
cannam@128:     return len;
cannam@128: }
cannam@128: 
cannam@128: /* read from file in, exit if end-of-file */
cannam@128: local int readmore(file *in)
cannam@128: {
cannam@128:     if (readin(in) == 0) bye("unexpected end of ", in->name);
cannam@128:     return 0;
cannam@128: }
cannam@128: 
cannam@128: #define read1(in) (in->left == 0 ? readmore(in) : 0, \
cannam@128:                    in->left--, *(in->next)++)
cannam@128: 
cannam@128: /* skip over n bytes of in */
cannam@128: local void skip(file *in, unsigned n)
cannam@128: {
cannam@128:     unsigned bypass;
cannam@128: 
cannam@128:     if (n > in->left) {
cannam@128:         n -= in->left;
cannam@128:         bypass = n & ~((1U << in->size) - 1);
cannam@128:         if (bypass) {
cannam@128:             if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1)
cannam@128:                 bye("seeking ", in->name);
cannam@128:             n -= bypass;
cannam@128:         }
cannam@128:         readmore(in);
cannam@128:         if (n > in->left)
cannam@128:             bye("unexpected end of ", in->name);
cannam@128:     }
cannam@128:     in->left -= n;
cannam@128:     in->next += n;
cannam@128: }
cannam@128: 
cannam@128: /* read a four-byte unsigned integer, little-endian, from in */
cannam@128: unsigned long read4(file *in)
cannam@128: {
cannam@128:     unsigned long val;
cannam@128: 
cannam@128:     val = read1(in);
cannam@128:     val += (unsigned)read1(in) << 8;
cannam@128:     val += (unsigned long)read1(in) << 16;
cannam@128:     val += (unsigned long)read1(in) << 24;
cannam@128:     return val;
cannam@128: }
cannam@128: 
cannam@128: /* skip over gzip header */
cannam@128: local void gzheader(file *in)
cannam@128: {
cannam@128:     int flags;
cannam@128:     unsigned n;
cannam@128: 
cannam@128:     if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file");
cannam@128:     if (read1(in) != 8) bye("unknown compression method in", in->name);
cannam@128:     flags = read1(in);
cannam@128:     if (flags & 0xe0) bye("unknown header flags set in", in->name);
cannam@128:     skip(in, 6);
cannam@128:     if (flags & 4) {
cannam@128:         n = read1(in);
cannam@128:         n += (unsigned)(read1(in)) << 8;
cannam@128:         skip(in, n);
cannam@128:     }
cannam@128:     if (flags & 8) while (read1(in) != 0) ;
cannam@128:     if (flags & 16) while (read1(in) != 0) ;
cannam@128:     if (flags & 2) skip(in, 2);
cannam@128: }
cannam@128: 
cannam@128: /* decompress gzip file "name", return strm with a deflate stream ready to
cannam@128:    continue compression of the data in the gzip file, and return a file
cannam@128:    descriptor pointing to where to write the compressed data -- the deflate
cannam@128:    stream is initialized to compress using level "level" */
cannam@128: local int gzscan(char *name, z_stream *strm, int level)
cannam@128: {
cannam@128:     int ret, lastbit, left, full;
cannam@128:     unsigned have;
cannam@128:     unsigned long crc, tot;
cannam@128:     unsigned char *window;
cannam@128:     off_t lastoff, end;
cannam@128:     file gz;
cannam@128: 
cannam@128:     /* open gzip file */
cannam@128:     gz.name = name;
cannam@128:     gz.fd = open(name, O_RDWR, 0);
cannam@128:     if (gz.fd == -1) bye("cannot open ", name);
cannam@128:     gz.buf = malloc(CHUNK);
cannam@128:     if (gz.buf == NULL) bye("out of memory", "");
cannam@128:     gz.size = LGCHUNK;
cannam@128:     gz.left = 0;
cannam@128: 
cannam@128:     /* skip gzip header */
cannam@128:     gzheader(&gz);
cannam@128: 
cannam@128:     /* prepare to decompress */
cannam@128:     window = malloc(DSIZE);
cannam@128:     if (window == NULL) bye("out of memory", "");
cannam@128:     strm->zalloc = Z_NULL;
cannam@128:     strm->zfree = Z_NULL;
cannam@128:     strm->opaque = Z_NULL;
cannam@128:     ret = inflateInit2(strm, -15);
cannam@128:     if (ret != Z_OK) bye("out of memory", " or library mismatch");
cannam@128: 
cannam@128:     /* decompress the deflate stream, saving append information */
cannam@128:     lastbit = 0;
cannam@128:     lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
cannam@128:     left = 0;
cannam@128:     strm->avail_in = gz.left;
cannam@128:     strm->next_in = gz.next;
cannam@128:     crc = crc32(0L, Z_NULL, 0);
cannam@128:     have = full = 0;
cannam@128:     do {
cannam@128:         /* if needed, get more input */
cannam@128:         if (strm->avail_in == 0) {
cannam@128:             readmore(&gz);
cannam@128:             strm->avail_in = gz.left;
cannam@128:             strm->next_in = gz.next;
cannam@128:         }
cannam@128: 
cannam@128:         /* set up output to next available section of sliding window */
cannam@128:         strm->avail_out = DSIZE - have;
cannam@128:         strm->next_out = window + have;
cannam@128: 
cannam@128:         /* inflate and check for errors */
cannam@128:         ret = inflate(strm, Z_BLOCK);
cannam@128:         if (ret == Z_STREAM_ERROR) bye("internal stream error!", "");
cannam@128:         if (ret == Z_MEM_ERROR) bye("out of memory", "");
cannam@128:         if (ret == Z_DATA_ERROR)
cannam@128:             bye("invalid compressed data--format violated in", name);
cannam@128: 
cannam@128:         /* update crc and sliding window pointer */
cannam@128:         crc = crc32(crc, window + have, DSIZE - have - strm->avail_out);
cannam@128:         if (strm->avail_out)
cannam@128:             have = DSIZE - strm->avail_out;
cannam@128:         else {
cannam@128:             have = 0;
cannam@128:             full = 1;
cannam@128:         }
cannam@128: 
cannam@128:         /* process end of block */
cannam@128:         if (strm->data_type & 128) {
cannam@128:             if (strm->data_type & 64)
cannam@128:                 left = strm->data_type & 0x1f;
cannam@128:             else {
cannam@128:                 lastbit = strm->data_type & 0x1f;
cannam@128:                 lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in;
cannam@128:             }
cannam@128:         }
cannam@128:     } while (ret != Z_STREAM_END);
cannam@128:     inflateEnd(strm);
cannam@128:     gz.left = strm->avail_in;
cannam@128:     gz.next = strm->next_in;
cannam@128: 
cannam@128:     /* save the location of the end of the compressed data */
cannam@128:     end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
cannam@128: 
cannam@128:     /* check gzip trailer and save total for deflate */
cannam@128:     if (crc != read4(&gz))
cannam@128:         bye("invalid compressed data--crc mismatch in ", name);
cannam@128:     tot = strm->total_out;
cannam@128:     if ((tot & 0xffffffffUL) != read4(&gz))
cannam@128:         bye("invalid compressed data--length mismatch in", name);
cannam@128: 
cannam@128:     /* if not at end of file, warn */
cannam@128:     if (gz.left || readin(&gz))
cannam@128:         fprintf(stderr,
cannam@128:             "gzappend warning: junk at end of gzip file overwritten\n");
cannam@128: 
cannam@128:     /* clear last block bit */
cannam@128:     lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET);
cannam@128:     if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
cannam@128:     *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7)));
cannam@128:     lseek(gz.fd, -1L, SEEK_CUR);
cannam@128:     if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name);
cannam@128: 
cannam@128:     /* if window wrapped, build dictionary from window by rotating */
cannam@128:     if (full) {
cannam@128:         rotate(window, DSIZE, have);
cannam@128:         have = DSIZE;
cannam@128:     }
cannam@128: 
cannam@128:     /* set up deflate stream with window, crc, total_in, and leftover bits */
cannam@128:     ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
cannam@128:     if (ret != Z_OK) bye("out of memory", "");
cannam@128:     deflateSetDictionary(strm, window, have);
cannam@128:     strm->adler = crc;
cannam@128:     strm->total_in = tot;
cannam@128:     if (left) {
cannam@128:         lseek(gz.fd, --end, SEEK_SET);
cannam@128:         if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
cannam@128:         deflatePrime(strm, 8 - left, *gz.buf);
cannam@128:     }
cannam@128:     lseek(gz.fd, end, SEEK_SET);
cannam@128: 
cannam@128:     /* clean up and return */
cannam@128:     free(window);
cannam@128:     free(gz.buf);
cannam@128:     return gz.fd;
cannam@128: }
cannam@128: 
cannam@128: /* append file "name" to gzip file gd using deflate stream strm -- if last
cannam@128:    is true, then finish off the deflate stream at the end */
cannam@128: local void gztack(char *name, int gd, z_stream *strm, int last)
cannam@128: {
cannam@128:     int fd, len, ret;
cannam@128:     unsigned left;
cannam@128:     unsigned char *in, *out;
cannam@128: 
cannam@128:     /* open file to compress and append */
cannam@128:     fd = 0;
cannam@128:     if (name != NULL) {
cannam@128:         fd = open(name, O_RDONLY, 0);
cannam@128:         if (fd == -1)
cannam@128:             fprintf(stderr, "gzappend warning: %s not found, skipping ...\n",
cannam@128:                     name);
cannam@128:     }
cannam@128: 
cannam@128:     /* allocate buffers */
cannam@128:     in = malloc(CHUNK);
cannam@128:     out = malloc(CHUNK);
cannam@128:     if (in == NULL || out == NULL) bye("out of memory", "");
cannam@128: 
cannam@128:     /* compress input file and append to gzip file */
cannam@128:     do {
cannam@128:         /* get more input */
cannam@128:         len = read(fd, in, CHUNK);
cannam@128:         if (len == -1) {
cannam@128:             fprintf(stderr,
cannam@128:                     "gzappend warning: error reading %s, skipping rest ...\n",
cannam@128:                     name);
cannam@128:             len = 0;
cannam@128:         }
cannam@128:         strm->avail_in = (unsigned)len;
cannam@128:         strm->next_in = in;
cannam@128:         if (len) strm->adler = crc32(strm->adler, in, (unsigned)len);
cannam@128: 
cannam@128:         /* compress and write all available output */
cannam@128:         do {
cannam@128:             strm->avail_out = CHUNK;
cannam@128:             strm->next_out = out;
cannam@128:             ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH);
cannam@128:             left = CHUNK - strm->avail_out;
cannam@128:             while (left) {
cannam@128:                 len = write(gd, out + CHUNK - strm->avail_out - left, left);
cannam@128:                 if (len == -1) bye("writing gzip file", "");
cannam@128:                 left -= (unsigned)len;
cannam@128:             }
cannam@128:         } while (strm->avail_out == 0 && ret != Z_STREAM_END);
cannam@128:     } while (len != 0);
cannam@128: 
cannam@128:     /* write trailer after last entry */
cannam@128:     if (last) {
cannam@128:         deflateEnd(strm);
cannam@128:         out[0] = (unsigned char)(strm->adler);
cannam@128:         out[1] = (unsigned char)(strm->adler >> 8);
cannam@128:         out[2] = (unsigned char)(strm->adler >> 16);
cannam@128:         out[3] = (unsigned char)(strm->adler >> 24);
cannam@128:         out[4] = (unsigned char)(strm->total_in);
cannam@128:         out[5] = (unsigned char)(strm->total_in >> 8);
cannam@128:         out[6] = (unsigned char)(strm->total_in >> 16);
cannam@128:         out[7] = (unsigned char)(strm->total_in >> 24);
cannam@128:         len = 8;
cannam@128:         do {
cannam@128:             ret = write(gd, out + 8 - len, len);
cannam@128:             if (ret == -1) bye("writing gzip file", "");
cannam@128:             len -= ret;
cannam@128:         } while (len);
cannam@128:         close(gd);
cannam@128:     }
cannam@128: 
cannam@128:     /* clean up and return */
cannam@128:     free(out);
cannam@128:     free(in);
cannam@128:     if (fd > 0) close(fd);
cannam@128: }
cannam@128: 
cannam@128: /* process the compression level option if present, scan the gzip file, and
cannam@128:    append the specified files, or append the data from stdin if no other file
cannam@128:    names are provided on the command line -- the gzip file must be writable
cannam@128:    and seekable */
cannam@128: int main(int argc, char **argv)
cannam@128: {
cannam@128:     int gd, level;
cannam@128:     z_stream strm;
cannam@128: 
cannam@128:     /* ignore command name */
cannam@128:     argc--; argv++;
cannam@128: 
cannam@128:     /* provide usage if no arguments */
cannam@128:     if (*argv == NULL) {
cannam@128:         printf(
cannam@128:             "gzappend 1.2 (11 Oct 2012) Copyright (C) 2003, 2012 Mark Adler\n"
cannam@128:                );
cannam@128:         printf(
cannam@128:             "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n");
cannam@128:         return 0;
cannam@128:     }
cannam@128: 
cannam@128:     /* set compression level */
cannam@128:     level = Z_DEFAULT_COMPRESSION;
cannam@128:     if (argv[0][0] == '-') {
cannam@128:         if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0)
cannam@128:             bye("invalid compression level", "");
cannam@128:         level = argv[0][1] - '0';
cannam@128:         if (*++argv == NULL) bye("no gzip file name after options", "");
cannam@128:     }
cannam@128: 
cannam@128:     /* prepare to append to gzip file */
cannam@128:     gd = gzscan(*argv++, &strm, level);
cannam@128: 
cannam@128:     /* append files on command line, or from stdin if none */
cannam@128:     if (*argv == NULL)
cannam@128:         gztack(NULL, gd, &strm, 1);
cannam@128:     else
cannam@128:         do {
cannam@128:             gztack(*argv, gd, &strm, argv[1] == NULL);
cannam@128:         } while (*++argv != NULL);
cannam@128:     return 0;
cannam@128: }