cannam@89: /* gzappend -- command to append to a gzip file
cannam@89: 
cannam@89:   Copyright (C) 2003 Mark Adler, all rights reserved
cannam@89:   version 1.1, 4 Nov 2003
cannam@89: 
cannam@89:   This software is provided 'as-is', without any express or implied
cannam@89:   warranty.  In no event will the author be held liable for any damages
cannam@89:   arising from the use of this software.
cannam@89: 
cannam@89:   Permission is granted to anyone to use this software for any purpose,
cannam@89:   including commercial applications, and to alter it and redistribute it
cannam@89:   freely, subject to the following restrictions:
cannam@89: 
cannam@89:   1. The origin of this software must not be misrepresented; you must not
cannam@89:      claim that you wrote the original software. If you use this software
cannam@89:      in a product, an acknowledgment in the product documentation would be
cannam@89:      appreciated but is not required.
cannam@89:   2. Altered source versions must be plainly marked as such, and must not be
cannam@89:      misrepresented as being the original software.
cannam@89:   3. This notice may not be removed or altered from any source distribution.
cannam@89: 
cannam@89:   Mark Adler    madler@alumni.caltech.edu
cannam@89:  */
cannam@89: 
cannam@89: /*
cannam@89:  * Change history:
cannam@89:  *
cannam@89:  * 1.0  19 Oct 2003     - First version
cannam@89:  * 1.1   4 Nov 2003     - Expand and clarify some comments and notes
cannam@89:  *                      - Add version and copyright to help
cannam@89:  *                      - Send help to stdout instead of stderr
cannam@89:  *                      - Add some preemptive typecasts
cannam@89:  *                      - Add L to constants in lseek() calls
cannam@89:  *                      - Remove some debugging information in error messages
cannam@89:  *                      - Use new data_type definition for zlib 1.2.1
cannam@89:  *                      - Simplfy and unify file operations
cannam@89:  *                      - Finish off gzip file in gztack()
cannam@89:  *                      - Use deflatePrime() instead of adding empty blocks
cannam@89:  *                      - Keep gzip file clean on appended file read errors
cannam@89:  *                      - Use in-place rotate instead of auxiliary buffer
cannam@89:  *                        (Why you ask?  Because it was fun to write!)
cannam@89:  */
cannam@89: 
cannam@89: /*
cannam@89:    gzappend takes a gzip file and appends to it, compressing files from the
cannam@89:    command line or data from stdin.  The gzip file is written to directly, to
cannam@89:    avoid copying that file, in case it's large.  Note that this results in the
cannam@89:    unfriendly behavior that if gzappend fails, the gzip file is corrupted.
cannam@89: 
cannam@89:    This program was written to illustrate the use of the new Z_BLOCK option of
cannam@89:    zlib 1.2.x's inflate() function.  This option returns from inflate() at each
cannam@89:    block boundary to facilitate locating and modifying the last block bit at
cannam@89:    the start of the final deflate block.  Also whether using Z_BLOCK or not,
cannam@89:    another required feature of zlib 1.2.x is that inflate() now provides the
cannam@89:    number of unusued bits in the last input byte used.  gzappend will not work
cannam@89:    with versions of zlib earlier than 1.2.1.
cannam@89: 
cannam@89:    gzappend first decompresses the gzip file internally, discarding all but
cannam@89:    the last 32K of uncompressed data, and noting the location of the last block
cannam@89:    bit and the number of unused bits in the last byte of the compressed data.
cannam@89:    The gzip trailer containing the CRC-32 and length of the uncompressed data
cannam@89:    is verified.  This trailer will be later overwritten.
cannam@89: 
cannam@89:    Then the last block bit is cleared by seeking back in the file and rewriting
cannam@89:    the byte that contains it.  Seeking forward, the last byte of the compressed
cannam@89:    data is saved along with the number of unused bits to initialize deflate.
cannam@89: 
cannam@89:    A deflate process is initialized, using the last 32K of the uncompressed
cannam@89:    data from the gzip file to initialize the dictionary.  If the total
cannam@89:    uncompressed data was less than 32K, then all of it is used to initialize
cannam@89:    the dictionary.  The deflate output bit buffer is also initialized with the
cannam@89:    last bits from the original deflate stream.  From here on, the data to
cannam@89:    append is simply compressed using deflate, and written to the gzip file.
cannam@89:    When that is complete, the new CRC-32 and uncompressed length are written
cannam@89:    as the trailer of the gzip file.
cannam@89:  */
cannam@89: 
cannam@89: #include <stdio.h>
cannam@89: #include <stdlib.h>
cannam@89: #include <string.h>
cannam@89: #include <fcntl.h>
cannam@89: #include <unistd.h>
cannam@89: #include "zlib.h"
cannam@89: 
cannam@89: #define local static
cannam@89: #define LGCHUNK 14
cannam@89: #define CHUNK (1U << LGCHUNK)
cannam@89: #define DSIZE 32768U
cannam@89: 
cannam@89: /* print an error message and terminate with extreme prejudice */
cannam@89: local void bye(char *msg1, char *msg2)
cannam@89: {
cannam@89:     fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2);
cannam@89:     exit(1);
cannam@89: }
cannam@89: 
cannam@89: /* return the greatest common divisor of a and b using Euclid's algorithm,
cannam@89:    modified to be fast when one argument much greater than the other, and
cannam@89:    coded to avoid unnecessary swapping */
cannam@89: local unsigned gcd(unsigned a, unsigned b)
cannam@89: {
cannam@89:     unsigned c;
cannam@89: 
cannam@89:     while (a && b)
cannam@89:         if (a > b) {
cannam@89:             c = b;
cannam@89:             while (a - c >= c)
cannam@89:                 c <<= 1;
cannam@89:             a -= c;
cannam@89:         }
cannam@89:         else {
cannam@89:             c = a;
cannam@89:             while (b - c >= c)
cannam@89:                 c <<= 1;
cannam@89:             b -= c;
cannam@89:         }
cannam@89:     return a + b;
cannam@89: }
cannam@89: 
cannam@89: /* rotate list[0..len-1] left by rot positions, in place */
cannam@89: local void rotate(unsigned char *list, unsigned len, unsigned rot)
cannam@89: {
cannam@89:     unsigned char tmp;
cannam@89:     unsigned cycles;
cannam@89:     unsigned char *start, *last, *to, *from;
cannam@89: 
cannam@89:     /* normalize rot and handle degenerate cases */
cannam@89:     if (len < 2) return;
cannam@89:     if (rot >= len) rot %= len;
cannam@89:     if (rot == 0) return;
cannam@89: 
cannam@89:     /* pointer to last entry in list */
cannam@89:     last = list + (len - 1);
cannam@89: 
cannam@89:     /* do simple left shift by one */
cannam@89:     if (rot == 1) {
cannam@89:         tmp = *list;
cannam@89:         memcpy(list, list + 1, len - 1);
cannam@89:         *last = tmp;
cannam@89:         return;
cannam@89:     }
cannam@89: 
cannam@89:     /* do simple right shift by one */
cannam@89:     if (rot == len - 1) {
cannam@89:         tmp = *last;
cannam@89:         memmove(list + 1, list, len - 1);
cannam@89:         *list = tmp;
cannam@89:         return;
cannam@89:     }
cannam@89: 
cannam@89:     /* otherwise do rotate as a set of cycles in place */
cannam@89:     cycles = gcd(len, rot);             /* number of cycles */
cannam@89:     do {
cannam@89:         start = from = list + cycles;   /* start index is arbitrary */
cannam@89:         tmp = *from;                    /* save entry to be overwritten */
cannam@89:         for (;;) {
cannam@89:             to = from;                  /* next step in cycle */
cannam@89:             from += rot;                /* go right rot positions */
cannam@89:             if (from > last) from -= len;   /* (pointer better not wrap) */
cannam@89:             if (from == start) break;   /* all but one shifted */
cannam@89:             *to = *from;                /* shift left */
cannam@89:         }
cannam@89:         *to = tmp;                      /* complete the circle */
cannam@89:     } while (--cycles);
cannam@89: }
cannam@89: 
cannam@89: /* structure for gzip file read operations */
cannam@89: typedef struct {
cannam@89:     int fd;                     /* file descriptor */
cannam@89:     int size;                   /* 1 << size is bytes in buf */
cannam@89:     unsigned left;              /* bytes available at next */
cannam@89:     unsigned char *buf;         /* buffer */
cannam@89:     unsigned char *next;        /* next byte in buffer */
cannam@89:     char *name;                 /* file name for error messages */
cannam@89: } file;
cannam@89: 
cannam@89: /* reload buffer */
cannam@89: local int readin(file *in)
cannam@89: {
cannam@89:     int len;
cannam@89: 
cannam@89:     len = read(in->fd, in->buf, 1 << in->size);
cannam@89:     if (len == -1) bye("error reading ", in->name);
cannam@89:     in->left = (unsigned)len;
cannam@89:     in->next = in->buf;
cannam@89:     return len;
cannam@89: }
cannam@89: 
cannam@89: /* read from file in, exit if end-of-file */
cannam@89: local int readmore(file *in)
cannam@89: {
cannam@89:     if (readin(in) == 0) bye("unexpected end of ", in->name);
cannam@89:     return 0;
cannam@89: }
cannam@89: 
cannam@89: #define read1(in) (in->left == 0 ? readmore(in) : 0, \
cannam@89:                    in->left--, *(in->next)++)
cannam@89: 
cannam@89: /* skip over n bytes of in */
cannam@89: local void skip(file *in, unsigned n)
cannam@89: {
cannam@89:     unsigned bypass;
cannam@89: 
cannam@89:     if (n > in->left) {
cannam@89:         n -= in->left;
cannam@89:         bypass = n & ~((1U << in->size) - 1);
cannam@89:         if (bypass) {
cannam@89:             if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1)
cannam@89:                 bye("seeking ", in->name);
cannam@89:             n -= bypass;
cannam@89:         }
cannam@89:         readmore(in);
cannam@89:         if (n > in->left)
cannam@89:             bye("unexpected end of ", in->name);
cannam@89:     }
cannam@89:     in->left -= n;
cannam@89:     in->next += n;
cannam@89: }
cannam@89: 
cannam@89: /* read a four-byte unsigned integer, little-endian, from in */
cannam@89: unsigned long read4(file *in)
cannam@89: {
cannam@89:     unsigned long val;
cannam@89: 
cannam@89:     val = read1(in);
cannam@89:     val += (unsigned)read1(in) << 8;
cannam@89:     val += (unsigned long)read1(in) << 16;
cannam@89:     val += (unsigned long)read1(in) << 24;
cannam@89:     return val;
cannam@89: }
cannam@89: 
cannam@89: /* skip over gzip header */
cannam@89: local void gzheader(file *in)
cannam@89: {
cannam@89:     int flags;
cannam@89:     unsigned n;
cannam@89: 
cannam@89:     if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file");
cannam@89:     if (read1(in) != 8) bye("unknown compression method in", in->name);
cannam@89:     flags = read1(in);
cannam@89:     if (flags & 0xe0) bye("unknown header flags set in", in->name);
cannam@89:     skip(in, 6);
cannam@89:     if (flags & 4) {
cannam@89:         n = read1(in);
cannam@89:         n += (unsigned)(read1(in)) << 8;
cannam@89:         skip(in, n);
cannam@89:     }
cannam@89:     if (flags & 8) while (read1(in) != 0) ;
cannam@89:     if (flags & 16) while (read1(in) != 0) ;
cannam@89:     if (flags & 2) skip(in, 2);
cannam@89: }
cannam@89: 
cannam@89: /* decompress gzip file "name", return strm with a deflate stream ready to
cannam@89:    continue compression of the data in the gzip file, and return a file
cannam@89:    descriptor pointing to where to write the compressed data -- the deflate
cannam@89:    stream is initialized to compress using level "level" */
cannam@89: local int gzscan(char *name, z_stream *strm, int level)
cannam@89: {
cannam@89:     int ret, lastbit, left, full;
cannam@89:     unsigned have;
cannam@89:     unsigned long crc, tot;
cannam@89:     unsigned char *window;
cannam@89:     off_t lastoff, end;
cannam@89:     file gz;
cannam@89: 
cannam@89:     /* open gzip file */
cannam@89:     gz.name = name;
cannam@89:     gz.fd = open(name, O_RDWR, 0);
cannam@89:     if (gz.fd == -1) bye("cannot open ", name);
cannam@89:     gz.buf = malloc(CHUNK);
cannam@89:     if (gz.buf == NULL) bye("out of memory", "");
cannam@89:     gz.size = LGCHUNK;
cannam@89:     gz.left = 0;
cannam@89: 
cannam@89:     /* skip gzip header */
cannam@89:     gzheader(&gz);
cannam@89: 
cannam@89:     /* prepare to decompress */
cannam@89:     window = malloc(DSIZE);
cannam@89:     if (window == NULL) bye("out of memory", "");
cannam@89:     strm->zalloc = Z_NULL;
cannam@89:     strm->zfree = Z_NULL;
cannam@89:     strm->opaque = Z_NULL;
cannam@89:     ret = inflateInit2(strm, -15);
cannam@89:     if (ret != Z_OK) bye("out of memory", " or library mismatch");
cannam@89: 
cannam@89:     /* decompress the deflate stream, saving append information */
cannam@89:     lastbit = 0;
cannam@89:     lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
cannam@89:     left = 0;
cannam@89:     strm->avail_in = gz.left;
cannam@89:     strm->next_in = gz.next;
cannam@89:     crc = crc32(0L, Z_NULL, 0);
cannam@89:     have = full = 0;
cannam@89:     do {
cannam@89:         /* if needed, get more input */
cannam@89:         if (strm->avail_in == 0) {
cannam@89:             readmore(&gz);
cannam@89:             strm->avail_in = gz.left;
cannam@89:             strm->next_in = gz.next;
cannam@89:         }
cannam@89: 
cannam@89:         /* set up output to next available section of sliding window */
cannam@89:         strm->avail_out = DSIZE - have;
cannam@89:         strm->next_out = window + have;
cannam@89: 
cannam@89:         /* inflate and check for errors */
cannam@89:         ret = inflate(strm, Z_BLOCK);
cannam@89:         if (ret == Z_STREAM_ERROR) bye("internal stream error!", "");
cannam@89:         if (ret == Z_MEM_ERROR) bye("out of memory", "");
cannam@89:         if (ret == Z_DATA_ERROR)
cannam@89:             bye("invalid compressed data--format violated in", name);
cannam@89: 
cannam@89:         /* update crc and sliding window pointer */
cannam@89:         crc = crc32(crc, window + have, DSIZE - have - strm->avail_out);
cannam@89:         if (strm->avail_out)
cannam@89:             have = DSIZE - strm->avail_out;
cannam@89:         else {
cannam@89:             have = 0;
cannam@89:             full = 1;
cannam@89:         }
cannam@89: 
cannam@89:         /* process end of block */
cannam@89:         if (strm->data_type & 128) {
cannam@89:             if (strm->data_type & 64)
cannam@89:                 left = strm->data_type & 0x1f;
cannam@89:             else {
cannam@89:                 lastbit = strm->data_type & 0x1f;
cannam@89:                 lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in;
cannam@89:             }
cannam@89:         }
cannam@89:     } while (ret != Z_STREAM_END);
cannam@89:     inflateEnd(strm);
cannam@89:     gz.left = strm->avail_in;
cannam@89:     gz.next = strm->next_in;
cannam@89: 
cannam@89:     /* save the location of the end of the compressed data */
cannam@89:     end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;
cannam@89: 
cannam@89:     /* check gzip trailer and save total for deflate */
cannam@89:     if (crc != read4(&gz))
cannam@89:         bye("invalid compressed data--crc mismatch in ", name);
cannam@89:     tot = strm->total_out;
cannam@89:     if ((tot & 0xffffffffUL) != read4(&gz))
cannam@89:         bye("invalid compressed data--length mismatch in", name);
cannam@89: 
cannam@89:     /* if not at end of file, warn */
cannam@89:     if (gz.left || readin(&gz))
cannam@89:         fprintf(stderr,
cannam@89:             "gzappend warning: junk at end of gzip file overwritten\n");
cannam@89: 
cannam@89:     /* clear last block bit */
cannam@89:     lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET);
cannam@89:     if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
cannam@89:     *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7)));
cannam@89:     lseek(gz.fd, -1L, SEEK_CUR);
cannam@89:     if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name);
cannam@89: 
cannam@89:     /* if window wrapped, build dictionary from window by rotating */
cannam@89:     if (full) {
cannam@89:         rotate(window, DSIZE, have);
cannam@89:         have = DSIZE;
cannam@89:     }
cannam@89: 
cannam@89:     /* set up deflate stream with window, crc, total_in, and leftover bits */
cannam@89:     ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
cannam@89:     if (ret != Z_OK) bye("out of memory", "");
cannam@89:     deflateSetDictionary(strm, window, have);
cannam@89:     strm->adler = crc;
cannam@89:     strm->total_in = tot;
cannam@89:     if (left) {
cannam@89:         lseek(gz.fd, --end, SEEK_SET);
cannam@89:         if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);
cannam@89:         deflatePrime(strm, 8 - left, *gz.buf);
cannam@89:     }
cannam@89:     lseek(gz.fd, end, SEEK_SET);
cannam@89: 
cannam@89:     /* clean up and return */
cannam@89:     free(window);
cannam@89:     free(gz.buf);
cannam@89:     return gz.fd;
cannam@89: }
cannam@89: 
cannam@89: /* append file "name" to gzip file gd using deflate stream strm -- if last
cannam@89:    is true, then finish off the deflate stream at the end */
cannam@89: local void gztack(char *name, int gd, z_stream *strm, int last)
cannam@89: {
cannam@89:     int fd, len, ret;
cannam@89:     unsigned left;
cannam@89:     unsigned char *in, *out;
cannam@89: 
cannam@89:     /* open file to compress and append */
cannam@89:     fd = 0;
cannam@89:     if (name != NULL) {
cannam@89:         fd = open(name, O_RDONLY, 0);
cannam@89:         if (fd == -1)
cannam@89:             fprintf(stderr, "gzappend warning: %s not found, skipping ...\n",
cannam@89:                     name);
cannam@89:     }
cannam@89: 
cannam@89:     /* allocate buffers */
cannam@89:     in = fd == -1 ? NULL : malloc(CHUNK);
cannam@89:     out = malloc(CHUNK);
cannam@89:     if (out == NULL) bye("out of memory", "");
cannam@89: 
cannam@89:     /* compress input file and append to gzip file */
cannam@89:     do {
cannam@89:         /* get more input */
cannam@89:         len = fd == -1 ? 0 : read(fd, in, CHUNK);
cannam@89:         if (len == -1) {
cannam@89:             fprintf(stderr,
cannam@89:                     "gzappend warning: error reading %s, skipping rest ...\n",
cannam@89:                     name);
cannam@89:             len = 0;
cannam@89:         }
cannam@89:         strm->avail_in = (unsigned)len;
cannam@89:         strm->next_in = in;
cannam@89:         if (len) strm->adler = crc32(strm->adler, in, (unsigned)len);
cannam@89: 
cannam@89:         /* compress and write all available output */
cannam@89:         do {
cannam@89:             strm->avail_out = CHUNK;
cannam@89:             strm->next_out = out;
cannam@89:             ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH);
cannam@89:             left = CHUNK - strm->avail_out;
cannam@89:             while (left) {
cannam@89:                 len = write(gd, out + CHUNK - strm->avail_out - left, left);
cannam@89:                 if (len == -1) bye("writing gzip file", "");
cannam@89:                 left -= (unsigned)len;
cannam@89:             }
cannam@89:         } while (strm->avail_out == 0 && ret != Z_STREAM_END);
cannam@89:     } while (len != 0);
cannam@89: 
cannam@89:     /* write trailer after last entry */
cannam@89:     if (last) {
cannam@89:         deflateEnd(strm);
cannam@89:         out[0] = (unsigned char)(strm->adler);
cannam@89:         out[1] = (unsigned char)(strm->adler >> 8);
cannam@89:         out[2] = (unsigned char)(strm->adler >> 16);
cannam@89:         out[3] = (unsigned char)(strm->adler >> 24);
cannam@89:         out[4] = (unsigned char)(strm->total_in);
cannam@89:         out[5] = (unsigned char)(strm->total_in >> 8);
cannam@89:         out[6] = (unsigned char)(strm->total_in >> 16);
cannam@89:         out[7] = (unsigned char)(strm->total_in >> 24);
cannam@89:         len = 8;
cannam@89:         do {
cannam@89:             ret = write(gd, out + 8 - len, len);
cannam@89:             if (ret == -1) bye("writing gzip file", "");
cannam@89:             len -= ret;
cannam@89:         } while (len);
cannam@89:         close(gd);
cannam@89:     }
cannam@89: 
cannam@89:     /* clean up and return */
cannam@89:     free(out);
cannam@89:     if (in != NULL) free(in);
cannam@89:     if (fd > 0) close(fd);
cannam@89: }
cannam@89: 
cannam@89: /* process the compression level option if present, scan the gzip file, and
cannam@89:    append the specified files, or append the data from stdin if no other file
cannam@89:    names are provided on the command line -- the gzip file must be writable
cannam@89:    and seekable */
cannam@89: int main(int argc, char **argv)
cannam@89: {
cannam@89:     int gd, level;
cannam@89:     z_stream strm;
cannam@89: 
cannam@89:     /* ignore command name */
cannam@89:     argv++;
cannam@89: 
cannam@89:     /* provide usage if no arguments */
cannam@89:     if (*argv == NULL) {
cannam@89:         printf("gzappend 1.1 (4 Nov 2003) Copyright (C) 2003 Mark Adler\n");
cannam@89:         printf(
cannam@89:             "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n");
cannam@89:         return 0;
cannam@89:     }
cannam@89: 
cannam@89:     /* set compression level */
cannam@89:     level = Z_DEFAULT_COMPRESSION;
cannam@89:     if (argv[0][0] == '-') {
cannam@89:         if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0)
cannam@89:             bye("invalid compression level", "");
cannam@89:         level = argv[0][1] - '0';
cannam@89:         if (*++argv == NULL) bye("no gzip file name after options", "");
cannam@89:     }
cannam@89: 
cannam@89:     /* prepare to append to gzip file */
cannam@89:     gd = gzscan(*argv++, &strm, level);
cannam@89: 
cannam@89:     /* append files on command line, or from stdin if none */
cannam@89:     if (*argv == NULL)
cannam@89:         gztack(NULL, gd, &strm, 1);
cannam@89:     else
cannam@89:         do {
cannam@89:             gztack(*argv, gd, &strm, argv[1] == NULL);
cannam@89:         } while (*++argv != NULL);
cannam@89:     return 0;
cannam@89: }