cannam@89: /* gzjoin -- command to join gzip files into one gzip file
cannam@89: 
cannam@89:   Copyright (C) 2004 Mark Adler, all rights reserved
cannam@89:   version 1.0, 11 Dec 2004
cannam@89: 
cannam@89:   This software is provided 'as-is', without any express or implied
cannam@89:   warranty.  In no event will the author be held liable for any damages
cannam@89:   arising from the use of this software.
cannam@89: 
cannam@89:   Permission is granted to anyone to use this software for any purpose,
cannam@89:   including commercial applications, and to alter it and redistribute it
cannam@89:   freely, subject to the following restrictions:
cannam@89: 
cannam@89:   1. The origin of this software must not be misrepresented; you must not
cannam@89:      claim that you wrote the original software. If you use this software
cannam@89:      in a product, an acknowledgment in the product documentation would be
cannam@89:      appreciated but is not required.
cannam@89:   2. Altered source versions must be plainly marked as such, and must not be
cannam@89:      misrepresented as being the original software.
cannam@89:   3. This notice may not be removed or altered from any source distribution.
cannam@89: 
cannam@89:   Mark Adler    madler@alumni.caltech.edu
cannam@89:  */
cannam@89: 
cannam@89: /*
cannam@89:  * Change history:
cannam@89:  *
cannam@89:  * 1.0  11 Dec 2004     - First version
cannam@89:  * 1.1  12 Jun 2005     - Changed ssize_t to long for portability
cannam@89:  */
cannam@89: 
cannam@89: /*
cannam@89:    gzjoin takes one or more gzip files on the command line and writes out a
cannam@89:    single gzip file that will uncompress to the concatenation of the
cannam@89:    uncompressed data from the individual gzip files.  gzjoin does this without
cannam@89:    having to recompress any of the data and without having to calculate a new
cannam@89:    crc32 for the concatenated uncompressed data.  gzjoin does however have to
cannam@89:    decompress all of the input data in order to find the bits in the compressed
cannam@89:    data that need to be modified to concatenate the streams.
cannam@89: 
cannam@89:    gzjoin does not do an integrity check on the input gzip files other than
cannam@89:    checking the gzip header and decompressing the compressed data.  They are
cannam@89:    otherwise assumed to be complete and correct.
cannam@89: 
cannam@89:    Each joint between gzip files removes at least 18 bytes of previous trailer
cannam@89:    and subsequent header, and inserts an average of about three bytes to the
cannam@89:    compressed data in order to connect the streams.  The output gzip file
cannam@89:    has a minimal ten-byte gzip header with no file name or modification time.
cannam@89: 
cannam@89:    This program was written to illustrate the use of the Z_BLOCK option of
cannam@89:    inflate() and the crc32_combine() function.  gzjoin will not compile with
cannam@89:    versions of zlib earlier than 1.2.3.
cannam@89:  */
cannam@89: 
cannam@89: #include <stdio.h>      /* fputs(), fprintf(), fwrite(), putc() */
cannam@89: #include <stdlib.h>     /* exit(), malloc(), free() */
cannam@89: #include <fcntl.h>      /* open() */
cannam@89: #include <unistd.h>     /* close(), read(), lseek() */
cannam@89: #include "zlib.h"
cannam@89:     /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
cannam@89: 
cannam@89: #define local static
cannam@89: 
cannam@89: /* exit with an error (return a value to allow use in an expression) */
cannam@89: local int bail(char *why1, char *why2)
cannam@89: {
cannam@89:     fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
cannam@89:     exit(1);
cannam@89:     return 0;
cannam@89: }
cannam@89: 
cannam@89: /* -- simple buffered file input with access to the buffer -- */
cannam@89: 
cannam@89: #define CHUNK 32768         /* must be a power of two and fit in unsigned */
cannam@89: 
cannam@89: /* bin buffered input file type */
cannam@89: typedef struct {
cannam@89:     char *name;             /* name of file for error messages */
cannam@89:     int fd;                 /* file descriptor */
cannam@89:     unsigned left;          /* bytes remaining at next */
cannam@89:     unsigned char *next;    /* next byte to read */
cannam@89:     unsigned char *buf;     /* allocated buffer of length CHUNK */
cannam@89: } bin;
cannam@89: 
cannam@89: /* close a buffered file and free allocated memory */
cannam@89: local void bclose(bin *in)
cannam@89: {
cannam@89:     if (in != NULL) {
cannam@89:         if (in->fd != -1)
cannam@89:             close(in->fd);
cannam@89:         if (in->buf != NULL)
cannam@89:             free(in->buf);
cannam@89:         free(in);
cannam@89:     }
cannam@89: }
cannam@89: 
cannam@89: /* open a buffered file for input, return a pointer to type bin, or NULL on
cannam@89:    failure */
cannam@89: local bin *bopen(char *name)
cannam@89: {
cannam@89:     bin *in;
cannam@89: 
cannam@89:     in = malloc(sizeof(bin));
cannam@89:     if (in == NULL)
cannam@89:         return NULL;
cannam@89:     in->buf = malloc(CHUNK);
cannam@89:     in->fd = open(name, O_RDONLY, 0);
cannam@89:     if (in->buf == NULL || in->fd == -1) {
cannam@89:         bclose(in);
cannam@89:         return NULL;
cannam@89:     }
cannam@89:     in->left = 0;
cannam@89:     in->next = in->buf;
cannam@89:     in->name = name;
cannam@89:     return in;
cannam@89: }
cannam@89: 
cannam@89: /* load buffer from file, return -1 on read error, 0 or 1 on success, with
cannam@89:    1 indicating that end-of-file was reached */
cannam@89: local int bload(bin *in)
cannam@89: {
cannam@89:     long len;
cannam@89: 
cannam@89:     if (in == NULL)
cannam@89:         return -1;
cannam@89:     if (in->left != 0)
cannam@89:         return 0;
cannam@89:     in->next = in->buf;
cannam@89:     do {
cannam@89:         len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
cannam@89:         if (len < 0)
cannam@89:             return -1;
cannam@89:         in->left += (unsigned)len;
cannam@89:     } while (len != 0 && in->left < CHUNK);
cannam@89:     return len == 0 ? 1 : 0;
cannam@89: }
cannam@89: 
cannam@89: /* get a byte from the file, bail if end of file */
cannam@89: #define bget(in) (in->left ? 0 : bload(in), \
cannam@89:                   in->left ? (in->left--, *(in->next)++) : \
cannam@89:                     bail("unexpected end of file on ", in->name))
cannam@89: 
cannam@89: /* get a four-byte little-endian unsigned integer from file */
cannam@89: local unsigned long bget4(bin *in)
cannam@89: {
cannam@89:     unsigned long val;
cannam@89: 
cannam@89:     val = bget(in);
cannam@89:     val += (unsigned long)(bget(in)) << 8;
cannam@89:     val += (unsigned long)(bget(in)) << 16;
cannam@89:     val += (unsigned long)(bget(in)) << 24;
cannam@89:     return val;
cannam@89: }
cannam@89: 
cannam@89: /* skip bytes in file */
cannam@89: local void bskip(bin *in, unsigned skip)
cannam@89: {
cannam@89:     /* check pointer */
cannam@89:     if (in == NULL)
cannam@89:         return;
cannam@89: 
cannam@89:     /* easy case -- skip bytes in buffer */
cannam@89:     if (skip <= in->left) {
cannam@89:         in->left -= skip;
cannam@89:         in->next += skip;
cannam@89:         return;
cannam@89:     }
cannam@89: 
cannam@89:     /* skip what's in buffer, discard buffer contents */
cannam@89:     skip -= in->left;
cannam@89:     in->left = 0;
cannam@89: 
cannam@89:     /* seek past multiples of CHUNK bytes */
cannam@89:     if (skip > CHUNK) {
cannam@89:         unsigned left;
cannam@89: 
cannam@89:         left = skip & (CHUNK - 1);
cannam@89:         if (left == 0) {
cannam@89:             /* exact number of chunks: seek all the way minus one byte to check
cannam@89:                for end-of-file with a read */
cannam@89:             lseek(in->fd, skip - 1, SEEK_CUR);
cannam@89:             if (read(in->fd, in->buf, 1) != 1)
cannam@89:                 bail("unexpected end of file on ", in->name);
cannam@89:             return;
cannam@89:         }
cannam@89: 
cannam@89:         /* skip the integral chunks, update skip with remainder */
cannam@89:         lseek(in->fd, skip - left, SEEK_CUR);
cannam@89:         skip = left;
cannam@89:     }
cannam@89: 
cannam@89:     /* read more input and skip remainder */
cannam@89:     bload(in);
cannam@89:     if (skip > in->left)
cannam@89:         bail("unexpected end of file on ", in->name);
cannam@89:     in->left -= skip;
cannam@89:     in->next += skip;
cannam@89: }
cannam@89: 
cannam@89: /* -- end of buffered input functions -- */
cannam@89: 
cannam@89: /* skip the gzip header from file in */
cannam@89: local void gzhead(bin *in)
cannam@89: {
cannam@89:     int flags;
cannam@89: 
cannam@89:     /* verify gzip magic header and compression method */
cannam@89:     if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
cannam@89:         bail(in->name, " is not a valid gzip file");
cannam@89: 
cannam@89:     /* get and verify flags */
cannam@89:     flags = bget(in);
cannam@89:     if ((flags & 0xe0) != 0)
cannam@89:         bail("unknown reserved bits set in ", in->name);
cannam@89: 
cannam@89:     /* skip modification time, extra flags, and os */
cannam@89:     bskip(in, 6);
cannam@89: 
cannam@89:     /* skip extra field if present */
cannam@89:     if (flags & 4) {
cannam@89:         unsigned len;
cannam@89: 
cannam@89:         len = bget(in);
cannam@89:         len += (unsigned)(bget(in)) << 8;
cannam@89:         bskip(in, len);
cannam@89:     }
cannam@89: 
cannam@89:     /* skip file name if present */
cannam@89:     if (flags & 8)
cannam@89:         while (bget(in) != 0)
cannam@89:             ;
cannam@89: 
cannam@89:     /* skip comment if present */
cannam@89:     if (flags & 16)
cannam@89:         while (bget(in) != 0)
cannam@89:             ;
cannam@89: 
cannam@89:     /* skip header crc if present */
cannam@89:     if (flags & 2)
cannam@89:         bskip(in, 2);
cannam@89: }
cannam@89: 
cannam@89: /* write a four-byte little-endian unsigned integer to out */
cannam@89: local void put4(unsigned long val, FILE *out)
cannam@89: {
cannam@89:     putc(val & 0xff, out);
cannam@89:     putc((val >> 8) & 0xff, out);
cannam@89:     putc((val >> 16) & 0xff, out);
cannam@89:     putc((val >> 24) & 0xff, out);
cannam@89: }
cannam@89: 
cannam@89: /* Load up zlib stream from buffered input, bail if end of file */
cannam@89: local void zpull(z_streamp strm, bin *in)
cannam@89: {
cannam@89:     if (in->left == 0)
cannam@89:         bload(in);
cannam@89:     if (in->left == 0)
cannam@89:         bail("unexpected end of file on ", in->name);
cannam@89:     strm->avail_in = in->left;
cannam@89:     strm->next_in = in->next;
cannam@89: }
cannam@89: 
cannam@89: /* Write header for gzip file to out and initialize trailer. */
cannam@89: local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
cannam@89: {
cannam@89:     fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
cannam@89:     *crc = crc32(0L, Z_NULL, 0);
cannam@89:     *tot = 0;
cannam@89: }
cannam@89: 
cannam@89: /* Copy the compressed data from name, zeroing the last block bit of the last
cannam@89:    block if clr is true, and adding empty blocks as needed to get to a byte
cannam@89:    boundary.  If clr is false, then the last block becomes the last block of
cannam@89:    the output, and the gzip trailer is written.  crc and tot maintains the
cannam@89:    crc and length (modulo 2^32) of the output for the trailer.  The resulting
cannam@89:    gzip file is written to out.  gzinit() must be called before the first call
cannam@89:    of gzcopy() to write the gzip header and to initialize crc and tot. */
cannam@89: local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
cannam@89:                   FILE *out)
cannam@89: {
cannam@89:     int ret;                /* return value from zlib functions */
cannam@89:     int pos;                /* where the "last block" bit is in byte */
cannam@89:     int last;               /* true if processing the last block */
cannam@89:     bin *in;                /* buffered input file */
cannam@89:     unsigned char *start;   /* start of compressed data in buffer */
cannam@89:     unsigned char *junk;    /* buffer for uncompressed data -- discarded */
cannam@89:     z_off_t len;            /* length of uncompressed data (support > 4 GB) */
cannam@89:     z_stream strm;          /* zlib inflate stream */
cannam@89: 
cannam@89:     /* open gzip file and skip header */
cannam@89:     in = bopen(name);
cannam@89:     if (in == NULL)
cannam@89:         bail("could not open ", name);
cannam@89:     gzhead(in);
cannam@89: 
cannam@89:     /* allocate buffer for uncompressed data and initialize raw inflate
cannam@89:        stream */
cannam@89:     junk = malloc(CHUNK);
cannam@89:     strm.zalloc = Z_NULL;
cannam@89:     strm.zfree = Z_NULL;
cannam@89:     strm.opaque = Z_NULL;
cannam@89:     strm.avail_in = 0;
cannam@89:     strm.next_in = Z_NULL;
cannam@89:     ret = inflateInit2(&strm, -15);
cannam@89:     if (junk == NULL || ret != Z_OK)
cannam@89:         bail("out of memory", "");
cannam@89: 
cannam@89:     /* inflate and copy compressed data, clear last-block bit if requested */
cannam@89:     len = 0;
cannam@89:     zpull(&strm, in);
cannam@89:     start = strm.next_in;
cannam@89:     last = start[0] & 1;
cannam@89:     if (last && clr)
cannam@89:         start[0] &= ~1;
cannam@89:     strm.avail_out = 0;
cannam@89:     for (;;) {
cannam@89:         /* if input used and output done, write used input and get more */
cannam@89:         if (strm.avail_in == 0 && strm.avail_out != 0) {
cannam@89:             fwrite(start, 1, strm.next_in - start, out);
cannam@89:             start = in->buf;
cannam@89:             in->left = 0;
cannam@89:             zpull(&strm, in);
cannam@89:         }
cannam@89: 
cannam@89:         /* decompress -- return early when end-of-block reached */
cannam@89:         strm.avail_out = CHUNK;
cannam@89:         strm.next_out = junk;
cannam@89:         ret = inflate(&strm, Z_BLOCK);
cannam@89:         switch (ret) {
cannam@89:         case Z_MEM_ERROR:
cannam@89:             bail("out of memory", "");
cannam@89:         case Z_DATA_ERROR:
cannam@89:             bail("invalid compressed data in ", in->name);
cannam@89:         }
cannam@89: 
cannam@89:         /* update length of uncompressed data */
cannam@89:         len += CHUNK - strm.avail_out;
cannam@89: 
cannam@89:         /* check for block boundary (only get this when block copied out) */
cannam@89:         if (strm.data_type & 128) {
cannam@89:             /* if that was the last block, then done */
cannam@89:             if (last)
cannam@89:                 break;
cannam@89: 
cannam@89:             /* number of unused bits in last byte */
cannam@89:             pos = strm.data_type & 7;
cannam@89: 
cannam@89:             /* find the next last-block bit */
cannam@89:             if (pos != 0) {
cannam@89:                 /* next last-block bit is in last used byte */
cannam@89:                 pos = 0x100 >> pos;
cannam@89:                 last = strm.next_in[-1] & pos;
cannam@89:                 if (last && clr)
cannam@89:                     strm.next_in[-1] &= ~pos;
cannam@89:             }
cannam@89:             else {
cannam@89:                 /* next last-block bit is in next unused byte */
cannam@89:                 if (strm.avail_in == 0) {
cannam@89:                     /* don't have that byte yet -- get it */
cannam@89:                     fwrite(start, 1, strm.next_in - start, out);
cannam@89:                     start = in->buf;
cannam@89:                     in->left = 0;
cannam@89:                     zpull(&strm, in);
cannam@89:                 }
cannam@89:                 last = strm.next_in[0] & 1;
cannam@89:                 if (last && clr)
cannam@89:                     strm.next_in[0] &= ~1;
cannam@89:             }
cannam@89:         }
cannam@89:     }
cannam@89: 
cannam@89:     /* update buffer with unused input */
cannam@89:     in->left = strm.avail_in;
cannam@89:     in->next = strm.next_in;
cannam@89: 
cannam@89:     /* copy used input, write empty blocks to get to byte boundary */
cannam@89:     pos = strm.data_type & 7;
cannam@89:     fwrite(start, 1, in->next - start - 1, out);
cannam@89:     last = in->next[-1];
cannam@89:     if (pos == 0 || !clr)
cannam@89:         /* already at byte boundary, or last file: write last byte */
cannam@89:         putc(last, out);
cannam@89:     else {
cannam@89:         /* append empty blocks to last byte */
cannam@89:         last &= ((0x100 >> pos) - 1);       /* assure unused bits are zero */
cannam@89:         if (pos & 1) {
cannam@89:             /* odd -- append an empty stored block */
cannam@89:             putc(last, out);
cannam@89:             if (pos == 1)
cannam@89:                 putc(0, out);               /* two more bits in block header */
cannam@89:             fwrite("\0\0\xff\xff", 1, 4, out);
cannam@89:         }
cannam@89:         else {
cannam@89:             /* even -- append 1, 2, or 3 empty fixed blocks */
cannam@89:             switch (pos) {
cannam@89:             case 6:
cannam@89:                 putc(last | 8, out);
cannam@89:                 last = 0;
cannam@89:             case 4:
cannam@89:                 putc(last | 0x20, out);
cannam@89:                 last = 0;
cannam@89:             case 2:
cannam@89:                 putc(last | 0x80, out);
cannam@89:                 putc(0, out);
cannam@89:             }
cannam@89:         }
cannam@89:     }
cannam@89: 
cannam@89:     /* update crc and tot */
cannam@89:     *crc = crc32_combine(*crc, bget4(in), len);
cannam@89:     *tot += (unsigned long)len;
cannam@89: 
cannam@89:     /* clean up */
cannam@89:     inflateEnd(&strm);
cannam@89:     free(junk);
cannam@89:     bclose(in);
cannam@89: 
cannam@89:     /* write trailer if this is the last gzip file */
cannam@89:     if (!clr) {
cannam@89:         put4(*crc, out);
cannam@89:         put4(*tot, out);
cannam@89:     }
cannam@89: }
cannam@89: 
cannam@89: /* join the gzip files on the command line, write result to stdout */
cannam@89: int main(int argc, char **argv)
cannam@89: {
cannam@89:     unsigned long crc, tot;     /* running crc and total uncompressed length */
cannam@89: 
cannam@89:     /* skip command name */
cannam@89:     argc--;
cannam@89:     argv++;
cannam@89: 
cannam@89:     /* show usage if no arguments */
cannam@89:     if (argc == 0) {
cannam@89:         fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
cannam@89:               stderr);
cannam@89:         return 0;
cannam@89:     }
cannam@89: 
cannam@89:     /* join gzip files on command line and write to stdout */
cannam@89:     gzinit(&crc, &tot, stdout);
cannam@89:     while (argc--)
cannam@89:         gzcopy(*argv++, argc, &crc, &tot, stdout);
cannam@89: 
cannam@89:     /* done */
cannam@89:     return 0;
cannam@89: }