Chris@43: /* gzjoin -- command to join gzip files into one gzip file
Chris@43: 
Chris@43:   Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved
Chris@43:   version 1.2, 14 Aug 2012
Chris@43: 
Chris@43:   This software is provided 'as-is', without any express or implied
Chris@43:   warranty.  In no event will the author be held liable for any damages
Chris@43:   arising from the use of this software.
Chris@43: 
Chris@43:   Permission is granted to anyone to use this software for any purpose,
Chris@43:   including commercial applications, and to alter it and redistribute it
Chris@43:   freely, subject to the following restrictions:
Chris@43: 
Chris@43:   1. The origin of this software must not be misrepresented; you must not
Chris@43:      claim that you wrote the original software. If you use this software
Chris@43:      in a product, an acknowledgment in the product documentation would be
Chris@43:      appreciated but is not required.
Chris@43:   2. Altered source versions must be plainly marked as such, and must not be
Chris@43:      misrepresented as being the original software.
Chris@43:   3. This notice may not be removed or altered from any source distribution.
Chris@43: 
Chris@43:   Mark Adler    madler@alumni.caltech.edu
Chris@43:  */
Chris@43: 
Chris@43: /*
Chris@43:  * Change history:
Chris@43:  *
Chris@43:  * 1.0  11 Dec 2004     - First version
Chris@43:  * 1.1  12 Jun 2005     - Changed ssize_t to long for portability
Chris@43:  * 1.2  14 Aug 2012     - Clean up for z_const usage
Chris@43:  */
Chris@43: 
Chris@43: /*
Chris@43:    gzjoin takes one or more gzip files on the command line and writes out a
Chris@43:    single gzip file that will uncompress to the concatenation of the
Chris@43:    uncompressed data from the individual gzip files.  gzjoin does this without
Chris@43:    having to recompress any of the data and without having to calculate a new
Chris@43:    crc32 for the concatenated uncompressed data.  gzjoin does however have to
Chris@43:    decompress all of the input data in order to find the bits in the compressed
Chris@43:    data that need to be modified to concatenate the streams.
Chris@43: 
Chris@43:    gzjoin does not do an integrity check on the input gzip files other than
Chris@43:    checking the gzip header and decompressing the compressed data.  They are
Chris@43:    otherwise assumed to be complete and correct.
Chris@43: 
Chris@43:    Each joint between gzip files removes at least 18 bytes of previous trailer
Chris@43:    and subsequent header, and inserts an average of about three bytes to the
Chris@43:    compressed data in order to connect the streams.  The output gzip file
Chris@43:    has a minimal ten-byte gzip header with no file name or modification time.
Chris@43: 
Chris@43:    This program was written to illustrate the use of the Z_BLOCK option of
Chris@43:    inflate() and the crc32_combine() function.  gzjoin will not compile with
Chris@43:    versions of zlib earlier than 1.2.3.
Chris@43:  */
Chris@43: 
Chris@43: #include <stdio.h>      /* fputs(), fprintf(), fwrite(), putc() */
Chris@43: #include <stdlib.h>     /* exit(), malloc(), free() */
Chris@43: #include <fcntl.h>      /* open() */
Chris@43: #include <unistd.h>     /* close(), read(), lseek() */
Chris@43: #include "zlib.h"
Chris@43:     /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
Chris@43: 
Chris@43: #define local static
Chris@43: 
Chris@43: /* exit with an error (return a value to allow use in an expression) */
Chris@43: local int bail(char *why1, char *why2)
Chris@43: {
Chris@43:     fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
Chris@43:     exit(1);
Chris@43:     return 0;
Chris@43: }
Chris@43: 
Chris@43: /* -- simple buffered file input with access to the buffer -- */
Chris@43: 
Chris@43: #define CHUNK 32768         /* must be a power of two and fit in unsigned */
Chris@43: 
Chris@43: /* bin buffered input file type */
Chris@43: typedef struct {
Chris@43:     char *name;             /* name of file for error messages */
Chris@43:     int fd;                 /* file descriptor */
Chris@43:     unsigned left;          /* bytes remaining at next */
Chris@43:     unsigned char *next;    /* next byte to read */
Chris@43:     unsigned char *buf;     /* allocated buffer of length CHUNK */
Chris@43: } bin;
Chris@43: 
Chris@43: /* close a buffered file and free allocated memory */
Chris@43: local void bclose(bin *in)
Chris@43: {
Chris@43:     if (in != NULL) {
Chris@43:         if (in->fd != -1)
Chris@43:             close(in->fd);
Chris@43:         if (in->buf != NULL)
Chris@43:             free(in->buf);
Chris@43:         free(in);
Chris@43:     }
Chris@43: }
Chris@43: 
Chris@43: /* open a buffered file for input, return a pointer to type bin, or NULL on
Chris@43:    failure */
Chris@43: local bin *bopen(char *name)
Chris@43: {
Chris@43:     bin *in;
Chris@43: 
Chris@43:     in = malloc(sizeof(bin));
Chris@43:     if (in == NULL)
Chris@43:         return NULL;
Chris@43:     in->buf = malloc(CHUNK);
Chris@43:     in->fd = open(name, O_RDONLY, 0);
Chris@43:     if (in->buf == NULL || in->fd == -1) {
Chris@43:         bclose(in);
Chris@43:         return NULL;
Chris@43:     }
Chris@43:     in->left = 0;
Chris@43:     in->next = in->buf;
Chris@43:     in->name = name;
Chris@43:     return in;
Chris@43: }
Chris@43: 
Chris@43: /* load buffer from file, return -1 on read error, 0 or 1 on success, with
Chris@43:    1 indicating that end-of-file was reached */
Chris@43: local int bload(bin *in)
Chris@43: {
Chris@43:     long len;
Chris@43: 
Chris@43:     if (in == NULL)
Chris@43:         return -1;
Chris@43:     if (in->left != 0)
Chris@43:         return 0;
Chris@43:     in->next = in->buf;
Chris@43:     do {
Chris@43:         len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
Chris@43:         if (len < 0)
Chris@43:             return -1;
Chris@43:         in->left += (unsigned)len;
Chris@43:     } while (len != 0 && in->left < CHUNK);
Chris@43:     return len == 0 ? 1 : 0;
Chris@43: }
Chris@43: 
Chris@43: /* get a byte from the file, bail if end of file */
Chris@43: #define bget(in) (in->left ? 0 : bload(in), \
Chris@43:                   in->left ? (in->left--, *(in->next)++) : \
Chris@43:                     bail("unexpected end of file on ", in->name))
Chris@43: 
Chris@43: /* get a four-byte little-endian unsigned integer from file */
Chris@43: local unsigned long bget4(bin *in)
Chris@43: {
Chris@43:     unsigned long val;
Chris@43: 
Chris@43:     val = bget(in);
Chris@43:     val += (unsigned long)(bget(in)) << 8;
Chris@43:     val += (unsigned long)(bget(in)) << 16;
Chris@43:     val += (unsigned long)(bget(in)) << 24;
Chris@43:     return val;
Chris@43: }
Chris@43: 
Chris@43: /* skip bytes in file */
Chris@43: local void bskip(bin *in, unsigned skip)
Chris@43: {
Chris@43:     /* check pointer */
Chris@43:     if (in == NULL)
Chris@43:         return;
Chris@43: 
Chris@43:     /* easy case -- skip bytes in buffer */
Chris@43:     if (skip <= in->left) {
Chris@43:         in->left -= skip;
Chris@43:         in->next += skip;
Chris@43:         return;
Chris@43:     }
Chris@43: 
Chris@43:     /* skip what's in buffer, discard buffer contents */
Chris@43:     skip -= in->left;
Chris@43:     in->left = 0;
Chris@43: 
Chris@43:     /* seek past multiples of CHUNK bytes */
Chris@43:     if (skip > CHUNK) {
Chris@43:         unsigned left;
Chris@43: 
Chris@43:         left = skip & (CHUNK - 1);
Chris@43:         if (left == 0) {
Chris@43:             /* exact number of chunks: seek all the way minus one byte to check
Chris@43:                for end-of-file with a read */
Chris@43:             lseek(in->fd, skip - 1, SEEK_CUR);
Chris@43:             if (read(in->fd, in->buf, 1) != 1)
Chris@43:                 bail("unexpected end of file on ", in->name);
Chris@43:             return;
Chris@43:         }
Chris@43: 
Chris@43:         /* skip the integral chunks, update skip with remainder */
Chris@43:         lseek(in->fd, skip - left, SEEK_CUR);
Chris@43:         skip = left;
Chris@43:     }
Chris@43: 
Chris@43:     /* read more input and skip remainder */
Chris@43:     bload(in);
Chris@43:     if (skip > in->left)
Chris@43:         bail("unexpected end of file on ", in->name);
Chris@43:     in->left -= skip;
Chris@43:     in->next += skip;
Chris@43: }
Chris@43: 
Chris@43: /* -- end of buffered input functions -- */
Chris@43: 
Chris@43: /* skip the gzip header from file in */
Chris@43: local void gzhead(bin *in)
Chris@43: {
Chris@43:     int flags;
Chris@43: 
Chris@43:     /* verify gzip magic header and compression method */
Chris@43:     if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
Chris@43:         bail(in->name, " is not a valid gzip file");
Chris@43: 
Chris@43:     /* get and verify flags */
Chris@43:     flags = bget(in);
Chris@43:     if ((flags & 0xe0) != 0)
Chris@43:         bail("unknown reserved bits set in ", in->name);
Chris@43: 
Chris@43:     /* skip modification time, extra flags, and os */
Chris@43:     bskip(in, 6);
Chris@43: 
Chris@43:     /* skip extra field if present */
Chris@43:     if (flags & 4) {
Chris@43:         unsigned len;
Chris@43: 
Chris@43:         len = bget(in);
Chris@43:         len += (unsigned)(bget(in)) << 8;
Chris@43:         bskip(in, len);
Chris@43:     }
Chris@43: 
Chris@43:     /* skip file name if present */
Chris@43:     if (flags & 8)
Chris@43:         while (bget(in) != 0)
Chris@43:             ;
Chris@43: 
Chris@43:     /* skip comment if present */
Chris@43:     if (flags & 16)
Chris@43:         while (bget(in) != 0)
Chris@43:             ;
Chris@43: 
Chris@43:     /* skip header crc if present */
Chris@43:     if (flags & 2)
Chris@43:         bskip(in, 2);
Chris@43: }
Chris@43: 
Chris@43: /* write a four-byte little-endian unsigned integer to out */
Chris@43: local void put4(unsigned long val, FILE *out)
Chris@43: {
Chris@43:     putc(val & 0xff, out);
Chris@43:     putc((val >> 8) & 0xff, out);
Chris@43:     putc((val >> 16) & 0xff, out);
Chris@43:     putc((val >> 24) & 0xff, out);
Chris@43: }
Chris@43: 
Chris@43: /* Load up zlib stream from buffered input, bail if end of file */
Chris@43: local void zpull(z_streamp strm, bin *in)
Chris@43: {
Chris@43:     if (in->left == 0)
Chris@43:         bload(in);
Chris@43:     if (in->left == 0)
Chris@43:         bail("unexpected end of file on ", in->name);
Chris@43:     strm->avail_in = in->left;
Chris@43:     strm->next_in = in->next;
Chris@43: }
Chris@43: 
Chris@43: /* Write header for gzip file to out and initialize trailer. */
Chris@43: local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
Chris@43: {
Chris@43:     fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
Chris@43:     *crc = crc32(0L, Z_NULL, 0);
Chris@43:     *tot = 0;
Chris@43: }
Chris@43: 
Chris@43: /* Copy the compressed data from name, zeroing the last block bit of the last
Chris@43:    block if clr is true, and adding empty blocks as needed to get to a byte
Chris@43:    boundary.  If clr is false, then the last block becomes the last block of
Chris@43:    the output, and the gzip trailer is written.  crc and tot maintains the
Chris@43:    crc and length (modulo 2^32) of the output for the trailer.  The resulting
Chris@43:    gzip file is written to out.  gzinit() must be called before the first call
Chris@43:    of gzcopy() to write the gzip header and to initialize crc and tot. */
Chris@43: local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
Chris@43:                   FILE *out)
Chris@43: {
Chris@43:     int ret;                /* return value from zlib functions */
Chris@43:     int pos;                /* where the "last block" bit is in byte */
Chris@43:     int last;               /* true if processing the last block */
Chris@43:     bin *in;                /* buffered input file */
Chris@43:     unsigned char *start;   /* start of compressed data in buffer */
Chris@43:     unsigned char *junk;    /* buffer for uncompressed data -- discarded */
Chris@43:     z_off_t len;            /* length of uncompressed data (support > 4 GB) */
Chris@43:     z_stream strm;          /* zlib inflate stream */
Chris@43: 
Chris@43:     /* open gzip file and skip header */
Chris@43:     in = bopen(name);
Chris@43:     if (in == NULL)
Chris@43:         bail("could not open ", name);
Chris@43:     gzhead(in);
Chris@43: 
Chris@43:     /* allocate buffer for uncompressed data and initialize raw inflate
Chris@43:        stream */
Chris@43:     junk = malloc(CHUNK);
Chris@43:     strm.zalloc = Z_NULL;
Chris@43:     strm.zfree = Z_NULL;
Chris@43:     strm.opaque = Z_NULL;
Chris@43:     strm.avail_in = 0;
Chris@43:     strm.next_in = Z_NULL;
Chris@43:     ret = inflateInit2(&strm, -15);
Chris@43:     if (junk == NULL || ret != Z_OK)
Chris@43:         bail("out of memory", "");
Chris@43: 
Chris@43:     /* inflate and copy compressed data, clear last-block bit if requested */
Chris@43:     len = 0;
Chris@43:     zpull(&strm, in);
Chris@43:     start = in->next;
Chris@43:     last = start[0] & 1;
Chris@43:     if (last && clr)
Chris@43:         start[0] &= ~1;
Chris@43:     strm.avail_out = 0;
Chris@43:     for (;;) {
Chris@43:         /* if input used and output done, write used input and get more */
Chris@43:         if (strm.avail_in == 0 && strm.avail_out != 0) {
Chris@43:             fwrite(start, 1, strm.next_in - start, out);
Chris@43:             start = in->buf;
Chris@43:             in->left = 0;
Chris@43:             zpull(&strm, in);
Chris@43:         }
Chris@43: 
Chris@43:         /* decompress -- return early when end-of-block reached */
Chris@43:         strm.avail_out = CHUNK;
Chris@43:         strm.next_out = junk;
Chris@43:         ret = inflate(&strm, Z_BLOCK);
Chris@43:         switch (ret) {
Chris@43:         case Z_MEM_ERROR:
Chris@43:             bail("out of memory", "");
Chris@43:         case Z_DATA_ERROR:
Chris@43:             bail("invalid compressed data in ", in->name);
Chris@43:         }
Chris@43: 
Chris@43:         /* update length of uncompressed data */
Chris@43:         len += CHUNK - strm.avail_out;
Chris@43: 
Chris@43:         /* check for block boundary (only get this when block copied out) */
Chris@43:         if (strm.data_type & 128) {
Chris@43:             /* if that was the last block, then done */
Chris@43:             if (last)
Chris@43:                 break;
Chris@43: 
Chris@43:             /* number of unused bits in last byte */
Chris@43:             pos = strm.data_type & 7;
Chris@43: 
Chris@43:             /* find the next last-block bit */
Chris@43:             if (pos != 0) {
Chris@43:                 /* next last-block bit is in last used byte */
Chris@43:                 pos = 0x100 >> pos;
Chris@43:                 last = strm.next_in[-1] & pos;
Chris@43:                 if (last && clr)
Chris@43:                     in->buf[strm.next_in - in->buf - 1] &= ~pos;
Chris@43:             }
Chris@43:             else {
Chris@43:                 /* next last-block bit is in next unused byte */
Chris@43:                 if (strm.avail_in == 0) {
Chris@43:                     /* don't have that byte yet -- get it */
Chris@43:                     fwrite(start, 1, strm.next_in - start, out);
Chris@43:                     start = in->buf;
Chris@43:                     in->left = 0;
Chris@43:                     zpull(&strm, in);
Chris@43:                 }
Chris@43:                 last = strm.next_in[0] & 1;
Chris@43:                 if (last && clr)
Chris@43:                     in->buf[strm.next_in - in->buf] &= ~1;
Chris@43:             }
Chris@43:         }
Chris@43:     }
Chris@43: 
Chris@43:     /* update buffer with unused input */
Chris@43:     in->left = strm.avail_in;
Chris@43:     in->next = in->buf + (strm.next_in - in->buf);
Chris@43: 
Chris@43:     /* copy used input, write empty blocks to get to byte boundary */
Chris@43:     pos = strm.data_type & 7;
Chris@43:     fwrite(start, 1, in->next - start - 1, out);
Chris@43:     last = in->next[-1];
Chris@43:     if (pos == 0 || !clr)
Chris@43:         /* already at byte boundary, or last file: write last byte */
Chris@43:         putc(last, out);
Chris@43:     else {
Chris@43:         /* append empty blocks to last byte */
Chris@43:         last &= ((0x100 >> pos) - 1);       /* assure unused bits are zero */
Chris@43:         if (pos & 1) {
Chris@43:             /* odd -- append an empty stored block */
Chris@43:             putc(last, out);
Chris@43:             if (pos == 1)
Chris@43:                 putc(0, out);               /* two more bits in block header */
Chris@43:             fwrite("\0\0\xff\xff", 1, 4, out);
Chris@43:         }
Chris@43:         else {
Chris@43:             /* even -- append 1, 2, or 3 empty fixed blocks */
Chris@43:             switch (pos) {
Chris@43:             case 6:
Chris@43:                 putc(last | 8, out);
Chris@43:                 last = 0;
Chris@43:             case 4:
Chris@43:                 putc(last | 0x20, out);
Chris@43:                 last = 0;
Chris@43:             case 2:
Chris@43:                 putc(last | 0x80, out);
Chris@43:                 putc(0, out);
Chris@43:             }
Chris@43:         }
Chris@43:     }
Chris@43: 
Chris@43:     /* update crc and tot */
Chris@43:     *crc = crc32_combine(*crc, bget4(in), len);
Chris@43:     *tot += (unsigned long)len;
Chris@43: 
Chris@43:     /* clean up */
Chris@43:     inflateEnd(&strm);
Chris@43:     free(junk);
Chris@43:     bclose(in);
Chris@43: 
Chris@43:     /* write trailer if this is the last gzip file */
Chris@43:     if (!clr) {
Chris@43:         put4(*crc, out);
Chris@43:         put4(*tot, out);
Chris@43:     }
Chris@43: }
Chris@43: 
Chris@43: /* join the gzip files on the command line, write result to stdout */
Chris@43: int main(int argc, char **argv)
Chris@43: {
Chris@43:     unsigned long crc, tot;     /* running crc and total uncompressed length */
Chris@43: 
Chris@43:     /* skip command name */
Chris@43:     argc--;
Chris@43:     argv++;
Chris@43: 
Chris@43:     /* show usage if no arguments */
Chris@43:     if (argc == 0) {
Chris@43:         fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
Chris@43:               stderr);
Chris@43:         return 0;
Chris@43:     }
Chris@43: 
Chris@43:     /* join gzip files on command line and write to stdout */
Chris@43:     gzinit(&crc, &tot, stdout);
Chris@43:     while (argc--)
Chris@43:         gzcopy(*argv++, argc, &crc, &tot, stdout);
Chris@43: 
Chris@43:     /* done */
Chris@43:     return 0;
Chris@43: }