471 lines
22 KiB
C
471 lines
22 KiB
C
|
/* gznorm.c -- normalize a gzip stream
|
||
|
|
* Copyright (C) 2018 Mark Adler
|
||
|
|
* For conditions of distribution and use, see copyright notice in zlib.h
|
||
|
|
* Version 1.0 7 Oct 2018 Mark Adler */
|
||
|
|
|
||
|
|
// gznorm takes a gzip stream, potentially containing multiple members, and
|
||
|
|
// converts it to a gzip stream with a single member. In addition the gzip
|
||
|
|
// header is normalized, removing the file name and time stamp, and setting the
|
||
|
|
// other header contents (XFL, OS) to fixed values. gznorm does not recompress
|
||
|
|
// the data, so it is fast, but no advantage is gained from the history that
|
||
|
|
// could be available across member boundaries.
|
||
|
|
|
||
|
|
#include <stdio.h> // fread, fwrite, putc, fflush, ferror, fprintf,
|
||
|
|
// vsnprintf, stdout, stderr, NULL, FILE
|
||
|
|
#include <stdlib.h> // malloc, free
|
||
|
|
#include <string.h> // strerror
|
||
|
|
#include <errno.h> // errno
|
||
|
|
#include <stdarg.h> // va_list, va_start, va_end
|
||
|
|
#include "zlib.h" // inflateInit2, inflate, inflateReset, inflateEnd,
|
||
|
|
// z_stream, z_off_t, crc32_combine, Z_NULL, Z_BLOCK,
|
||
|
|
// Z_OK, Z_STREAM_END, Z_BUF_ERROR, Z_DATA_ERROR,
|
||
|
|
// Z_MEM_ERROR
|
||
|
|
|
||
|
|
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
|
||
|
|
# include <fcntl.h>
|
||
|
|
# include <io.h>
|
||
|
|
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
|
||
|
|
#else
|
||
|
|
# define SET_BINARY_MODE(file)
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#define local static
|
||
|
|
|
||
|
|
// printf to an allocated string. Return the string, or NULL if the printf or
|
||
|
|
// allocation fails.
|
||
|
|
local char *aprintf(char *fmt, ...) {
|
||
|
|
// Get the length of the result of the printf.
|
||
|
|
va_list args;
|
||
|
|
va_start(args, fmt);
|
||
|
|
int len = vsnprintf(NULL, 0, fmt, args);
|
||
|
|
va_end(args);
|
||
|
|
if (len < 0)
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
// Allocate the required space and printf to it.
|
||
|
|
char *str = malloc(len + 1);
|
||
|
|
if (str == NULL)
|
||
|
|
return NULL;
|
||
|
|
va_start(args, fmt);
|
||
|
|
vsnprintf(str, len + 1, fmt, args);
|
||
|
|
va_end(args);
|
||
|
|
return str;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Return with an error, putting an allocated error message in *err. Doing an
|
||
|
|
// inflateEnd() on an already ended state, or one with state set to Z_NULL, is
|
||
|
|
// permitted.
|
||
|
|
#define BYE(...) \
|
||
|
|
do { \
|
||
|
|
inflateEnd(&strm); \
|
||
|
|
*err = aprintf(__VA_ARGS__); \
|
||
|
|
return 1; \
|
||
|
|
} while (0)
|
||
|
|
|
||
|
|
// Chunk size for buffered reads and for decompression. Twice this many bytes
|
||
|
|
// will be allocated on the stack by gzip_normalize(). Must fit in an unsigned.
|
||
|
|
#define CHUNK 16384
|
||
|
|
|
||
|
|
// Read a gzip stream from in and write an equivalent normalized gzip stream to
|
||
|
|
// out. If given no input, an empty gzip stream will be written. If successful,
|
||
|
|
// 0 is returned, and *err is set to NULL. On error, 1 is returned, where the
|
||
|
|
// details of the error are returned in *err, a pointer to an allocated string.
|
||
|
|
//
|
||
|
|
// The input may be a stream with multiple gzip members, which is converted to
|
||
|
|
// a single gzip member on the output. Each gzip member is decompressed at the
|
||
|
|
// level of deflate blocks. This enables clearing the last-block bit, shifting
|
||
|
|
// the compressed data to concatenate to the previous member's compressed data,
|
||
|
|
// which can end at an arbitrary bit boundary, and identifying stored blocks in
|
||
|
|
// order to resynchronize those to byte boundaries. The deflate compressed data
|
||
|
|
// is terminated with a 10-bit empty fixed block. If any members on the input
|
||
|
|
// end with a 10-bit empty fixed block, then that block is excised from the
|
||
|
|
// stream. This avoids appending empty fixed blocks for every normalization,
|
||
|
|
// and assures that gzip_normalize applied a second time will not change the
|
||
|
|
// input. The pad bits after stored block headers and after the final deflate
|
||
|
|
// block are all forced to zeros.
|
||
|
|
local int gzip_normalize(FILE *in, FILE *out, char **err) {
|
||
|
|
// initialize the inflate engine to process a gzip member
|
||
|
|
z_stream strm;
|
||
|
|
strm.zalloc = Z_NULL;
|
||
|
|
strm.zfree = Z_NULL;
|
||
|
|
strm.opaque = Z_NULL;
|
||
|
|
strm.avail_in = 0;
|
||
|
|
strm.next_in = Z_NULL;
|
||
|
|
if (inflateInit2(&strm, 15 + 16) != Z_OK)
|
||
|
|
BYE("out of memory");
|
||
|
|
|
||
|
|
// State while processing the input gzip stream.
|
||
|
|
enum { // BETWEEN -> HEAD -> BLOCK -> TAIL -> BETWEEN -> ...
|
||
|
|
BETWEEN, // between gzip members (must end in this state)
|
||
|
|
HEAD, // reading a gzip header
|
||
|
|
BLOCK, // reading deflate blocks
|
||
|
|
TAIL // reading a gzip trailer
|
||
|
|
} state = BETWEEN; // current component being processed
|
||
|
|
unsigned long crc = 0; // accumulated CRC of uncompressed data
|
||
|
|
unsigned long len = 0; // accumulated length of uncompressed data
|
||
|
|
unsigned long buf = 0; // deflate stream bit buffer of num bits
|
||
|
|
int num = 0; // number of bits in buf (at bottom)
|
||
|
|
|
||
|
|
// Write a canonical gzip header (no mod time, file name, comment, extra
|
||
|
|
// block, or extra flags, and OS is marked as unknown).
|
||
|
|
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
|
||
|
|
|
||
|
|
// Process the gzip stream from in until reaching the end of the input,
|
||
|
|
// encountering invalid input, or experiencing an i/o error.
|
||
|
|
int more; // true if not at the end of the input
|
||
|
|
do {
|
||
|
|
// State inside this loop.
|
||
|
|
unsigned char *put; // next input buffer location to process
|
||
|
|
int prev; // number of bits from previous block in
|
||
|
|
// the bit buffer, or -1 if not at the
|
||
|
|
// start of a block
|
||
|
|
unsigned long long memb; // uncompressed length of member
|
||
|
|
size_t tail; // number of trailer bytes read (0..8)
|
||
|
|
unsigned long part; // accumulated trailer component
|
||
|
|
|
||
|
|
// Get the next chunk of input from in.
|
||
|
|
unsigned char dat[CHUNK];
|
||
|
|
strm.avail_in = fread(dat, 1, CHUNK, in);
|
||
|
|
if (strm.avail_in == 0)
|
||
|
|
break;
|
||
|
|
more = strm.avail_in == CHUNK;
|
||
|
|
strm.next_in = put = dat;
|
||
|
|
|
||
|
|
// Run that chunk of input through the inflate engine to exhaustion.
|
||
|
|
do {
|
||
|
|
// At this point it is assured that strm.avail_in > 0.
|
||
|
|
|
||
|
|
// Inflate until the end of a gzip component (header, deflate
|
||
|
|
// block, trailer) is reached, or until all of the chunk is
|
||
|
|
// consumed. The resulting decompressed data is discarded, though
|
||
|
|
// the total size of the decompressed data in each member is
|
||
|
|
// tracked, for the calculation of the total CRC.
|
||
|
|
do {
|
||
|
|
// inflate and handle any errors
|
||
|
|
unsigned char scrap[CHUNK];
|
||
|
|
strm.avail_out = CHUNK;
|
||
|
|
strm.next_out = scrap;
|
||
|
|
int ret = inflate(&strm, Z_BLOCK);
|
||
|
|
if (ret == Z_MEM_ERROR)
|
||
|
|
BYE("out of memory");
|
||
|
|
if (ret == Z_DATA_ERROR)
|
||
|
|
BYE("input invalid: %s", strm.msg);
|
||
|
|
if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_STREAM_END)
|
||
|
|
BYE("internal error");
|
||
|
|
|
||
|
|
// Update the number of uncompressed bytes generated in this
|
||
|
|
// member. The actual count (not modulo 2^32) is required to
|
||
|
|
// correctly compute the total CRC.
|
||
|
|
unsigned got = CHUNK - strm.avail_out;
|
||
|
|
memb += got;
|
||
|
|
if (memb < got)
|
||
|
|
BYE("overflow error");
|
||
|
|
|
||
|
|
// Continue to process this chunk until it is consumed, or
|
||
|
|
// until the end of a component (header, deflate block, or
|
||
|
|
// trailer) is reached.
|
||
|
|
} while (strm.avail_out == 0 && (strm.data_type & 0x80) == 0);
|
||
|
|
|
||
|
|
// Since strm.avail_in was > 0 for the inflate call, some input was
|
||
|
|
// just consumed. It is therefore assured that put < strm.next_in.
|
||
|
|
|
||
|
|
// Disposition the consumed component or part of a component.
|
||
|
|
switch (state) {
|
||
|
|
case BETWEEN:
|
||
|
|
state = HEAD;
|
||
|
|
// Fall through to HEAD when some or all of the header is
|
||
|
|
// processed.
|
||
|
|
|
||
|
|
case HEAD:
|
||
|
|
// Discard the header.
|
||
|
|
if (strm.data_type & 0x80) {
|
||
|
|
// End of header reached -- deflate blocks follow.
|
||
|
|
put = strm.next_in;
|
||
|
|
prev = num;
|
||
|
|
memb = 0;
|
||
|
|
state = BLOCK;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
case BLOCK:
|
||
|
|
// Copy the deflate stream to the output, but with the
|
||
|
|
// last-block-bit cleared. Re-synchronize stored block
|
||
|
|
// headers to the output byte boundaries. The bytes at
|
||
|
|
// put..strm.next_in-1 is the compressed data that has been
|
||
|
|
// processed and is ready to be copied to the output.
|
||
|
|
|
||
|
|
// At this point, it is assured that new compressed data is
|
||
|
|
// available, i.e., put < strm.next_in. If prev is -1, then
|
||
|
|
// that compressed data starts in the middle of a deflate
|
||
|
|
// block. If prev is not -1, then the bits in the bit
|
||
|
|
// buffer, possibly combined with the bits in *put, contain
|
||
|
|
// the three-bit header of the new deflate block. In that
|
||
|
|
// case, prev is the number of bits from the previous block
|
||
|
|
// that remain in the bit buffer. Since num is the number
|
||
|
|
// of bits in the bit buffer, we have that num - prev is
|
||
|
|
// the number of bits from the new block currently in the
|
||
|
|
// bit buffer.
|
||
|
|
|
||
|
|
// If strm.data_type & 0xc0 is 0x80, then the last byte of
|
||
|
|
// the available compressed data includes the last bits of
|
||
|
|
// the end of a deflate block. In that case, that last byte
|
||
|
|
// also has strm.data_type & 0x1f bits of the next deflate
|
||
|
|
// block, in the range 0..7. If strm.data_type & 0xc0 is
|
||
|
|
// 0xc0, then the last byte of the compressed data is the
|
||
|
|
// end of the deflate stream, followed by strm.data_type &
|
||
|
|
// 0x1f pad bits, also in the range 0..7.
|
||
|
|
|
||
|
|
// Set bits to the number of bits not yet consumed from the
|
||
|
|
// last byte. If we are at the end of the block, bits is
|
||
|
|
// either the number of bits in the last byte belonging to
|
||
|
|
// the next block, or the number of pad bits after the
|
||
|
|
// final block. In either of those cases, bits is in the
|
||
|
|
// range 0..7.
|
||
|
|
; // (required due to C syntax oddity)
|
||
|
|
int bits = strm.data_type & 0x1f;
|
||
|
|
|
||
|
|
if (prev != -1) {
|
||
|
|
// We are at the start of a new block. Clear the last
|
||
|
|
// block bit, and check for special cases. If it is a
|
||
|
|
// stored block, then emit the header and pad to the
|
||
|
|
// next byte boundary. If it is a final, empty fixed
|
||
|
|
// block, then excise it.
|
||
|
|
|
||
|
|
// Some or all of the three header bits for this block
|
||
|
|
// may already be in the bit buffer. Load any remaining
|
||
|
|
// header bits into the bit buffer.
|
||
|
|
if (num - prev < 3) {
|
||
|
|
buf += (unsigned long)*put++ << num;
|
||
|
|
num += 8;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Set last to have a 1 in the position of the last
|
||
|
|
// block bit in the bit buffer.
|
||
|
|
unsigned long last = (unsigned long)1 << prev;
|
||
|
|
|
||
|
|
if (((buf >> prev) & 7) == 3) {
|
||
|
|
// This is a final fixed block. Load at least ten
|
||
|
|
// bits from this block, including the header, into
|
||
|
|
// the bit buffer. We already have at least three,
|
||
|
|
// so at most one more byte needs to be loaded.
|
||
|
|
if (num - prev < 10) {
|
||
|
|
if (put == strm.next_in)
|
||
|
|
// Need to go get and process more input.
|
||
|
|
// We'll end up back here to finish this.
|
||
|
|
break;
|
||
|
|
buf += (unsigned long)*put++ << num;
|
||
|
|
num += 8;
|
||
|
|
}
|
||
|
|
if (((buf >> prev) & 0x3ff) == 3) {
|
||
|
|
// That final fixed block is empty. Delete it
|
||
|
|
// to avoid adding an empty block every time a
|
||
|
|
// gzip stream is normalized.
|
||
|
|
num = prev;
|
||
|
|
buf &= last - 1; // zero the pad bits
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if (((buf >> prev) & 6) == 0) {
|
||
|
|
// This is a stored block. Flush to the next
|
||
|
|
// byte boundary after the three-bit header.
|
||
|
|
num = (prev + 10) & ~7;
|
||
|
|
buf &= last - 1; // zero the pad bits
|
||
|
|
}
|
||
|
|
|
||
|
|
// Clear the last block bit.
|
||
|
|
buf &= ~last;
|
||
|
|
|
||
|
|
// Write out complete bytes in the bit buffer.
|
||
|
|
while (num >= 8) {
|
||
|
|
putc(buf, out);
|
||
|
|
buf >>= 8;
|
||
|
|
num -= 8;
|
||
|
|
}
|
||
|
|
|
||
|
|
// If no more bytes left to process, then we have
|
||
|
|
// consumed the byte that had bits from the next block.
|
||
|
|
if (put == strm.next_in)
|
||
|
|
bits = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
// We are done handling the deflate block header. Now copy
|
||
|
|
// all or almost all of the remaining compressed data that
|
||
|
|
// has been processed so far. Don't copy one byte at the
|
||
|
|
// end if it contains bits from the next deflate block or
|
||
|
|
// pad bits at the end of a deflate block.
|
||
|
|
|
||
|
|
// mix is 1 if we are at the end of a deflate block, and if
|
||
|
|
// some of the bits in the last byte follow this block. mix
|
||
|
|
// is 0 if we are in the middle of a deflate block, if the
|
||
|
|
// deflate block ended on a byte boundary, or if all of the
|
||
|
|
// compressed data processed so far has been consumed.
|
||
|
|
int mix = (strm.data_type & 0x80) && bits;
|
||
|
|
|
||
|
|
// Copy all of the processed compressed data to the output,
|
||
|
|
// except for the last byte if it contains bits from the
|
||
|
|
// next deflate block or pad bits at the end of the deflate
|
||
|
|
// stream. Copy the data after shifting in num bits from
|
||
|
|
// buf in front of it, leaving num bits from the end of the
|
||
|
|
// compressed data in buf when done.
|
||
|
|
unsigned char *end = strm.next_in - mix;
|
||
|
|
if (put < end) {
|
||
|
|
if (num)
|
||
|
|
// Insert num bits from buf before the data being
|
||
|
|
// copied.
|
||
|
|
do {
|
||
|
|
buf += (unsigned)(*put++) << num;
|
||
|
|
putc(buf, out);
|
||
|
|
buf >>= 8;
|
||
|
|
} while (put < end);
|
||
|
|
else {
|
||
|
|
// No shifting needed -- write directly.
|
||
|
|
fwrite(put, 1, end - put, out);
|
||
|
|
put = end;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Process the last processed byte if it wasn't written.
|
||
|
|
if (mix) {
|
||
|
|
// Load the last byte into the bit buffer.
|
||
|
|
buf += (unsigned)(*put++) << num;
|
||
|
|
num += 8;
|
||
|
|
|
||
|
|
if (strm.data_type & 0x40) {
|
||
|
|
// We are at the end of the deflate stream and
|
||
|
|
// there are bits pad bits. Discard the pad bits
|
||
|
|
// and write a byte to the output, if available.
|
||
|
|
// Leave the num bits left over in buf to prepend
|
||
|
|
// to the next deflate stream.
|
||
|
|
num -= bits;
|
||
|
|
if (num >= 8) {
|
||
|
|
putc(buf, out);
|
||
|
|
num -= 8;
|
||
|
|
buf >>= 8;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Force the pad bits in the bit buffer to zeros.
|
||
|
|
buf &= ((unsigned long)1 << num) - 1;
|
||
|
|
|
||
|
|
// Don't need to set prev here since going to TAIL.
|
||
|
|
}
|
||
|
|
else
|
||
|
|
// At the end of an internal deflate block. Leave
|
||
|
|
// the last byte in the bit buffer to examine on
|
||
|
|
// the next entry to BLOCK, when more bits from the
|
||
|
|
// next block will be available.
|
||
|
|
prev = num - bits; // number of bits in buffer
|
||
|
|
// from current block
|
||
|
|
}
|
||
|
|
|
||
|
|
// Don't have a byte left over, so we are in the middle of
|
||
|
|
// a deflate block, or the deflate block ended on a byte
|
||
|
|
// boundary. Set prev appropriately for the next entry into
|
||
|
|
// BLOCK.
|
||
|
|
else if (strm.data_type & 0x80)
|
||
|
|
// The block ended on a byte boundary, so no header
|
||
|
|
// bits are in the bit buffer.
|
||
|
|
prev = num;
|
||
|
|
else
|
||
|
|
// In the middle of a deflate block, so no header here.
|
||
|
|
prev = -1;
|
||
|
|
|
||
|
|
// Check for the end of the deflate stream.
|
||
|
|
if ((strm.data_type & 0xc0) == 0xc0) {
|
||
|
|
// That ends the deflate stream on the input side, the
|
||
|
|
// pad bits were discarded, and any remaining bits from
|
||
|
|
// the last block in the stream are saved in the bit
|
||
|
|
// buffer to prepend to the next stream. Process the
|
||
|
|
// gzip trailer next.
|
||
|
|
tail = 0;
|
||
|
|
part = 0;
|
||
|
|
state = TAIL;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TAIL:
|
||
|
|
// Accumulate available trailer bytes to update the total
|
||
|
|
// CRC and the total uncompressed length.
|
||
|
|
do {
|
||
|
|
part = (part >> 8) + ((unsigned long)(*put++) << 24);
|
||
|
|
tail++;
|
||
|
|
if (tail == 4) {
|
||
|
|
// Update the total CRC.
|
||
|
|
z_off_t len2 = memb;
|
||
|
|
if (len2 < 0 || (unsigned long long)len2 != memb)
|
||
|
|
BYE("overflow error");
|
||
|
|
crc = crc ? crc32_combine(crc, part, len2) : part;
|
||
|
|
part = 0;
|
||
|
|
}
|
||
|
|
else if (tail == 8) {
|
||
|
|
// Update the total uncompressed length. (It's ok
|
||
|
|
// if this sum is done modulo 2^32.)
|
||
|
|
len += part;
|
||
|
|
|
||
|
|
// At the end of a member. Set up to inflate an
|
||
|
|
// immediately following gzip member. (If we made
|
||
|
|
// it this far, then the trailer was valid.)
|
||
|
|
if (inflateReset(&strm) != Z_OK)
|
||
|
|
BYE("internal error");
|
||
|
|
state = BETWEEN;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
} while (put < strm.next_in);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Process the input buffer until completely consumed.
|
||
|
|
} while (strm.avail_in > 0);
|
||
|
|
|
||
|
|
// Process input until end of file, invalid input, or i/o error.
|
||
|
|
} while (more);
|
||
|
|
|
||
|
|
// Done with the inflate engine.
|
||
|
|
inflateEnd(&strm);
|
||
|
|
|
||
|
|
// Verify the validity of the input.
|
||
|
|
if (state != BETWEEN)
|
||
|
|
BYE("input invalid: incomplete gzip stream");
|
||
|
|
|
||
|
|
// Write the remaining deflate stream bits, followed by a terminating
|
||
|
|
// deflate fixed block.
|
||
|
|
buf += (unsigned long)3 << num;
|
||
|
|
putc(buf, out);
|
||
|
|
putc(buf >> 8, out);
|
||
|
|
if (num > 6)
|
||
|
|
putc(0, out);
|
||
|
|
|
||
|
|
// Write the gzip trailer, which is the CRC and the uncompressed length
|
||
|
|
// modulo 2^32, both in little-endian order.
|
||
|
|
putc(crc, out);
|
||
|
|
putc(crc >> 8, out);
|
||
|
|
putc(crc >> 16, out);
|
||
|
|
putc(crc >> 24, out);
|
||
|
|
putc(len, out);
|
||
|
|
putc(len >> 8, out);
|
||
|
|
putc(len >> 16, out);
|
||
|
|
putc(len >> 24, out);
|
||
|
|
fflush(out);
|
||
|
|
|
||
|
|
// Check for any i/o errors.
|
||
|
|
if (ferror(in) || ferror(out))
|
||
|
|
BYE("i/o error: %s", strerror(errno));
|
||
|
|
|
||
|
|
// All good!
|
||
|
|
*err = NULL;
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Normalize the gzip stream on stdin, writing the result to stdout.
|
||
|
|
int main(void) {
|
||
|
|
// Avoid end-of-line conversions on evil operating systems.
|
||
|
|
SET_BINARY_MODE(stdin);
|
||
|
|
SET_BINARY_MODE(stdout);
|
||
|
|
|
||
|
|
// Normalize from stdin to stdout, returning 1 on error, 0 if ok.
|
||
|
|
char *err;
|
||
|
|
int ret = gzip_normalize(stdin, stdout, &err);
|
||
|
|
if (ret)
|
||
|
|
fprintf(stderr, "gznorm error: %s\n", err);
|
||
|
|
free(err);
|
||
|
|
return ret;
|
||
|
|
}
|