2 * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <sys/queue.h>
26 #include "got_error.h"
27 #include "got_object.h"
30 #include "got_lib_deflate.h"
31 #include "got_lib_hash.h"
32 #include "got_lib_poll.h"
35 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
38 const struct got_error *
39 got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
41 const struct got_error *err = NULL;
44 memset(zb, 0, sizeof(*zb));
46 zb->z.zalloc = Z_NULL;
48 zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);
51 return got_error_from_errno("deflateInit");
52 if (zerr == Z_MEM_ERROR) {
54 return got_error_from_errno("deflateInit");
56 return got_error(GOT_ERR_COMPRESSION);
59 zb->inlen = zb->outlen = bufsize;
61 zb->inbuf = calloc(1, zb->inlen);
62 if (zb->inbuf == NULL) {
63 err = got_error_from_errno("calloc");
69 zb->outbuf = calloc(1, zb->outlen);
70 if (zb->outbuf == NULL) {
71 err = got_error_from_errno("calloc");
74 zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;
84 csum_output(struct got_deflate_checksum *csum, const uint8_t *buf, size_t len)
87 *csum->output_crc = crc32(*csum->output_crc, buf, len);
89 if (csum->output_sha1)
90 SHA1Update(csum->output_sha1, buf, len);
93 got_hash_update(csum->output_ctx, buf, len);
96 const struct got_error *
97 got_deflate_read(struct got_deflate_buf *zb, FILE *f, off_t len,
98 size_t *outlenp, off_t *consumed)
100 size_t last_total_out = zb->z.total_out;
101 z_stream *z = &zb->z;
104 z->next_out = zb->outbuf;
105 z->avail_out = zb->outlen;
110 size_t last_total_in = z->total_in;
111 if (z->avail_in == 0) {
113 if (*consumed < len) {
114 n = fread(zb->inbuf, 1,
115 MIN(zb->inlen, len - *consumed), f);
119 return got_ferror(f, GOT_ERR_IO);
121 ret = deflate(z, Z_FINISH);
124 z->next_in = zb->inbuf;
127 ret = deflate(z, Z_NO_FLUSH);
128 *consumed += z->total_in - last_total_in;
129 } while (ret == Z_OK && z->avail_out > 0);
132 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
134 if (ret != Z_STREAM_END)
135 return got_error(GOT_ERR_COMPRESSION);
136 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
139 *outlenp = z->total_out - last_total_out;
143 static const struct got_error *
144 deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
145 size_t len, size_t *outlenp, size_t *consumed, int flush_on_eof)
147 z_stream *z = &zb->z;
148 size_t last_total_out = z->total_out;
151 z->next_out = zb->outbuf;
152 z->avail_out = zb->outlen;
157 size_t last_total_in = z->total_in;
158 if (z->avail_in == 0) {
159 z->next_in = map + offset + *consumed;
160 if (len - *consumed > UINT_MAX)
161 z->avail_in = UINT_MAX;
163 z->avail_in = len - *consumed;
164 if (z->avail_in == 0) {
167 ret = deflate(z, Z_FINISH);
171 ret = deflate(z, Z_NO_FLUSH);
172 *consumed += z->total_in - last_total_in;
173 } while (ret == Z_OK && z->avail_out > 0);
176 zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
178 if (ret != Z_STREAM_END)
179 return got_error(GOT_ERR_COMPRESSION);
180 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
183 *outlenp = z->total_out - last_total_out;
187 const struct got_error *
188 got_deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
189 size_t len, size_t *outlenp, size_t *consumed)
191 return deflate_read_mmap(zb, map, offset, len, outlenp, consumed, 1);
194 const struct got_error *
195 got_deflate_flush(struct got_deflate_buf *zb, FILE *outfile,
196 struct got_deflate_checksum *csum, off_t *outlenp)
200 z_stream *z = &zb->z;
202 if (z->avail_in != 0)
203 return got_error_msg(GOT_ERR_COMPRESSION,
204 "cannot flush zb with pending input data");
207 size_t avail, last_total_out = zb->z.total_out;
209 z->next_out = zb->outbuf;
210 z->avail_out = zb->outlen;
212 ret = deflate(z, Z_FINISH);
213 if (ret != Z_STREAM_END && ret != Z_OK)
214 return got_error(GOT_ERR_COMPRESSION);
216 avail = z->total_out - last_total_out;
218 n = fwrite(zb->outbuf, avail, 1, outfile);
220 return got_ferror(outfile, GOT_ERR_IO);
222 csum_output(csum, zb->outbuf, avail);
226 } while (ret != Z_STREAM_END);
228 zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
233 got_deflate_end(struct got_deflate_buf *zb)
236 if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
241 const struct got_error *
242 got_deflate_to_fd(off_t *outlen, FILE *infile, off_t len, int outfd,
243 struct got_deflate_checksum *csum)
245 const struct got_error *err;
248 struct got_deflate_buf zb;
250 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
257 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
262 err = got_poll_write_full(outfd, zb.outbuf, avail);
266 csum_output(csum, zb.outbuf, avail);
269 } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
272 got_deflate_end(&zb);
276 const struct got_error *
277 got_deflate_to_fd_mmap(off_t *outlen, uint8_t *map, size_t offset,
278 size_t len, int outfd, struct got_deflate_checksum *csum)
280 const struct got_error *err;
281 size_t avail, consumed;
282 struct got_deflate_buf zb;
284 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
290 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
297 err = got_poll_write_full(outfd, zb.outbuf, avail);
301 csum_output(csum, zb.outbuf, avail);
304 } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
307 got_deflate_end(&zb);
311 const struct got_error *
312 got_deflate_to_file(off_t *outlen, FILE *infile, off_t len,
313 FILE *outfile, struct got_deflate_checksum *csum)
315 const struct got_error *err;
318 struct got_deflate_buf zb;
320 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
327 err = got_deflate_read(&zb, infile, len, &avail, &consumed);
333 n = fwrite(zb.outbuf, avail, 1, outfile);
335 err = got_ferror(outfile, GOT_ERR_IO);
339 csum_output(csum, zb.outbuf, avail);
342 } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
345 got_deflate_end(&zb);
349 const struct got_error *
350 got_deflate_to_file_mmap(off_t *outlen, uint8_t *map, size_t offset,
351 size_t len, FILE *outfile, struct got_deflate_checksum *csum)
353 const struct got_error *err;
354 size_t avail, consumed;
355 struct got_deflate_buf zb;
357 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
363 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
371 n = fwrite(zb.outbuf, avail, 1, outfile);
373 err = got_ferror(outfile, GOT_ERR_IO);
377 csum_output(csum, zb.outbuf, avail);
380 } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
383 got_deflate_end(&zb);
387 const struct got_error *
388 got_deflate_append_to_file_mmap(struct got_deflate_buf *zb, off_t *outlen,
389 uint8_t *map, size_t offset, size_t len, FILE *outfile,
390 struct got_deflate_checksum *csum)
392 const struct got_error *err;
393 size_t avail, consumed;
396 err = deflate_read_mmap(zb, map, offset, len, &avail,
404 n = fwrite(zb->outbuf, avail, 1, outfile);
406 err = got_ferror(outfile, GOT_ERR_IO);
410 csum_output(csum, zb->outbuf, avail);
414 } while ((zb->flags & GOT_DEFLATE_F_HAVE_MORE) && len > 0);
419 const struct got_error *
420 got_deflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
421 size_t *consumed_total, struct got_deflate_checksum *csum, uint8_t *map,
422 size_t offset, size_t len)
424 const struct got_error *err;
425 size_t avail, consumed;
426 struct got_deflate_buf zb;
431 *outbuf = malloc(GOT_DEFLATE_BUFSIZE);
433 return got_error_from_errno("malloc");
434 err = got_deflate_init(&zb, *outbuf, GOT_DEFLATE_BUFSIZE);
441 err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
450 err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
456 *consumed_total += consumed;
458 if (avail > 0 && csum)
459 csum_output(csum, zb.outbuf, avail);
461 if ((zb.flags & GOT_DEFLATE_F_HAVE_MORE) && outbuf != NULL) {
462 newbuf = reallocarray(*outbuf, ++nbuf,
463 GOT_DEFLATE_BUFSIZE);
464 if (newbuf == NULL) {
465 err = got_error_from_errno("reallocarray");
472 zb.outbuf = newbuf + *outlen;
473 zb.outlen = (nbuf * GOT_DEFLATE_BUFSIZE) - *outlen;
475 } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
477 got_deflate_end(&zb);