Gotweb

Blob

Date:: Thu May 12 23:10:56 2022 UTC
Message:: store deltas in compressed form while packing, both in memory and cache file This reduces memory and disk space consumption during packing. with tweaks + memleak on error fix from op@ ok op@
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 
18 #include <errno.h>
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22 #include <zlib.h>
23 #include <time.h>
24 
25 #include "got_error.h"
26 #include "got_object.h"
27 #include "got_path.h"
28 
29 #include "got_lib_deflate.h"
30 
31 #ifndef MIN
32 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
33 #endif
34 
35 const struct got_error *
36 got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
37 {
38 	const struct got_error *err = NULL;
39 	int zerr;
40 
41 	memset(&zb->z, 0, sizeof(zb->z));
42 
43 	zb->z.zalloc = Z_NULL;
44 	zb->z.zfree = Z_NULL;
45 	zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);
46 	if (zerr != Z_OK) {
47 		if  (zerr == Z_ERRNO)
48 			return got_error_from_errno("deflateInit");
49 		if  (zerr == Z_MEM_ERROR) {
50 			errno = ENOMEM;
51 			return got_error_from_errno("deflateInit");
52 		}
53 		return got_error(GOT_ERR_COMPRESSION);
54 	}
55 
56 	zb->inlen = zb->outlen = bufsize;
57 
58 	zb->inbuf = calloc(1, zb->inlen);
59 	if (zb->inbuf == NULL) {
60 		err = got_error_from_errno("calloc");
61 		goto done;
62 	}
63 
64 	zb->flags = 0;
65 	if (outbuf == NULL) {
66 		zb->outbuf = calloc(1, zb->outlen);
67 		if (zb->outbuf == NULL) {
68 			err = got_error_from_errno("calloc");
69 			goto done;
70 		}
71 		zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;
72 	} else
73 		zb->outbuf = outbuf;
74 done:
75 	if (err)
76 		got_deflate_end(zb);
77 	return err;
78 }
79 
80 static void
81 csum_output(struct got_deflate_checksum *csum, const uint8_t *buf, size_t len)
82 {
83 	if (csum->output_crc)
84 		*csum->output_crc = crc32(*csum->output_crc, buf, len);
85 
86 	if (csum->output_sha1)
87 		SHA1Update(csum->output_sha1, buf, len);
88 }
89 
90 const struct got_error *
91 got_deflate_read(struct got_deflate_buf *zb, FILE *f, off_t len,
92     size_t *outlenp, off_t *consumed)
93 {
94 	size_t last_total_out = zb->z.total_out;
95 	z_stream *z = &zb->z;
96 	int ret = Z_ERRNO;
97 
98 	z->next_out = zb->outbuf;
99 	z->avail_out = zb->outlen;
100 
101 	*outlenp = 0;
102 	*consumed = 0;
103 	do {
104 		size_t last_total_in = z->total_in;
105 		if (z->avail_in == 0) {
106 			size_t n = 0;
107 			if (*consumed < len) {
108 				n = fread(zb->inbuf, 1,
109 				    MIN(zb->inlen, len - *consumed), f);
110 			}
111 			if (n == 0) {
112 				if (ferror(f))
113 					return got_ferror(f, GOT_ERR_IO);
114 				/* EOF */
115 				ret = deflate(z, Z_FINISH);
116 				break;
117 			}
118 			z->next_in = zb->inbuf;
119 			z->avail_in = n;
120 		}
121 		ret = deflate(z, Z_NO_FLUSH);
122 		*consumed += z->total_in - last_total_in;
123 	} while (ret == Z_OK && z->avail_out > 0);
124 
125 	if (ret == Z_OK) {
126 		zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
127 	} else {
128 		if (ret != Z_STREAM_END)
129 			return got_error(GOT_ERR_COMPRESSION);
130 		zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
131 	}
132 
133 	*outlenp = z->total_out - last_total_out;
134 	return NULL;
135 }
136 
137 static const struct got_error *
138 deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
139     size_t len, size_t *outlenp, size_t *consumed, int flush_on_eof)
140 {
141 	z_stream *z = &zb->z;
142 	size_t last_total_out = z->total_out;
143 	int ret = Z_ERRNO;
144 
145 	z->next_out = zb->outbuf;
146 	z->avail_out = zb->outlen;
147 
148 	*outlenp = 0;
149 	*consumed = 0;
150 	do {
151 		size_t last_total_in = z->total_in;
152 		if (z->avail_in == 0) {
153 			z->next_in = map + offset + *consumed;
154 			if (len - *consumed > UINT_MAX)
155 				z->avail_in = UINT_MAX;
156 			else
157 				z->avail_in = len - *consumed;
158 			if (z->avail_in == 0) {
159 				/* EOF */
160 				if (flush_on_eof)
161 					ret = deflate(z, Z_FINISH);
162 				break;
163 			}
164 		}
165 		ret = deflate(z, Z_NO_FLUSH);
166 		*consumed += z->total_in - last_total_in;
167 	} while (ret == Z_OK && z->avail_out > 0);
168 
169 	if (ret == Z_OK) {
170 		zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
171 	} else {
172 		if (ret != Z_STREAM_END)
173 			return got_error(GOT_ERR_COMPRESSION);
174 		zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
175 	}
176 
177 	*outlenp = z->total_out - last_total_out;
178 	return NULL;
179 }
180 
181 const struct got_error *
182 got_deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,
183     size_t len, size_t *outlenp, size_t *consumed)
184 {
185 	return deflate_read_mmap(zb, map, offset, len, outlenp, consumed, 1);
186 }
187 
188 const struct got_error *
189 got_deflate_flush(struct got_deflate_buf *zb, FILE *outfile,
190     struct got_deflate_checksum *csum, off_t *outlenp)
191 {
192 	int ret;
193 	size_t n;
194 	z_stream *z = &zb->z;
195 
196 	if (z->avail_in != 0)
197 		return got_error_msg(GOT_ERR_COMPRESSION,
198 		    "cannot flush zb with pending input data");
199 
200 	do {
201 		size_t avail, last_total_out = zb->z.total_out;
202 
203 		z->next_out = zb->outbuf;
204 		z->avail_out = zb->outlen;
205 
206 		ret = deflate(z, Z_FINISH);
207 		if (ret != Z_STREAM_END && ret != Z_OK)
208 			return got_error(GOT_ERR_COMPRESSION);
209 
210 		avail = z->total_out - last_total_out;
211 		if (avail > 0) {
212 			n = fwrite(zb->outbuf, avail, 1, outfile);
213 			if (n != 1)
214 				return got_ferror(outfile, GOT_ERR_IO);
215 			if (csum)
216 				csum_output(csum, zb->outbuf, avail);
217 			if (outlenp)
218 				*outlenp += avail;
219 		}
220 	} while (ret != Z_STREAM_END);
221 
222 	zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
223 	return NULL;
224 }
225 
226 void
227 got_deflate_end(struct got_deflate_buf *zb)
228 {
229 	free(zb->inbuf);
230 	if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
231 		free(zb->outbuf);
232 	deflateEnd(&zb->z);
233 }
234 
235 const struct got_error *
236 got_deflate_to_file(off_t *outlen, FILE *infile, off_t len,
237     FILE *outfile, struct got_deflate_checksum *csum)
238 {
239 	const struct got_error *err;
240 	size_t avail;
241 	off_t consumed;
242 	struct got_deflate_buf zb;
243 
244 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
245 	if (err)
246 		goto done;
247 
248 	*outlen = 0;
249 
250 	do {
251 		err = got_deflate_read(&zb, infile, len, &avail, &consumed);
252 		if (err)
253 			goto done;
254 		len -= consumed;
255 		if (avail > 0) {
256 			size_t n;
257 			n = fwrite(zb.outbuf, avail, 1, outfile);
258 			if (n != 1) {
259 				err = got_ferror(outfile, GOT_ERR_IO);
260 				goto done;
261 			}
262 			if (csum)
263 				csum_output(csum, zb.outbuf, avail);
264 			*outlen += avail;
265 		}
266 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
267 
268 done:
269 	got_deflate_end(&zb);
270 	return err;
271 }
272 
273 const struct got_error *
274 got_deflate_to_file_mmap(off_t *outlen, uint8_t *map, size_t offset,
275     size_t len, FILE *outfile, struct got_deflate_checksum *csum)
276 {
277 	const struct got_error *err;
278 	size_t avail, consumed;
279 	struct got_deflate_buf zb;
280 
281 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
282 	if (err)
283 		goto done;
284 
285 	*outlen = 0;
286 	do {
287 		err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
288 		    &consumed);
289 		if (err)
290 			goto done;
291 		offset += consumed;
292 		len -= consumed;
293 		if (avail > 0) {
294 			size_t n;
295 			n = fwrite(zb.outbuf, avail, 1, outfile);
296 			if (n != 1) {
297 				err = got_ferror(outfile, GOT_ERR_IO);
298 				goto done;
299 			}
300 			if (csum)
301 				csum_output(csum, zb.outbuf, avail);
302 			*outlen += avail;
303 		}
304 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
305 
306 done:
307 	got_deflate_end(&zb);
308 	return err;
309 }
310 
311 const struct got_error *
312 got_deflate_append_to_file_mmap(struct got_deflate_buf *zb, off_t *outlen,
313     uint8_t *map, size_t offset, size_t len, FILE *outfile,
314     struct got_deflate_checksum *csum)
315 {
316 	const struct got_error *err;
317 	size_t avail, consumed;
318 
319 	do {
320 		err = deflate_read_mmap(zb, map, offset, len, &avail,
321 		    &consumed, 0);
322 		if (err)
323 			break;
324 		offset += consumed;
325 		len -= consumed;
326 		if (avail > 0) {
327 			size_t n;
328 			n = fwrite(zb->outbuf, avail, 1, outfile);
329 			if (n != 1) {
330 				err = got_ferror(outfile, GOT_ERR_IO);
331 				break;
332 			}
333 			if (csum)
334 				csum_output(csum, zb->outbuf, avail);
335 			if (outlen)
336 				*outlen += avail;
337 		}
338 	} while ((zb->flags & GOT_DEFLATE_F_HAVE_MORE) && len > 0);
339 
340 	return err;
341 }
342 
343 const struct got_error *
344 got_deflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
345     size_t *consumed_total, struct got_deflate_checksum *csum, uint8_t *map,
346     size_t offset, size_t len)
347 {
348 	const struct got_error *err;
349 	size_t avail, consumed;
350 	struct got_deflate_buf zb;
351 	void *newbuf;
352 	size_t nbuf = 1;
353 
354 	if (outbuf) {
355 		*outbuf = malloc(GOT_DEFLATE_BUFSIZE);
356 		if (*outbuf == NULL)
357 			return got_error_from_errno("malloc");
358 		err = got_deflate_init(&zb, *outbuf, GOT_DEFLATE_BUFSIZE);
359 		if (err) {
360 			free(*outbuf);
361 			*outbuf = NULL;
362 			return err;
363 		}
364 	} else {
365 		err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
366 		if (err)
367 			return err;
368 	}
369 
370 	*outlen = 0;
371 	if (consumed_total)
372 		*consumed_total = 0;
373 	do {
374 		err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
375 		    &consumed);
376 		if (err)
377 			goto done;
378 		offset += consumed;
379 		if (consumed_total)
380 			*consumed_total += consumed;
381 		len -= consumed;
382 		if (avail > 0 && csum)
383 			csum_output(csum, zb.outbuf, avail);
384 		*outlen += avail;
385 		if ((zb.flags & GOT_DEFLATE_F_HAVE_MORE) && outbuf != NULL) {
386 			newbuf = reallocarray(*outbuf, ++nbuf,
387 			    GOT_DEFLATE_BUFSIZE);
388 			if (newbuf == NULL) {
389 				err = got_error_from_errno("reallocarray");
390 				free(*outbuf);
391 				*outbuf = NULL;
392 				*outlen = 0;
393 				goto done;
394 			}
395 			*outbuf = newbuf;
396 			zb.outbuf = newbuf + *outlen;
397 			zb.outlen = (nbuf * GOT_DEFLATE_BUFSIZE) - *outlen;
398 		}
399 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
400 done:
401 	got_deflate_end(&zb);
402 	return err;
403 }