Gotweb

Blob

Date:: Sun Feb 13 00:43:46 2022 UTC
Message:: reuse existing deltas when creating pack files tested by thomas, naddy, and myself
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/uio.h>
19 #include <sys/time.h>
20 #include <sys/mman.h>
21 
22 #include <limits.h>
23 #include <signal.h>
24 #include <stdint.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <unistd.h>
29 #include <zlib.h>
30 
31 #include "got_compat.h"
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_path.h"
36 
37 #include "got_lib_delta.h"
38 #include "got_lib_delta_cache.h"
39 #include "got_lib_object.h"
40 #include "got_lib_object_cache.h"
41 #include "got_lib_object_parse.h"
42 #include "got_lib_privsep.h"
43 #include "got_lib_pack.h"
44 
45 static volatile sig_atomic_t sigint_received;
46 
47 static void
48 catch_sigint(int signo)
49 {
50 	sigint_received = 1;
51 }
52 
53 static const struct got_error *
54 open_object(struct got_object **obj, struct got_pack *pack,
55     struct got_packidx *packidx, int idx, struct got_object_id *id,
56     struct got_object_cache *objcache)
57 {
58 	const struct got_error *err;
59 
60 	err = got_packfile_open_object(obj, pack, packidx, idx, id);
61 	if (err)
62 		return err;
63 	(*obj)->refcnt++;
64 
65 	err = got_object_cache_add(objcache, id, *obj);
66 	if (err) {
67 		if (err->code == GOT_ERR_OBJ_EXISTS ||
68 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
69 			err = NULL;
70 		return err;
71 	}
72 	(*obj)->refcnt++;
73 	return NULL;
74 }
75 
76 static const struct got_error *
77 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
78     struct got_packidx *packidx, struct got_object_cache *objcache)
79 {
80 	const struct got_error *err = NULL;
81 	struct got_imsg_packed_object iobj;
82 	struct got_object *obj;
83 	struct got_object_id id;
84 	size_t datalen;
85 
86 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
87 	if (datalen != sizeof(iobj))
88 		return got_error(GOT_ERR_PRIVSEP_LEN);
89 	memcpy(&iobj, imsg->data, sizeof(iobj));
90 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
91 
92 	obj = got_object_cache_get(objcache, &id);
93 	if (obj) {
94 		obj->refcnt++;
95 	} else {
96 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
97 		    objcache);
98 		if (err)
99 			goto done;
100 	}
101 
102 	err = got_privsep_send_obj(ibuf, obj);
103 done:
104 	got_object_close(obj);
105 	return err;
106 }
107 
108 static const struct got_error *
109 open_commit(struct got_commit_object **commit, struct got_pack *pack,
110     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
111     struct got_object_cache *objcache)
112 {
113 	const struct got_error *err = NULL;
114 	struct got_object *obj = NULL;
115 	uint8_t *buf = NULL;
116 	size_t len;
117 
118 	*commit = NULL;
119 
120 	obj = got_object_cache_get(objcache, id);
121 	if (obj) {
122 		obj->refcnt++;
123 	} else {
124 		err = open_object(&obj, pack, packidx, obj_idx, id,
125 		    objcache);
126 		if (err)
127 			return err;
128 	}
129 
130 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
131 	if (err)
132 		goto done;
133 
134 	obj->size = len;
135 
136 	err = got_object_parse_commit(commit, buf, len);
137 done:
138 	got_object_close(obj);
139 	free(buf);
140 	return err;
141 }
142 
143 static const struct got_error *
144 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
145     struct got_packidx *packidx, struct got_object_cache *objcache)
146 {
147 	const struct got_error *err = NULL;
148 	struct got_imsg_packed_object iobj;
149 	struct got_commit_object *commit = NULL;
150 	struct got_object_id id;
151 	size_t datalen;
152 
153 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
154 	if (datalen != sizeof(iobj))
155 		return got_error(GOT_ERR_PRIVSEP_LEN);
156 	memcpy(&iobj, imsg->data, sizeof(iobj));
157 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
158 
159 	err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
160 	if (err)
161 		goto done;
162 
163 	err = got_privsep_send_commit(ibuf, commit);
164 done:
165 	if (commit)
166 		got_object_commit_close(commit);
167 	if (err) {
168 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
169 			err = NULL;
170 		else
171 			got_privsep_send_error(ibuf, err);
172 	}
173 
174 	return err;
175 }
176 
177 static const struct got_error *
178 open_tree(uint8_t **buf, struct got_pathlist_head *entries, int *nentries,
179     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
180     struct got_object_id *id, struct got_object_cache *objcache)
181 {
182 	const struct got_error *err = NULL;
183 	struct got_object *obj = NULL;
184 	size_t len;
185 
186 	*buf = NULL;
187 	*nentries = 0;
188 
189 	obj = got_object_cache_get(objcache, id);
190 	if (obj) {
191 		obj->refcnt++;
192 	} else {
193 		err = open_object(&obj, pack, packidx, obj_idx, id,
194 		    objcache);
195 		if (err)
196 			return err;
197 	}
198 
199 	err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
200 	if (err)
201 		goto done;
202 
203 	obj->size = len;
204 
205 	err = got_object_parse_tree(entries, nentries, *buf, len);
206 done:
207 	got_object_close(obj);
208 	if (err) {
209 		free(*buf);
210 		*buf = NULL;
211 	}
212 	return err;
213 }
214 
215 static const struct got_error *
216 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
217     struct got_packidx *packidx, struct got_object_cache *objcache)
218 {
219 	const struct got_error *err = NULL;
220 	struct got_imsg_packed_object iobj;
221 	struct got_pathlist_head entries;
222 	int nentries = 0;
223 	uint8_t *buf = NULL;
224 	struct got_object_id id;
225 	size_t datalen;
226 
227 	TAILQ_INIT(&entries);
228 
229 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
230 	if (datalen != sizeof(iobj))
231 		return got_error(GOT_ERR_PRIVSEP_LEN);
232 	memcpy(&iobj, imsg->data, sizeof(iobj));
233 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
234 
235 	err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
236 	    &id, objcache);
237 	if (err)
238 		return err;
239 
240 	err = got_privsep_send_tree(ibuf, &entries, nentries);
241 	got_object_parsed_tree_entries_free(&entries);
242 	free(buf);
243 	if (err) {
244 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
245 			err = NULL;
246 		else
247 			got_privsep_send_error(ibuf, err);
248 	}
249 
250 	return err;
251 }
252 
253 static const struct got_error *
254 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
255 {
256 	const struct got_error *err;
257 	struct imsg imsg;
258 	size_t datalen;
259 
260 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
261 	if (err)
262 		return err;
263 
264 	if (imsg.hdr.type != imsg_code) {
265 		err = got_error(GOT_ERR_PRIVSEP_MSG);
266 		goto done;
267 	}
268 
269 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
270 	if (datalen != 0) {
271 		err = got_error(GOT_ERR_PRIVSEP_LEN);
272 		goto done;
273 	}
274 	if (imsg.fd == -1) {
275 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
276 		goto done;
277 	}
278 
279 	*f = fdopen(imsg.fd, "w+");
280 	if (*f == NULL) {
281 		err = got_error_from_errno("fdopen");
282 		close(imsg.fd);
283 		goto done;
284 	}
285 done:
286 	imsg_free(&imsg);
287 	return err;
288 }
289 
290 static const struct got_error *
291 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
292     struct imsgbuf *ibuf)
293 {
294 	size_t datalen;
295 
296 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
297 	if (datalen != 0)
298 		return got_error(GOT_ERR_PRIVSEP_LEN);
299 
300 	if (imsg->fd == -1)
301 		return got_error(GOT_ERR_PRIVSEP_NO_FD);
302 
303 	*f = fdopen(imsg->fd, mode);
304 	if (*f == NULL)
305 		return got_error_from_errno("fdopen");
306 	imsg->fd = -1;
307 
308 	return NULL;
309 }
310 
311 static const struct got_error *
312 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
313     struct got_packidx *packidx, struct got_object_cache *objcache,
314     FILE *basefile, FILE *accumfile)
315 {
316 	const struct got_error *err = NULL;
317 	struct got_imsg_packed_object iobj;
318 	struct got_object *obj = NULL;
319 	FILE *outfile = NULL;
320 	struct got_object_id id;
321 	size_t datalen;
322 	uint64_t blob_size;
323 	uint8_t *buf = NULL;
324 
325 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
326 	if (datalen != sizeof(iobj))
327 		return got_error(GOT_ERR_PRIVSEP_LEN);
328 	memcpy(&iobj, imsg->data, sizeof(iobj));
329 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
330 
331 	obj = got_object_cache_get(objcache, &id);
332 	if (obj) {
333 		obj->refcnt++;
334 	} else {
335 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
336 		    objcache);
337 		if (err)
338 			return err;
339 	}
340 
341 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
342 	if (err)
343 		goto done;
344 
345 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
346 		err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
347 		if (err)
348 			goto done;
349 	} else
350 		blob_size = obj->size;
351 
352 	if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
353 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
354 		    obj, pack);
355 	else
356 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
357 		    accumfile);
358 	if (err)
359 		goto done;
360 
361 	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
362 done:
363 	free(buf);
364 	if (outfile && fclose(outfile) == EOF && err == NULL)
365 		err = got_error_from_errno("fclose");
366 	got_object_close(obj);
367 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
368 		got_privsep_send_error(ibuf, err);
369 
370 	return err;
371 }
372 
373 static const struct got_error *
374 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
375     struct got_packidx *packidx, struct got_object_cache *objcache)
376 {
377 	const struct got_error *err = NULL;
378 	struct got_imsg_packed_object iobj;
379 	struct got_object *obj = NULL;
380 	struct got_tag_object *tag = NULL;
381 	uint8_t *buf = NULL;
382 	size_t len;
383 	struct got_object_id id;
384 	size_t datalen;
385 
386 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
387 	if (datalen != sizeof(iobj))
388 		return got_error(GOT_ERR_PRIVSEP_LEN);
389 	memcpy(&iobj, imsg->data, sizeof(iobj));
390 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
391 
392 	obj = got_object_cache_get(objcache, &id);
393 	if (obj) {
394 		obj->refcnt++;
395 	} else {
396 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
397 		    objcache);
398 		if (err)
399 			return err;
400 	}
401 
402 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
403 	if (err)
404 		goto done;
405 
406 	obj->size = len;
407 	err = got_object_parse_tag(&tag, buf, len);
408 	if (err)
409 		goto done;
410 
411 	err = got_privsep_send_tag(ibuf, tag);
412 done:
413 	free(buf);
414 	got_object_close(obj);
415 	if (tag)
416 		got_object_tag_close(tag);
417 	if (err) {
418 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
419 			err = NULL;
420 		else
421 			got_privsep_send_error(ibuf, err);
422 	}
423 
424 	return err;
425 }
426 
427 static struct got_parsed_tree_entry *
428 find_entry_by_name(struct got_pathlist_head *entries, int nentries,
429     const char *name, size_t len)
430 {
431 	struct got_pathlist_entry *pe;
432 
433 	/* Note that tree entries are sorted in strncmp() order. */
434 	TAILQ_FOREACH(pe, entries, entry) {
435 		int cmp = strncmp(pe->path, name, len);
436 		if (cmp < 0)
437 			continue;
438 		if (cmp > 0)
439 			break;
440 		if (pe->path[len] == '\0')
441 			return (struct got_parsed_tree_entry *)pe->data;
442 	}
443 	return NULL;
444 }
445 
446 static const struct got_error *
447 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
448     struct got_pathlist_head *entries1, int *nentries1,
449     struct got_pathlist_head *entries2, int *nentries2,
450     const char *path, struct got_pack *pack, struct got_packidx *packidx,
451     struct imsgbuf *ibuf, struct got_object_cache *objcache)
452 {
453 	const struct got_error *err = NULL;
454 	struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
455 	const char *seg, *s;
456 	size_t seglen;
457 
458 	*changed = 0;
459 
460 	/* We not do support comparing the root path. */
461 	if (got_path_is_root_dir(path))
462 		return got_error_path(path, GOT_ERR_BAD_PATH);
463 
464 	s = path;
465 	while (*s == '/')
466 		s++;
467 	seg = s;
468 	seglen = 0;
469 	while (*s) {
470 		if (*s != '/') {
471 			s++;
472 			seglen++;
473 			if (*s)
474 				continue;
475 		}
476 
477 		pte1 = find_entry_by_name(entries1, *nentries1, seg, seglen);
478 		if (pte1 == NULL) {
479 			err = got_error(GOT_ERR_NO_OBJ);
480 			break;
481 		}
482 
483 		pte2 = find_entry_by_name(entries2, *nentries2, seg, seglen);
484 		if (pte2 == NULL) {
485 			*changed = 1;
486 			break;
487 		}
488 
489 		if (pte1->mode != pte2->mode) {
490 			*changed = 1;
491 			break;
492 		}
493 
494 		if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
495 			*changed = 0;
496 			break;
497 		}
498 
499 		if (*s == '\0') { /* final path element */
500 			*changed = 1;
501 			break;
502 		}
503 
504 		seg = s + 1;
505 		s++;
506 		seglen = 0;
507 		if (*s) {
508 			struct got_object_id id1, id2;
509 			int idx;
510 
511 			memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
512 			idx = got_packidx_get_object_idx(packidx, &id1);
513 			if (idx == -1) {
514 				err = got_error_no_obj(&id1);
515 				break;
516 			}
517 			got_object_parsed_tree_entries_free(entries1);
518 			*nentries1 = 0;
519 			free(*buf1);
520 			*buf1 = NULL;
521 			err = open_tree(buf1, entries1, nentries1, pack,
522 			    packidx, idx, &id1, objcache);
523 			pte1 = NULL;
524 			if (err)
525 				break;
526 
527 			memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
528 			idx = got_packidx_get_object_idx(packidx, &id2);
529 			if (idx == -1) {
530 				err = got_error_no_obj(&id2);
531 				break;
532 			}
533 			got_object_parsed_tree_entries_free(entries2);
534 			*nentries2 = 0;
535 			free(*buf2);
536 			*buf2 = NULL;
537 			err = open_tree(buf2, entries2, nentries2, pack,
538 			    packidx, idx, &id2, objcache);
539 			pte2 = NULL;
540 			if (err)
541 				break;
542 		}
543 	}
544 
545 	return err;
546 }
547 
548 static const struct got_error *
549 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
550     struct imsgbuf *ibuf)
551 {
552 	const struct got_error *err;
553 	struct ibuf *wbuf;
554 	size_t i;
555 
556 	wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
557 	    sizeof(struct got_imsg_traversed_commits) +
558 	    ncommits * SHA1_DIGEST_LENGTH);
559 	if (wbuf == NULL)
560 		return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
561 
562 	if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1) {
563 		err = got_error_from_errno("imsg_add TRAVERSED_COMMITS");
564 		ibuf_free(wbuf);
565 		return err;
566 	}
567 	for (i = 0; i < ncommits; i++) {
568 		struct got_object_id *id = &commit_ids[i];
569 		if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
570 			err = got_error_from_errno(
571 			    "imsg_add TRAVERSED_COMMITS");
572 			ibuf_free(wbuf);
573 			return err;
574 		}
575 	}
576 
577 	wbuf->fd = -1;
578 	imsg_close(ibuf, wbuf);
579 
580 	return got_privsep_flush_imsg(ibuf);
581 }
582 
583 static const struct got_error *
584 send_commit_traversal_done(struct imsgbuf *ibuf)
585 {
586 	if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
587 	    NULL, 0) == -1)
588 		return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
589 
590 	return got_privsep_flush_imsg(ibuf);
591 }
592 
593 
594 static const struct got_error *
595 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
596     struct got_pack *pack, struct got_packidx *packidx,
597     struct got_object_cache *objcache)
598 {
599 	const struct got_error *err = NULL;
600 	struct got_imsg_packed_object iobj;
601 	struct got_object_qid *pid;
602 	struct got_commit_object *commit = NULL, *pcommit = NULL;
603 	struct got_pathlist_head entries, pentries;
604 	int nentries = 0, pnentries = 0;
605 	struct got_object_id id;
606 	size_t datalen, path_len;
607 	char *path = NULL;
608 	const int min_alloc = 64;
609 	int changed = 0, ncommits = 0, nallocated = 0;
610 	struct got_object_id *commit_ids = NULL;
611 
612 	TAILQ_INIT(&entries);
613 	TAILQ_INIT(&pentries);
614 
615 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
616 	if (datalen < sizeof(iobj))
617 		return got_error(GOT_ERR_PRIVSEP_LEN);
618 	memcpy(&iobj, imsg->data, sizeof(iobj));
619 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
620 
621 	path_len = datalen - sizeof(iobj) - 1;
622 	if (path_len < 0)
623 		return got_error(GOT_ERR_PRIVSEP_LEN);
624 	if (path_len > 0) {
625 		path = imsg->data + sizeof(iobj);
626 		if (path[path_len] != '\0')
627 			return got_error(GOT_ERR_PRIVSEP_LEN);
628 	}
629 
630 	nallocated = min_alloc;
631 	commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
632 	if (commit_ids == NULL)
633 		return got_error_from_errno("reallocarray");
634 
635 	do {
636 		const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
637 		int idx;
638 
639 		if (sigint_received) {
640 			err = got_error(GOT_ERR_CANCELLED);
641 			goto done;
642 		}
643 
644 		if (commit == NULL) {
645 			idx = got_packidx_get_object_idx(packidx, &id);
646 			if (idx == -1)
647 				break;
648 			err = open_commit(&commit, pack, packidx,
649 			    idx, &id, objcache);
650 			if (err) {
651 				if (err->code != GOT_ERR_NO_OBJ)
652 					goto done;
653 				err = NULL;
654 				break;
655 			}
656 		}
657 
658 		if (sizeof(struct got_imsg_traversed_commits) +
659 		    ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
660 			err = send_traversed_commits(commit_ids, ncommits,
661 			    ibuf);
662 			if (err)
663 				goto done;
664 			ncommits = 0;
665 		}
666 		ncommits++;
667 		if (ncommits > nallocated) {
668 			struct got_object_id *new;
669 			nallocated += min_alloc;
670 			new = reallocarray(commit_ids, nallocated,
671 			    sizeof(*commit_ids));
672 			if (new == NULL) {
673 				err = got_error_from_errno("reallocarray");
674 				goto done;
675 			}
676 			commit_ids = new;
677 		}
678 		memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
679 		    SHA1_DIGEST_LENGTH);
680 
681 		pid = STAILQ_FIRST(&commit->parent_ids);
682 		if (pid == NULL)
683 			break;
684 
685 		idx = got_packidx_get_object_idx(packidx, pid->id);
686 		if (idx == -1)
687 			break;
688 
689 		err = open_commit(&pcommit, pack, packidx, idx, pid->id,
690 		    objcache);
691 		if (err) {
692 			if (err->code != GOT_ERR_NO_OBJ)
693 				goto done;
694 			err = NULL;
695 			break;
696 		}
697 
698 		if (path[0] == '/' && path[1] == '\0') {
699 			if (got_object_id_cmp(pcommit->tree_id,
700 			    commit->tree_id) != 0) {
701 				changed = 1;
702 				break;
703 			}
704 		} else {
705 			int pidx;
706 			uint8_t *buf = NULL, *pbuf = NULL;
707 
708 			idx = got_packidx_get_object_idx(packidx,
709 			    commit->tree_id);
710 			if (idx == -1)
711 				break;
712 			pidx = got_packidx_get_object_idx(packidx,
713 			    pcommit->tree_id);
714 			if (pidx == -1)
715 				break;
716 
717 			err = open_tree(&buf, &entries, &nentries, pack,
718 			    packidx, idx, commit->tree_id, objcache);
719 			if (err)
720 				goto done;
721 			err = open_tree(&pbuf, &pentries, &pnentries, pack,
722 			    packidx, pidx, pcommit->tree_id, objcache);
723 			if (err) {
724 				free(buf);
725 				goto done;
726 			}
727 
728 			err = tree_path_changed(&changed, &buf, &pbuf,
729 			    &entries, &nentries, &pentries, &pnentries, path,
730 			    pack, packidx, ibuf, objcache);
731 
732 			got_object_parsed_tree_entries_free(&entries);
733 			nentries = 0;
734 			free(buf);
735 			got_object_parsed_tree_entries_free(&pentries);
736 			pnentries = 0;
737 			free(pbuf);
738 			if (err) {
739 				if (err->code != GOT_ERR_NO_OBJ)
740 					goto done;
741 				err = NULL;
742 				break;
743 			}
744 		}
745 
746 		if (!changed) {
747 			memcpy(id.sha1, pid->id->sha1, SHA1_DIGEST_LENGTH);
748 			got_object_commit_close(commit);
749 			commit = pcommit;
750 			pcommit = NULL;
751 		}
752 	} while (!changed);
753 
754 	if (ncommits > 0) {
755 		err = send_traversed_commits(commit_ids, ncommits, ibuf);
756 		if (err)
757 			goto done;
758 
759 		if (changed) {
760 			err = got_privsep_send_commit(ibuf, commit);
761 			if (err)
762 				goto done;
763 		}
764 	}
765 	err = send_commit_traversal_done(ibuf);
766 done:
767 	free(commit_ids);
768 	if (commit)
769 		got_object_commit_close(commit);
770 	if (pcommit)
771 		got_object_commit_close(pcommit);
772 	if (nentries != 0)
773 		got_object_parsed_tree_entries_free(&entries);
774 	if (pnentries != 0)
775 		got_object_parsed_tree_entries_free(&pentries);
776 	if (err) {
777 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
778 			err = NULL;
779 		else
780 			got_privsep_send_error(ibuf, err);
781 	}
782 
783 	return err;
784 }
785 
786 static const struct got_error *
787 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
788     struct got_pack *pack, struct got_packidx *packidx,
789     struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
790 {
791 	const struct got_error *err = NULL;
792 	uint8_t *buf = NULL;
793 	uint64_t size = 0;
794 	FILE *outfile = NULL;
795 	struct got_imsg_packed_object iobj;
796 	struct got_object *obj;
797 	struct got_object_id id;
798 	size_t datalen;
799 
800 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
801 	if (datalen != sizeof(iobj))
802 		return got_error(GOT_ERR_PRIVSEP_LEN);
803 	memcpy(&iobj, imsg->data, sizeof(iobj));
804 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
805 
806 	obj = got_object_cache_get(objcache, &id);
807 	if (obj) {
808 		obj->refcnt++;
809 	} else {
810 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
811 		    objcache);
812 		if (err)
813 			return err;
814 	}
815 
816 	err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
817 	if (err)
818 		return err;
819 
820 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
821 		err = got_pack_get_max_delta_object_size(&size, obj, pack);
822 		if (err)
823 			goto done;
824 	} else
825 		size = obj->size;
826 
827 	if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
828 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
829 		    obj, pack);
830 	else
831 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
832 		    accumfile);
833 	if (err)
834 		goto done;
835 
836 	err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
837 done:
838 	free(buf);
839 	if (outfile && fclose(outfile) == EOF && err == NULL)
840 		err = got_error_from_errno("fclose");
841 	got_object_close(obj);
842 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
843 		got_privsep_send_error(ibuf, err);
844 
845 	return err;
846 }
847 
848 static const struct got_error *
849 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
850     off_t base_offset)
851 {
852 	const struct got_error *err;
853 	int idx;
854 
855 	err = got_packidx_get_offset_idx(&idx, packidx, base_offset); 
856 	if (err)
857 		return err;
858 	if (idx == -1)
859 		return got_error(GOT_ERR_BAD_PACKIDX);
860 
861 	return got_packidx_get_object_id(base_id, packidx, idx);
862 }
863 
864 static const struct got_error *
865 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
866     FILE *delta_outfile, struct got_pack *pack,
867     struct got_packidx *packidx)
868 {
869 	const struct got_error *err = NULL;
870 	struct got_imsg_raw_delta_request req;
871 	size_t datalen, delta_size;
872 	off_t delta_offset;
873 	uint8_t *delta_buf = NULL;
874 	struct got_object_id id, base_id;
875 	off_t base_offset, delta_out_offset = 0;
876 	uint64_t base_size = 0, result_size = 0;
877 	size_t w;
878 
879 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
880 	if (datalen != sizeof(req))
881 		return got_error(GOT_ERR_PRIVSEP_LEN);
882 	memcpy(&req, imsg->data, sizeof(req));
883 	memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
884 
885 	imsg->fd = -1;
886 
887 	err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
888 	    &delta_offset, &base_offset, &base_id, &base_size, &result_size,
889 	    pack, packidx, req.idx);
890 	if (err)
891 		goto done;
892 
893 	/*
894 	 * If this is an offset delta we must determine the base
895 	 * object ID ourselves.
896 	 */
897 	if (base_offset != 0) {
898 		err = get_base_object_id(&base_id, packidx, base_offset);
899 		if (err)
900 			goto done;
901 	}
902 
903 	delta_out_offset = ftello(delta_outfile);
904 	w = fwrite(delta_buf, 1, delta_size, delta_outfile);
905 	if (w != delta_size) {
906 		err = got_ferror(delta_outfile, GOT_ERR_IO);
907 		goto done;
908 	}
909 	if (fflush(delta_outfile) == -1) {
910 		err = got_error_from_errno("fflush");
911 		goto done;
912 	}
913 
914 	err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
915 	    delta_size, delta_offset, delta_out_offset, &base_id);
916 done:
917 	free(delta_buf);
918 	return err;
919 }
920 
921 static const struct got_error *
922 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
923 {
924 	const struct got_error *err = NULL;
925 	struct imsg imsg;
926 	struct got_imsg_packidx ipackidx;
927 	size_t datalen;
928 	struct got_packidx *p;
929 
930 	*packidx = NULL;
931 
932 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
933 	if (err)
934 		return err;
935 
936 	p = calloc(1, sizeof(*p));
937 	if (p == NULL) {
938 		err = got_error_from_errno("calloc");
939 		goto done;
940 	}
941 
942 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
943 		err = got_error(GOT_ERR_PRIVSEP_MSG);
944 		goto done;
945 	}
946 
947 	if (imsg.fd == -1) {
948 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
949 		goto done;
950 	}
951 
952 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
953 	if (datalen != sizeof(ipackidx)) {
954 		err = got_error(GOT_ERR_PRIVSEP_LEN);
955 		goto done;
956 	}
957 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
958 
959 	p->len = ipackidx.len;
960 	p->fd = dup(imsg.fd);
961 	if (p->fd == -1) {
962 		err = got_error_from_errno("dup");
963 		goto done;
964 	}
965 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
966 		err = got_error_from_errno("lseek");
967 		goto done;
968 	}
969 
970 #ifndef GOT_PACK_NO_MMAP
971 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
972 	if (p->map == MAP_FAILED)
973 		p->map = NULL; /* fall back to read(2) */
974 #endif
975 	err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
976 done:
977 	if (err) {
978 		if (imsg.fd != -1)
979 			close(imsg.fd);
980 		got_packidx_close(p);
981 	} else
982 		*packidx = p;
983 	imsg_free(&imsg);
984 	return err;
985 }
986 
987 static const struct got_error *
988 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
989 {
990 	const struct got_error *err = NULL;
991 	struct imsg imsg;
992 	struct got_imsg_pack ipack;
993 	size_t datalen;
994 	struct got_pack *pack;
995 
996 	*packp = NULL;
997 
998 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
999 	if (err)
1000 		return err;
1001 
1002 	pack = calloc(1, sizeof(*pack));
1003 	if (pack == NULL) {
1004 		err = got_error_from_errno("calloc");
1005 		goto done;
1006 	}
1007 
1008 	if (imsg.hdr.type != GOT_IMSG_PACK) {
1009 		err = got_error(GOT_ERR_PRIVSEP_MSG);
1010 		goto done;
1011 	}
1012 
1013 	if (imsg.fd == -1) {
1014 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1015 		goto done;
1016 	}
1017 
1018 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1019 	if (datalen != sizeof(ipack)) {
1020 		err = got_error(GOT_ERR_PRIVSEP_LEN);
1021 		goto done;
1022 	}
1023 	memcpy(&ipack, imsg.data, sizeof(ipack));
1024 
1025 	pack->filesize = ipack.filesize;
1026 	pack->fd = dup(imsg.fd);
1027 	if (pack->fd == -1) {
1028 		err = got_error_from_errno("dup");
1029 		goto done;
1030 	}
1031 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1032 		err = got_error_from_errno("lseek");
1033 		goto done;
1034 	}
1035 	pack->path_packfile = strdup(ipack.path_packfile);
1036 	if (pack->path_packfile == NULL) {
1037 		err = got_error_from_errno("strdup");
1038 		goto done;
1039 	}
1040 
1041 	pack->delta_cache = got_delta_cache_alloc(100,
1042 	    GOT_DELTA_RESULT_SIZE_CACHED_MAX);
1043 	if (pack->delta_cache == NULL) {
1044 		err = got_error_from_errno("got_delta_cache_alloc");
1045 		goto done;
1046 	}
1047 
1048 #ifndef GOT_PACK_NO_MMAP
1049 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1050 	    pack->fd, 0);
1051 	if (pack->map == MAP_FAILED)
1052 		pack->map = NULL; /* fall back to read(2) */
1053 #endif
1054 done:
1055 	if (err) {
1056 		if (imsg.fd != -1)
1057 			close(imsg.fd);
1058 		free(pack);
1059 	} else
1060 		*packp = pack;
1061 	imsg_free(&imsg);
1062 	return err;
1063 }
1064 
1065 int
1066 main(int argc, char *argv[])
1067 {
1068 	const struct got_error *err = NULL;
1069 	struct imsgbuf ibuf;
1070 	struct imsg imsg;
1071 	struct got_packidx *packidx = NULL;
1072 	struct got_pack *pack = NULL;
1073 	struct got_object_cache objcache;
1074 	FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1075 
1076 	//static int attached;
1077 	//while (!attached) sleep(1);
1078 
1079 	signal(SIGINT, catch_sigint);
1080 
1081 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1082 
1083 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1084 	if (err) {
1085 		err = got_error_from_errno("got_object_cache_init");
1086 		got_privsep_send_error(&ibuf, err);
1087 		return 1;
1088 	}
1089 
1090 #ifndef PROFILE
1091 	/* revoke access to most system calls */
1092 	if (pledge("stdio recvfd", NULL) == -1) {
1093 		err = got_error_from_errno("pledge");
1094 		got_privsep_send_error(&ibuf, err);
1095 		return 1;
1096 	}
1097 
1098 	/* revoke fs access */
1099 	if (landlock_no_fs() == -1) {
1100 		err = got_error_from_errno("landlock_no_fs");
1101 		got_privsep_send_error(&ibuf, err);
1102 		return 1;
1103 	}
1104 #endif
1105 
1106 	err = receive_packidx(&packidx, &ibuf);
1107 	if (err) {
1108 		got_privsep_send_error(&ibuf, err);
1109 		return 1;
1110 	}
1111 
1112 	err = receive_pack(&pack, &ibuf);
1113 	if (err) {
1114 		got_privsep_send_error(&ibuf, err);
1115 		return 1;
1116 	}
1117 
1118 	for (;;) {
1119 		imsg.fd = -1;
1120 
1121 		if (sigint_received) {
1122 			err = got_error(GOT_ERR_CANCELLED);
1123 			break;
1124 		}
1125 
1126 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1127 		if (err) {
1128 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
1129 				err = NULL;
1130 			break;
1131 		}
1132 
1133 		if (imsg.hdr.type == GOT_IMSG_STOP)
1134 			break;
1135 
1136 		switch (imsg.hdr.type) {
1137 		case GOT_IMSG_TMPFD:
1138 			if (basefile == NULL) {
1139 				err = receive_tempfile(&basefile, "w+",
1140 				   &imsg, &ibuf);
1141 			} else if (accumfile == NULL) {
1142 				err = receive_tempfile(&accumfile, "w+",
1143 				   &imsg, &ibuf);
1144 			} else
1145 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1146 			break;
1147 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
1148 			err = object_request(&imsg, &ibuf, pack, packidx,
1149 			    &objcache);
1150 			break;
1151 		case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1152 			if (basefile == NULL || accumfile == NULL) {
1153 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1154 				break;
1155 			}
1156 			err = raw_object_request(&imsg, &ibuf, pack, packidx,
1157 			    &objcache, basefile, accumfile);
1158 			break;
1159 		case GOT_IMSG_RAW_DELTA_OUTFD:
1160 			if (delta_outfile != NULL) {
1161 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1162 				break;
1163 			}
1164 			err = receive_tempfile(&delta_outfile, "w",
1165 			    &imsg, &ibuf);
1166 			break;
1167 		case GOT_IMSG_RAW_DELTA_REQUEST:
1168 			if (delta_outfile == NULL) {
1169 				err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1170 				break;
1171 			}
1172 			err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1173 			    pack, packidx);
1174 			break;
1175 		case GOT_IMSG_COMMIT_REQUEST:
1176 			err = commit_request(&imsg, &ibuf, pack, packidx,
1177 			    &objcache);
1178 			break;
1179 		case GOT_IMSG_TREE_REQUEST:
1180 			err = tree_request(&imsg, &ibuf, pack, packidx,
1181 			    &objcache);
1182 			break;
1183 		case GOT_IMSG_BLOB_REQUEST:
1184 			if (basefile == NULL || accumfile == NULL) {
1185 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1186 				break;
1187 			}
1188 			err = blob_request(&imsg, &ibuf, pack, packidx,
1189 			    &objcache, basefile, accumfile);
1190 			break;
1191 		case GOT_IMSG_TAG_REQUEST:
1192 			err = tag_request(&imsg, &ibuf, pack, packidx,
1193 			    &objcache);
1194 			break;
1195 		case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1196 			err = commit_traversal_request(&imsg, &ibuf, pack,
1197 			    packidx, &objcache);
1198 			break;
1199 		default:
1200 			err = got_error(GOT_ERR_PRIVSEP_MSG);
1201 			break;
1202 		}
1203 
1204 		if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1205 			err = got_error_from_errno("close");
1206 		imsg_free(&imsg);
1207 		if (err)
1208 			break;
1209 	}
1210 
1211 	if (packidx)
1212 		got_packidx_close(packidx);
1213 	if (pack)
1214 		got_pack_close(pack);
1215 	got_object_cache_close(&objcache);
1216 	imsg_clear(&ibuf);
1217 	if (basefile && fclose(basefile) == EOF && err == NULL)
1218 		err = got_error_from_errno("fclose");
1219 	if (accumfile && fclose(accumfile) == EOF && err == NULL)
1220 		err = got_error_from_errno("fclose");
1221 	if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1222 		err = got_error_from_errno("fclose");
1223 	if (err) {
1224 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1225 			fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1226 			got_privsep_send_error(&ibuf, err);
1227 		}
1228 	}
1229 	if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1230 		err = got_error_from_errno("close");
1231 	return err ? 1 : 0;
1232 }