Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16 #include "got_compat.h"
18 #include <sys/stat.h>
19 #include <sys/types.h>
20 #include <sys/queue.h>
21 #include <sys/uio.h>
22 #include <sys/time.h>
23 #include <sys/mman.h>
25 #include <inttypes.h>
26 #include <limits.h>
27 #include <signal.h>
28 #include <stdint.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <zlib.h>
35 #include "got_error.h"
36 #include "got_object.h"
37 #include "got_path.h"
39 #include "got_lib_delta.h"
40 #include "got_lib_delta_cache.h"
41 #include "got_lib_object.h"
42 #include "got_lib_object_cache.h"
43 #include "got_lib_object_parse.h"
44 #include "got_lib_object_idset.h"
45 #include "got_lib_privsep.h"
46 #include "got_lib_pack.h"
48 #ifndef nitems
49 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
50 #endif
52 static volatile sig_atomic_t sigint_received;
54 static void
55 catch_sigint(int signo)
56 {
57 sigint_received = 1;
58 }
60 static const struct got_error *
61 open_object(struct got_object **obj, struct got_pack *pack,
62 struct got_packidx *packidx, int idx, struct got_object_id *id,
63 struct got_object_cache *objcache)
64 {
65 const struct got_error *err;
67 err = got_packfile_open_object(obj, pack, packidx, idx, id);
68 if (err)
69 return err;
70 (*obj)->refcnt++;
72 err = got_object_cache_add(objcache, id, *obj);
73 if (err) {
74 if (err->code == GOT_ERR_OBJ_EXISTS ||
75 err->code == GOT_ERR_OBJ_TOO_LARGE)
76 err = NULL;
77 return err;
78 }
79 (*obj)->refcnt++;
80 return NULL;
81 }
83 static const struct got_error *
84 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
85 struct got_packidx *packidx, struct got_object_cache *objcache)
86 {
87 const struct got_error *err = NULL;
88 struct got_imsg_packed_object iobj;
89 struct got_object *obj;
90 struct got_object_id id;
91 size_t datalen;
93 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
94 if (datalen != sizeof(iobj))
95 return got_error(GOT_ERR_PRIVSEP_LEN);
96 memcpy(&iobj, imsg->data, sizeof(iobj));
97 memcpy(&id, &iobj.id, sizeof(id));
99 obj = got_object_cache_get(objcache, &id);
100 if (obj) {
101 obj->refcnt++;
102 } else {
103 err = open_object(&obj, pack, packidx, iobj.idx, &id,
104 objcache);
105 if (err)
106 goto done;
109 err = got_privsep_send_obj(ibuf, obj);
110 done:
111 got_object_close(obj);
112 return err;
115 static const struct got_error *
116 open_commit(struct got_commit_object **commit, struct got_pack *pack,
117 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
118 struct got_object_cache *objcache)
120 const struct got_error *err = NULL;
121 struct got_object *obj = NULL;
122 uint8_t *buf = NULL;
123 size_t len;
125 *commit = NULL;
127 obj = got_object_cache_get(objcache, id);
128 if (obj) {
129 obj->refcnt++;
130 } else {
131 err = open_object(&obj, pack, packidx, obj_idx, id,
132 objcache);
133 if (err)
134 return err;
137 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
138 if (err)
139 goto done;
141 obj->size = len;
143 err = got_object_parse_commit(commit, buf, len);
144 done:
145 got_object_close(obj);
146 free(buf);
147 return err;
150 static const struct got_error *
151 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
152 struct got_packidx *packidx, struct got_object_cache *objcache)
154 const struct got_error *err = NULL;
155 struct got_imsg_packed_object iobj;
156 struct got_commit_object *commit = NULL;
157 struct got_object_id id;
158 size_t datalen;
160 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
161 if (datalen != sizeof(iobj))
162 return got_error(GOT_ERR_PRIVSEP_LEN);
163 memcpy(&iobj, imsg->data, sizeof(iobj));
164 memcpy(&id, &iobj.id, sizeof(id));
166 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
167 if (err)
168 goto done;
170 err = got_privsep_send_commit(ibuf, commit);
171 done:
172 if (commit)
173 got_object_commit_close(commit);
174 if (err) {
175 if (err->code == GOT_ERR_PRIVSEP_PIPE)
176 err = NULL;
177 else
178 got_privsep_send_error(ibuf, err);
181 return err;
184 static const struct got_error *
185 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, size_t *nentries,
186 size_t *nentries_alloc, struct got_pack *pack, struct got_packidx *packidx,
187 int obj_idx, struct got_object_id *id, struct got_object_cache *objcache)
189 const struct got_error *err = NULL;
190 struct got_object *obj = NULL;
191 size_t len;
193 *buf = NULL;
194 *nentries = 0;
196 obj = got_object_cache_get(objcache, id);
197 if (obj) {
198 obj->refcnt++;
199 } else {
200 err = open_object(&obj, pack, packidx, obj_idx, id,
201 objcache);
202 if (err)
203 return err;
206 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
207 if (err)
208 goto done;
210 obj->size = len;
212 err = got_object_parse_tree(entries, nentries, nentries_alloc,
213 *buf, len);
214 done:
215 got_object_close(obj);
216 if (err) {
217 free(*buf);
218 *buf = NULL;
220 return err;
223 static const struct got_error *
224 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
225 struct got_packidx *packidx, struct got_object_cache *objcache,
226 struct got_parsed_tree_entry **entries, size_t *nentries,
227 size_t *nentries_alloc)
229 const struct got_error *err = NULL;
230 struct got_imsg_packed_object iobj;
231 uint8_t *buf = NULL;
232 struct got_object_id id;
233 size_t datalen;
235 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
236 if (datalen != sizeof(iobj))
237 return got_error(GOT_ERR_PRIVSEP_LEN);
238 memcpy(&iobj, imsg->data, sizeof(iobj));
239 memcpy(&id, &iobj.id, sizeof(id));
241 err = open_tree(&buf, entries, nentries, nentries_alloc,
242 pack, packidx, iobj.idx, &id, objcache);
243 if (err)
244 return err;
246 err = got_privsep_send_tree(ibuf, *entries, *nentries);
247 free(buf);
248 if (err) {
249 if (err->code == GOT_ERR_PRIVSEP_PIPE)
250 err = NULL;
251 else
252 got_privsep_send_error(ibuf, err);
255 return err;
258 static const struct got_error *
259 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
261 const struct got_error *err;
262 struct imsg imsg;
263 size_t datalen;
265 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
266 if (err)
267 return err;
269 if (imsg.hdr.type != imsg_code) {
270 err = got_error(GOT_ERR_PRIVSEP_MSG);
271 goto done;
274 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
275 if (datalen != 0) {
276 err = got_error(GOT_ERR_PRIVSEP_LEN);
277 goto done;
279 if (imsg.fd == -1) {
280 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
281 goto done;
284 *f = fdopen(imsg.fd, "w+");
285 if (*f == NULL) {
286 err = got_error_from_errno("fdopen");
287 close(imsg.fd);
288 goto done;
290 done:
291 imsg_free(&imsg);
292 return err;
295 static const struct got_error *
296 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
297 struct imsgbuf *ibuf)
299 size_t datalen;
301 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
302 if (datalen != 0)
303 return got_error(GOT_ERR_PRIVSEP_LEN);
305 if (imsg->fd == -1)
306 return got_error(GOT_ERR_PRIVSEP_NO_FD);
308 *f = fdopen(imsg->fd, mode);
309 if (*f == NULL)
310 return got_error_from_errno("fdopen");
311 imsg->fd = -1;
313 return NULL;
316 static const struct got_error *
317 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
318 struct got_packidx *packidx, struct got_object_cache *objcache,
319 FILE *basefile, FILE *accumfile)
321 const struct got_error *err = NULL;
322 struct got_imsg_packed_object iobj;
323 struct got_object *obj = NULL;
324 FILE *outfile = NULL;
325 struct got_object_id id;
326 size_t datalen;
327 uint64_t blob_size;
328 uint8_t *buf = NULL;
330 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
331 if (datalen != sizeof(iobj))
332 return got_error(GOT_ERR_PRIVSEP_LEN);
333 memcpy(&iobj, imsg->data, sizeof(iobj));
334 memcpy(&id, &iobj.id, sizeof(id));
336 obj = got_object_cache_get(objcache, &id);
337 if (obj) {
338 obj->refcnt++;
339 } else {
340 err = open_object(&obj, pack, packidx, iobj.idx, &id,
341 objcache);
342 if (err)
343 return err;
346 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
347 if (err)
348 goto done;
350 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
351 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
352 if (err)
353 goto done;
354 } else
355 blob_size = obj->size;
357 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
358 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
359 obj, pack);
360 else
361 err = got_packfile_extract_object(pack, obj, outfile, basefile,
362 accumfile);
363 if (err)
364 goto done;
366 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
367 done:
368 free(buf);
369 if (outfile && fclose(outfile) == EOF && err == NULL)
370 err = got_error_from_errno("fclose");
371 got_object_close(obj);
372 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
373 got_privsep_send_error(ibuf, err);
375 return err;
378 static const struct got_error *
379 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
380 struct got_packidx *packidx, struct got_object_cache *objcache)
382 const struct got_error *err = NULL;
383 struct got_imsg_packed_object iobj;
384 struct got_object *obj = NULL;
385 struct got_tag_object *tag = NULL;
386 uint8_t *buf = NULL;
387 size_t len;
388 struct got_object_id id;
389 size_t datalen;
391 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
392 if (datalen != sizeof(iobj))
393 return got_error(GOT_ERR_PRIVSEP_LEN);
394 memcpy(&iobj, imsg->data, sizeof(iobj));
395 memcpy(&id, &iobj.id, sizeof(id));
397 obj = got_object_cache_get(objcache, &id);
398 if (obj) {
399 obj->refcnt++;
400 } else {
401 err = open_object(&obj, pack, packidx, iobj.idx, &id,
402 objcache);
403 if (err)
404 return err;
407 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
408 if (err)
409 goto done;
411 obj->size = len;
412 err = got_object_parse_tag(&tag, buf, len);
413 if (err)
414 goto done;
416 err = got_privsep_send_tag(ibuf, tag);
417 done:
418 free(buf);
419 got_object_close(obj);
420 if (tag)
421 got_object_tag_close(tag);
422 if (err) {
423 if (err->code == GOT_ERR_PRIVSEP_PIPE)
424 err = NULL;
425 else
426 got_privsep_send_error(ibuf, err);
429 return err;
432 static struct got_parsed_tree_entry *
433 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
434 const char *name, size_t len)
436 struct got_parsed_tree_entry *pte;
437 int cmp, i;
439 /* Note that tree entries are sorted in strncmp() order. */
440 for (i = 0; i < nentries; i++) {
441 pte = &entries[i];
442 cmp = strncmp(pte->name, name, len);
443 if (cmp < 0)
444 continue;
445 if (cmp > 0)
446 break;
447 if (pte->name[len] == '\0')
448 return pte;
450 return NULL;
453 static const struct got_error *
454 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
455 struct got_parsed_tree_entry **entries1, size_t *nentries1,
456 size_t *nentries_alloc1,
457 struct got_parsed_tree_entry **entries2, size_t *nentries2,
458 size_t *nentries_alloc2,
459 const char *path, struct got_pack *pack, struct got_packidx *packidx,
460 struct imsgbuf *ibuf, struct got_object_cache *objcache)
462 const struct got_error *err = NULL;
463 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
464 const char *seg, *s;
465 size_t seglen;
467 *changed = 0;
469 /* We not do support comparing the root path. */
470 if (got_path_is_root_dir(path))
471 return got_error_path(path, GOT_ERR_BAD_PATH);
473 s = path;
474 while (*s == '/')
475 s++;
476 seg = s;
477 seglen = 0;
478 while (*s) {
479 if (*s != '/') {
480 s++;
481 seglen++;
482 if (*s)
483 continue;
486 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
487 if (pte1 == NULL) {
488 err = got_error(GOT_ERR_NO_OBJ);
489 break;
492 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
493 if (pte2 == NULL) {
494 *changed = 1;
495 break;
498 if (pte1->mode != pte2->mode) {
499 *changed = 1;
500 break;
503 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
504 *changed = 0;
505 break;
508 if (*s == '\0') { /* final path element */
509 *changed = 1;
510 break;
513 seg = s + 1;
514 s++;
515 seglen = 0;
516 if (*s) {
517 struct got_object_id id1, id2;
518 int idx;
520 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
521 idx = got_packidx_get_object_idx(packidx, &id1);
522 if (idx == -1) {
523 err = got_error_no_obj(&id1);
524 break;
526 *nentries1 = 0;
527 free(*buf1);
528 *buf1 = NULL;
529 err = open_tree(buf1, entries1, nentries1,
530 nentries_alloc1, pack, packidx, idx, &id1,
531 objcache);
532 pte1 = NULL;
533 if (err)
534 break;
536 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
537 idx = got_packidx_get_object_idx(packidx, &id2);
538 if (idx == -1) {
539 err = got_error_no_obj(&id2);
540 break;
542 *nentries2 = 0;
543 free(*buf2);
544 *buf2 = NULL;
545 err = open_tree(buf2, entries2, nentries2,
546 nentries_alloc2, pack, packidx, idx, &id2,
547 objcache);
548 pte2 = NULL;
549 if (err)
550 break;
554 return err;
557 static const struct got_error *
558 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
559 struct imsgbuf *ibuf)
561 struct ibuf *wbuf;
562 size_t i;
564 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
565 sizeof(struct got_imsg_traversed_commits) +
566 ncommits * sizeof(commit_ids[0]));
567 if (wbuf == NULL)
568 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
570 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
571 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
573 for (i = 0; i < ncommits; i++) {
574 struct got_object_id *id = &commit_ids[i];
575 if (imsg_add(wbuf, id, sizeof(*id)) == -1) {
576 return got_error_from_errno(
577 "imsg_add TRAVERSED_COMMITS");
581 wbuf->fd = -1;
582 imsg_close(ibuf, wbuf);
584 return got_privsep_flush_imsg(ibuf);
587 static const struct got_error *
588 send_commit_traversal_done(struct imsgbuf *ibuf)
590 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
591 NULL, 0) == -1)
592 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
594 return got_privsep_flush_imsg(ibuf);
597 static const struct got_error *
598 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
599 struct got_pack *pack, struct got_packidx *packidx,
600 struct got_object_cache *objcache)
602 const struct got_error *err = NULL;
603 struct got_imsg_commit_traversal_request ctreq;
604 struct got_object_qid *pid;
605 struct got_commit_object *commit = NULL, *pcommit = NULL;
606 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
607 size_t nentries = 0, nentries_alloc = 0;
608 size_t pnentries = 0, pnentries_alloc = 0;
609 struct got_object_id id;
610 size_t datalen;
611 char *path = NULL;
612 const int min_alloc = 64;
613 int changed = 0, ncommits = 0, nallocated = 0;
614 struct got_object_id *commit_ids = NULL;
616 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
617 if (datalen < sizeof(ctreq))
618 return got_error(GOT_ERR_PRIVSEP_LEN);
619 memcpy(&ctreq, imsg->data, sizeof(ctreq));
620 memcpy(&id, &ctreq.iobj.id, sizeof(id));
622 if (datalen != sizeof(ctreq) + ctreq.path_len)
623 return got_error(GOT_ERR_PRIVSEP_LEN);
624 if (ctreq.path_len == 0)
625 return got_error(GOT_ERR_PRIVSEP_LEN);
627 path = strndup(imsg->data + sizeof(ctreq), ctreq.path_len);
628 if (path == NULL)
629 return got_error_from_errno("strndup");
631 nallocated = min_alloc;
632 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
633 if (commit_ids == NULL)
634 return got_error_from_errno("reallocarray");
636 do {
637 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
638 int idx;
640 if (sigint_received) {
641 err = got_error(GOT_ERR_CANCELLED);
642 goto done;
645 if (commit == NULL) {
646 idx = got_packidx_get_object_idx(packidx, &id);
647 if (idx == -1)
648 break;
649 err = open_commit(&commit, pack, packidx,
650 idx, &id, objcache);
651 if (err) {
652 if (err->code != GOT_ERR_NO_OBJ)
653 goto done;
654 err = NULL;
655 break;
659 if (sizeof(struct got_imsg_traversed_commits) +
660 ncommits * sizeof(commit_ids[0]) >= max_datalen) {
661 err = send_traversed_commits(commit_ids, ncommits,
662 ibuf);
663 if (err)
664 goto done;
665 ncommits = 0;
667 ncommits++;
668 if (ncommits > nallocated) {
669 struct got_object_id *new;
670 nallocated += min_alloc;
671 new = reallocarray(commit_ids, nallocated,
672 sizeof(*commit_ids));
673 if (new == NULL) {
674 err = got_error_from_errno("reallocarray");
675 goto done;
677 commit_ids = new;
679 memcpy(&commit_ids[ncommits - 1], &id, sizeof(id));
681 pid = STAILQ_FIRST(&commit->parent_ids);
682 if (pid == NULL)
683 break;
685 idx = got_packidx_get_object_idx(packidx, &pid->id);
686 if (idx == -1)
687 break;
689 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
690 objcache);
691 if (err) {
692 if (err->code != GOT_ERR_NO_OBJ)
693 goto done;
694 err = NULL;
695 break;
698 if (path[0] == '/' && path[1] == '\0') {
699 if (got_object_id_cmp(pcommit->tree_id,
700 commit->tree_id) != 0) {
701 changed = 1;
702 break;
704 } else {
705 int pidx;
706 uint8_t *buf = NULL, *pbuf = NULL;
708 idx = got_packidx_get_object_idx(packidx,
709 commit->tree_id);
710 if (idx == -1)
711 break;
712 pidx = got_packidx_get_object_idx(packidx,
713 pcommit->tree_id);
714 if (pidx == -1)
715 break;
717 err = open_tree(&buf, &entries, &nentries,
718 &nentries_alloc, pack, packidx, idx,
719 commit->tree_id, objcache);
720 if (err)
721 goto done;
722 err = open_tree(&pbuf, &pentries, &pnentries,
723 &pnentries_alloc, pack, packidx, pidx,
724 pcommit->tree_id, objcache);
725 if (err) {
726 free(buf);
727 goto done;
730 err = tree_path_changed(&changed, &buf, &pbuf,
731 &entries, &nentries, &nentries_alloc,
732 &pentries, &pnentries, &pnentries_alloc,
733 path, pack, packidx, ibuf, objcache);
735 nentries = 0;
736 free(buf);
737 pnentries = 0;
738 free(pbuf);
739 if (err) {
740 if (err->code != GOT_ERR_NO_OBJ)
741 goto done;
742 err = NULL;
743 break;
747 if (!changed) {
748 memcpy(&id, &pid->id, sizeof(id));
749 got_object_commit_close(commit);
750 commit = pcommit;
751 pcommit = NULL;
753 } while (!changed);
755 if (ncommits > 0) {
756 err = send_traversed_commits(commit_ids, ncommits, ibuf);
757 if (err)
758 goto done;
760 if (changed) {
761 err = got_privsep_send_commit(ibuf, commit);
762 if (err)
763 goto done;
766 err = send_commit_traversal_done(ibuf);
767 done:
768 free(path);
769 free(commit_ids);
770 if (commit)
771 got_object_commit_close(commit);
772 if (pcommit)
773 got_object_commit_close(pcommit);
774 free(entries);
775 free(pentries);
776 if (err) {
777 if (err->code == GOT_ERR_PRIVSEP_PIPE)
778 err = NULL;
779 else
780 got_privsep_send_error(ibuf, err);
783 return err;
786 static const struct got_error *
787 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
788 struct got_pack *pack, struct got_packidx *packidx,
789 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
791 const struct got_error *err = NULL;
792 uint8_t *buf = NULL;
793 uint64_t size = 0;
794 FILE *outfile = NULL;
795 struct got_imsg_packed_object iobj;
796 struct got_object *obj;
797 struct got_object_id id;
798 size_t datalen;
800 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
801 if (datalen != sizeof(iobj))
802 return got_error(GOT_ERR_PRIVSEP_LEN);
803 memcpy(&iobj, imsg->data, sizeof(iobj));
804 memcpy(&id, &iobj.id, sizeof(id));
806 obj = got_object_cache_get(objcache, &id);
807 if (obj) {
808 obj->refcnt++;
809 } else {
810 err = open_object(&obj, pack, packidx, iobj.idx, &id,
811 objcache);
812 if (err)
813 return err;
816 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
817 if (err)
818 return err;
820 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
821 err = got_pack_get_max_delta_object_size(&size, obj, pack);
822 if (err)
823 goto done;
824 } else
825 size = obj->size;
827 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
828 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
829 obj, pack);
830 else
831 err = got_packfile_extract_object(pack, obj, outfile, basefile,
832 accumfile);
833 if (err)
834 goto done;
836 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
837 done:
838 free(buf);
839 if (outfile && fclose(outfile) == EOF && err == NULL)
840 err = got_error_from_errno("fclose");
841 got_object_close(obj);
842 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
843 got_privsep_send_error(ibuf, err);
845 return err;
848 static const struct got_error *
849 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
850 off_t base_offset)
852 const struct got_error *err;
853 int idx;
855 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
856 if (err)
857 return err;
858 if (idx == -1)
859 return got_error(GOT_ERR_BAD_PACKIDX);
861 return got_packidx_get_object_id(base_id, packidx, idx);
864 static const struct got_error *
865 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
866 FILE *delta_outfile, struct got_pack *pack,
867 struct got_packidx *packidx)
869 const struct got_error *err = NULL;
870 struct got_imsg_raw_delta_request req;
871 size_t datalen, delta_size, delta_compressed_size;
872 off_t delta_offset, delta_data_offset;
873 uint8_t *delta_buf = NULL;
874 struct got_object_id id, base_id;
875 off_t base_offset, delta_out_offset = 0;
876 uint64_t base_size = 0, result_size = 0;
877 size_t w;
879 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
880 if (datalen != sizeof(req))
881 return got_error(GOT_ERR_PRIVSEP_LEN);
882 memcpy(&req, imsg->data, sizeof(req));
883 memcpy(&id, &req.id, sizeof(id));
885 imsg->fd = -1;
887 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
888 &delta_compressed_size, &delta_offset, &delta_data_offset,
889 &base_offset, &base_id, &base_size, &result_size,
890 pack, packidx, req.idx);
891 if (err)
892 goto done;
894 /*
895 * If this is an offset delta we must determine the base
896 * object ID ourselves.
897 */
898 if (base_offset != 0) {
899 err = get_base_object_id(&base_id, packidx, base_offset);
900 if (err)
901 goto done;
904 delta_out_offset = ftello(delta_outfile);
905 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
906 if (w != delta_compressed_size) {
907 err = got_ferror(delta_outfile, GOT_ERR_IO);
908 goto done;
910 if (fflush(delta_outfile) == -1) {
911 err = got_error_from_errno("fflush");
912 goto done;
915 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
916 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
917 &base_id);
918 done:
919 free(delta_buf);
920 return err;
923 struct search_deltas_arg {
924 struct imsgbuf *ibuf;
925 struct got_packidx *packidx;
926 struct got_pack *pack;
927 struct got_object_idset *idset;
928 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
929 size_t ndeltas;
930 };
932 static const struct got_error *
933 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
935 const struct got_error *err;
936 struct search_deltas_arg *a = arg;
937 int obj_idx;
938 uint8_t *delta_buf = NULL;
939 uint64_t base_size, result_size;
940 size_t delta_size, delta_compressed_size;
941 off_t delta_offset, delta_data_offset, base_offset;
942 struct got_object_id base_id;
944 if (sigint_received)
945 return got_error(GOT_ERR_CANCELLED);
947 obj_idx = got_packidx_get_object_idx(a->packidx, id);
948 if (obj_idx == -1)
949 return NULL; /* object not present in our pack file */
951 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
952 &delta_compressed_size, &delta_offset, &delta_data_offset,
953 &base_offset, &base_id, &base_size, &result_size,
954 a->pack, a->packidx, obj_idx);
955 if (err) {
956 if (err->code == GOT_ERR_OBJ_TYPE)
957 return NULL; /* object not stored as a delta */
958 return err;
961 /*
962 * If this is an offset delta we must determine the base
963 * object ID ourselves.
964 */
965 if (base_offset != 0) {
966 err = get_base_object_id(&base_id, a->packidx, base_offset);
967 if (err)
968 goto done;
971 if (got_object_idset_contains(a->idset, &base_id)) {
972 struct got_imsg_reused_delta *delta;
974 delta = &a->deltas[a->ndeltas++];
975 memcpy(&delta->id, id, sizeof(delta->id));
976 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
977 delta->base_size = base_size;
978 delta->result_size = result_size;
979 delta->delta_size = delta_size;
980 delta->delta_compressed_size = delta_compressed_size;
981 delta->delta_offset = delta_data_offset;
983 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
984 err = got_privsep_send_reused_deltas(a->ibuf,
985 a->deltas, a->ndeltas);
986 if (err)
987 goto done;
988 a->ndeltas = 0;
991 done:
992 free(delta_buf);
993 return err;
996 static const struct got_error *
997 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
999 const struct got_error *err = NULL;
1000 int done = 0;
1001 struct got_object_id *ids;
1002 size_t nids, i;
1004 for (;;) {
1005 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1006 if (err || done)
1007 break;
1008 for (i = 0; i < nids; i++) {
1009 err = got_object_idset_add(idset, &ids[i], NULL);
1010 if (err) {
1011 free(ids);
1012 return err;
1015 free(ids);
1018 return err;
1021 static const struct got_error *
1022 recv_object_id_queue(struct got_object_id_queue *queue,
1023 struct got_object_idset *queued_ids, struct imsgbuf *ibuf)
1025 const struct got_error *err = NULL;
1026 int done = 0;
1027 struct got_object_qid *qid;
1028 struct got_object_id *ids;
1029 size_t nids, i;
1031 for (;;) {
1032 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1033 if (err || done)
1034 break;
1035 for (i = 0; i < nids; i++) {
1036 err = got_object_qid_alloc_partial(&qid);
1037 if (err)
1038 return err;
1039 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1040 STAILQ_INSERT_TAIL(queue, qid, entry);
1041 err = got_object_idset_add(queued_ids, &qid->id, NULL);
1042 if (err)
1043 return err;
1047 return err;
1050 static const struct got_error *
1051 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1052 struct got_pack *pack, struct got_packidx *packidx)
1054 const struct got_error *err = NULL;
1055 struct got_object_idset *idset;
1056 struct search_deltas_arg sda;
1058 idset = got_object_idset_alloc();
1059 if (idset == NULL)
1060 return got_error_from_errno("got_object_idset_alloc");
1062 err = recv_object_ids(idset, ibuf);
1063 if (err)
1064 return err;
1066 memset(&sda, 0, sizeof(sda));
1067 sda.ibuf = ibuf;
1068 sda.idset = idset;
1069 sda.pack = pack;
1070 sda.packidx = packidx;
1071 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1072 if (err)
1073 goto done;
1075 if (sda.ndeltas > 0) {
1076 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1077 sda.ndeltas);
1078 if (err)
1079 goto done;
1082 err = got_privsep_send_reused_deltas_done(ibuf);
1083 done:
1084 got_object_idset_free(idset);
1085 return err;
1088 static const struct got_error *
1089 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1091 const struct got_error *err = NULL;
1092 struct imsg imsg;
1093 struct got_imsg_packidx ipackidx;
1094 size_t datalen;
1095 struct got_packidx *p;
1097 *packidx = NULL;
1099 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1100 if (err)
1101 return err;
1103 p = calloc(1, sizeof(*p));
1104 if (p == NULL) {
1105 err = got_error_from_errno("calloc");
1106 goto done;
1109 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1110 err = got_error(GOT_ERR_PRIVSEP_MSG);
1111 goto done;
1114 if (imsg.fd == -1) {
1115 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1116 goto done;
1119 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1120 if (datalen != sizeof(ipackidx)) {
1121 err = got_error(GOT_ERR_PRIVSEP_LEN);
1122 goto done;
1124 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1126 p->len = ipackidx.len;
1127 p->fd = dup(imsg.fd);
1128 if (p->fd == -1) {
1129 err = got_error_from_errno("dup");
1130 goto done;
1132 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1133 err = got_error_from_errno("lseek");
1134 goto done;
1137 #ifndef GOT_PACK_NO_MMAP
1138 if (p->len > 0 && p->len <= SIZE_MAX) {
1139 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1140 if (p->map == MAP_FAILED)
1141 p->map = NULL; /* fall back to read(2) */
1143 #endif
1144 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1145 done:
1146 if (err) {
1147 if (imsg.fd != -1)
1148 close(imsg.fd);
1149 got_packidx_close(p);
1150 } else
1151 *packidx = p;
1152 imsg_free(&imsg);
1153 return err;
1156 static const struct got_error *
1157 send_tree_enumeration_done(struct imsgbuf *ibuf)
1159 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1160 NULL, 0) == -1)
1161 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1163 return got_privsep_flush_imsg(ibuf);
1166 struct enumerated_tree {
1167 struct got_object_id id;
1168 char *path;
1169 uint8_t *buf;
1170 struct got_parsed_tree_entry *entries;
1171 int nentries;
1174 static const struct got_error *
1175 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1176 struct got_object_id *tree_id,
1177 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1178 struct got_object_cache *objcache, struct got_object_idset *idset,
1179 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1181 const struct got_error *err = NULL;
1182 struct got_object_id_queue ids;
1183 struct got_object_qid *qid;
1184 uint8_t *buf = NULL;
1185 struct got_parsed_tree_entry *entries = NULL;
1186 size_t nentries = 0, nentries_alloc = 0, i;
1187 struct enumerated_tree *tree;
1189 *ntrees = 0;
1190 *have_all_entries = 1;
1191 STAILQ_INIT(&ids);
1193 err = got_object_qid_alloc_partial(&qid);
1194 if (err)
1195 return err;
1196 memcpy(&qid->id, tree_id, sizeof(*tree_id));
1197 qid->data = strdup(path);
1198 if (qid->data == NULL) {
1199 err = got_error_from_errno("strdup");
1200 goto done;
1202 STAILQ_INSERT_TAIL(&ids, qid, entry);
1203 qid = NULL;
1205 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1206 do {
1207 const char *path;
1208 int idx, i;
1210 if (sigint_received) {
1211 err = got_error(GOT_ERR_CANCELLED);
1212 goto done;
1215 qid = STAILQ_FIRST(&ids);
1216 STAILQ_REMOVE_HEAD(&ids, entry);
1217 path = qid->data;
1219 idx = got_packidx_get_object_idx(packidx, &qid->id);
1220 if (idx == -1) {
1221 *have_all_entries = 0;
1222 break;
1225 err = open_tree(&buf, &entries, &nentries, &nentries_alloc,
1226 pack, packidx, idx, &qid->id, objcache);
1227 if (err) {
1228 if (err->code != GOT_ERR_NO_OBJ)
1229 goto done;
1232 err = got_object_idset_add(idset, &qid->id, NULL);
1233 if (err)
1234 goto done;
1236 for (i = 0; i < nentries; i++) {
1237 struct got_object_qid *eqid = NULL;
1238 struct got_parsed_tree_entry *pte = &entries[i];
1239 char *p;
1241 if (!S_ISDIR(pte->mode))
1242 continue;
1244 err = got_object_qid_alloc_partial(&eqid);
1245 if (err)
1246 goto done;
1247 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1249 if (got_object_idset_contains(idset, &eqid->id)) {
1250 got_object_qid_free(eqid);
1251 continue;
1254 if (asprintf(&p, "%s%s%s", path,
1255 got_path_is_root_dir(path) ? "" : "/",
1256 pte->name) == -1) {
1257 err = got_error_from_errno("asprintf");
1258 got_object_qid_free(eqid);
1259 goto done;
1261 eqid->data = p;
1262 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1265 if (*ntrees >= *nalloc) {
1266 struct enumerated_tree *new;
1267 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1268 sizeof(*new));
1269 if (new == NULL) {
1270 err = got_error_from_errno("malloc");
1271 goto done;
1273 *trees = new;
1274 *nalloc += 16;
1276 tree = &(*trees)[*ntrees];
1277 (*ntrees)++;
1278 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1279 tree->path = qid->data;
1280 tree->buf = buf;
1281 buf = NULL;
1282 tree->entries = entries;
1283 entries = NULL;
1284 nentries_alloc = 0;
1285 tree->nentries = nentries;
1286 nentries = 0;
1288 got_object_qid_free(qid);
1289 qid = NULL;
1290 } while (!STAILQ_EMPTY(&ids));
1292 if (*have_all_entries) {
1293 int i;
1295 * We have managed to traverse all entries in the hierarchy.
1296 * Tell the main process what we have found.
1298 for (i = 0; i < *ntrees; i++) {
1299 tree = &(*trees)[i];
1300 err = got_privsep_send_enumerated_tree(totlen,
1301 ibuf, &tree->id, tree->path, tree->entries,
1302 tree->nentries);
1303 if (err)
1304 goto done;
1305 free(tree->buf);
1306 tree->buf = NULL;
1307 free(tree->path);
1308 tree->path = NULL;
1309 free(tree->entries);
1310 tree->entries = NULL;
1312 *ntrees = 0; /* don't loop again below to free memory */
1314 err = send_tree_enumeration_done(ibuf);
1315 } else {
1317 * We can only load fully packed tree hierarchies on
1318 * behalf of the main process, otherwise the main process
1319 * gets a wrong idea about which tree objects have
1320 * already been traversed.
1321 * Indicate a missing entry for the root of this tree.
1322 * The main process should continue by loading this
1323 * entire tree the slow way.
1325 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1326 tree_id, "/", NULL, -1);
1327 if (err)
1328 goto done;
1330 done:
1331 free(buf);
1332 free(entries);
1333 for (i = 0; i < *ntrees; i++) {
1334 tree = &(*trees)[i];
1335 free(tree->buf);
1336 tree->buf = NULL;
1337 free(tree->path);
1338 tree->path = NULL;
1339 free(tree->entries);
1340 tree->entries = NULL;
1342 if (qid)
1343 free(qid->data);
1344 got_object_qid_free(qid);
1345 got_object_id_queue_free(&ids);
1346 if (err) {
1347 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1348 err = NULL;
1349 else
1350 got_privsep_send_error(ibuf, err);
1353 return err;
1356 static const struct got_error *
1357 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1358 struct got_pack *pack, struct got_packidx *packidx,
1359 struct got_object_cache *objcache)
1361 const struct got_error *err = NULL;
1362 struct got_object_id_queue commit_ids;
1363 const struct got_object_id_queue *parents = NULL;
1364 struct got_object_qid *qid = NULL;
1365 struct got_object *obj = NULL;
1366 struct got_commit_object *commit = NULL;
1367 struct got_object_id *tree_id = NULL;
1368 size_t totlen = 0;
1369 struct got_object_idset *idset, *queued_ids = NULL;
1370 int i, idx, have_all_entries = 1;
1371 struct enumerated_tree *trees = NULL;
1372 size_t ntrees = 0, nalloc = 16;
1374 STAILQ_INIT(&commit_ids);
1376 trees = calloc(nalloc, sizeof(*trees));
1377 if (trees == NULL)
1378 return got_error_from_errno("calloc");
1380 idset = got_object_idset_alloc();
1381 if (idset == NULL) {
1382 err = got_error_from_errno("got_object_idset_alloc");
1383 goto done;
1386 queued_ids = got_object_idset_alloc();
1387 if (queued_ids == NULL) {
1388 err = got_error_from_errno("got_object_idset_alloc");
1389 goto done;
1392 err = recv_object_id_queue(&commit_ids, queued_ids, ibuf);
1393 if (err)
1394 goto done;
1396 if (STAILQ_EMPTY(&commit_ids)) {
1397 err = got_error(GOT_ERR_PRIVSEP_MSG);
1398 goto done;
1401 err = recv_object_ids(idset, ibuf);
1402 if (err)
1403 goto done;
1405 while (!STAILQ_EMPTY(&commit_ids)) {
1406 if (sigint_received) {
1407 err = got_error(GOT_ERR_CANCELLED);
1408 goto done;
1411 qid = STAILQ_FIRST(&commit_ids);
1412 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1414 if (got_object_idset_contains(idset, &qid->id)) {
1415 got_object_qid_free(qid);
1416 qid = NULL;
1417 continue;
1420 idx = got_packidx_get_object_idx(packidx, &qid->id);
1421 if (idx == -1) {
1422 have_all_entries = 0;
1423 break;
1426 err = open_object(&obj, pack, packidx, idx, &qid->id,
1427 objcache);
1428 if (err)
1429 goto done;
1430 if (obj->type == GOT_OBJ_TYPE_TAG) {
1431 struct got_tag_object *tag;
1432 uint8_t *buf;
1433 size_t len;
1434 err = got_packfile_extract_object_to_mem(&buf,
1435 &len, obj, pack);
1436 if (err)
1437 goto done;
1438 obj->size = len;
1439 err = got_object_parse_tag(&tag, buf, len);
1440 if (err) {
1441 free(buf);
1442 goto done;
1444 idx = got_packidx_get_object_idx(packidx, &tag->id);
1445 if (idx == -1) {
1446 have_all_entries = 0;
1447 break;
1449 err = open_commit(&commit, pack, packidx, idx,
1450 &tag->id, objcache);
1451 got_object_tag_close(tag);
1452 free(buf);
1453 if (err)
1454 goto done;
1455 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1456 err = open_commit(&commit, pack, packidx, idx,
1457 &qid->id, objcache);
1458 if (err)
1459 goto done;
1460 } else {
1461 err = got_error(GOT_ERR_OBJ_TYPE);
1462 goto done;
1464 got_object_close(obj);
1465 obj = NULL;
1467 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1468 got_object_commit_get_committer_time(commit));
1469 if (err)
1470 goto done;
1472 tree_id = got_object_commit_get_tree_id(commit);
1473 idx = got_packidx_get_object_idx(packidx, tree_id);
1474 if (idx == -1) {
1475 have_all_entries = 0;
1476 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1477 tree_id, "/", NULL, -1);
1478 if (err)
1479 goto done;
1480 break;
1483 if (got_object_idset_contains(idset, tree_id)) {
1484 got_object_qid_free(qid);
1485 qid = NULL;
1486 err = send_tree_enumeration_done(ibuf);
1487 if (err)
1488 goto done;
1489 continue;
1492 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1493 tree_id, "/", pack, packidx, objcache, idset,
1494 &trees, &nalloc, &ntrees);
1495 if (err)
1496 goto done;
1498 if (!have_all_entries)
1499 break;
1501 got_object_qid_free(qid);
1502 qid = NULL;
1504 parents = got_object_commit_get_parent_ids(commit);
1505 if (parents) {
1506 struct got_object_qid *pid;
1507 STAILQ_FOREACH(pid, parents, entry) {
1508 if (got_object_idset_contains(idset, &pid->id))
1509 continue;
1510 if (got_object_idset_contains(queued_ids, &pid->id))
1511 continue;
1512 err = got_object_qid_alloc_partial(&qid);
1513 if (err)
1514 goto done;
1515 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1516 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1517 qid = NULL;
1521 got_object_commit_close(commit);
1522 commit = NULL;
1525 if (have_all_entries) {
1526 err = got_privsep_send_object_enumeration_done(ibuf);
1527 if (err)
1528 goto done;
1529 } else {
1530 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1531 if (err)
1532 goto done;
1534 done:
1535 if (obj)
1536 got_object_close(obj);
1537 if (commit)
1538 got_object_commit_close(commit);
1539 got_object_qid_free(qid);
1540 got_object_id_queue_free(&commit_ids);
1541 if (idset)
1542 got_object_idset_free(idset);
1543 if (queued_ids)
1544 got_object_idset_free(queued_ids);
1545 for (i = 0; i < ntrees; i++) {
1546 struct enumerated_tree *tree = &trees[i];
1547 free(tree->buf);
1548 free(tree->path);
1549 free(tree->entries);
1551 free(trees);
1552 return err;
1555 enum findtwixt_color {
1556 COLOR_KEEP = 0,
1557 COLOR_DROP,
1558 COLOR_SKIP,
1559 COLOR_MAX,
1562 static const struct got_error *
1563 paint_commit(struct got_object_qid *qid, intptr_t color)
1565 if (color < 0 || color >= COLOR_MAX)
1566 return got_error(GOT_ERR_RANGE);
1568 qid->data = (void *)color;
1569 return NULL;
1572 static const struct got_error *
1573 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1574 intptr_t color)
1576 const struct got_error *err;
1577 struct got_object_qid *qid;
1579 err = got_object_qid_alloc_partial(&qid);
1580 if (err)
1581 return err;
1583 memcpy(&qid->id, id, sizeof(qid->id));
1584 STAILQ_INSERT_TAIL(ids, qid, entry);
1585 return paint_commit(qid, color);
1588 static const struct got_error *
1589 paint_commits(struct got_object_id_queue *ids, int *nids,
1590 struct got_object_idset *keep, struct got_object_idset *drop,
1591 struct got_object_idset *skip, struct got_pack *pack,
1592 struct got_packidx *packidx, struct imsgbuf *ibuf,
1593 struct got_object_cache *objcache)
1595 const struct got_error *err = NULL;
1596 struct got_commit_object *commit = NULL;
1597 struct got_object_id_queue painted;
1598 const struct got_object_id_queue *parents;
1599 struct got_object_qid *qid = NULL;
1600 int nqueued = *nids, nskip = 0, npainted = 0;
1602 STAILQ_INIT(&painted);
1604 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1605 int idx;
1606 intptr_t color;
1608 if (sigint_received) {
1609 err = got_error(GOT_ERR_CANCELLED);
1610 goto done;
1613 qid = STAILQ_FIRST(ids);
1614 idx = got_packidx_get_object_idx(packidx, &qid->id);
1615 if (idx == -1) {
1616 qid = NULL;
1617 break;
1620 STAILQ_REMOVE_HEAD(ids, entry);
1621 nqueued--;
1622 color = (intptr_t)qid->data;
1623 if (color == COLOR_SKIP)
1624 nskip--;
1626 if (got_object_idset_contains(skip, &qid->id)) {
1627 got_object_qid_free(qid);
1628 qid = NULL;
1629 continue;
1632 switch (color) {
1633 case COLOR_KEEP:
1634 if (got_object_idset_contains(keep, &qid->id)) {
1635 got_object_qid_free(qid);
1636 qid = NULL;
1637 continue;
1639 if (got_object_idset_contains(drop, &qid->id)) {
1640 err = paint_commit(qid, COLOR_SKIP);
1641 if (err)
1642 goto done;
1644 err = got_object_idset_add(keep, &qid->id, NULL);
1645 if (err)
1646 goto done;
1647 break;
1648 case COLOR_DROP:
1649 if (got_object_idset_contains(drop, &qid->id)) {
1650 got_object_qid_free(qid);
1651 qid = NULL;
1652 continue;
1654 if (got_object_idset_contains(keep, &qid->id)) {
1655 err = paint_commit(qid, COLOR_SKIP);
1656 if (err)
1657 goto done;
1659 err = got_object_idset_add(drop, &qid->id, NULL);
1660 if (err)
1661 goto done;
1662 break;
1663 case COLOR_SKIP:
1664 if (!got_object_idset_contains(skip, &qid->id)) {
1665 err = got_object_idset_add(skip, &qid->id,
1666 NULL);
1667 if (err)
1668 goto done;
1670 break;
1671 default:
1672 /* should not happen */
1673 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1674 "%s invalid commit color %"PRIdPTR, __func__,
1675 color);
1676 goto done;
1679 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1680 objcache);
1681 if (err)
1682 goto done;
1684 parents = got_object_commit_get_parent_ids(commit);
1685 if (parents) {
1686 struct got_object_qid *pid;
1687 color = (intptr_t)qid->data;
1688 STAILQ_FOREACH(pid, parents, entry) {
1689 err = queue_commit_id(ids, &pid->id, color);
1690 if (err)
1691 goto done;
1692 nqueued++;
1693 if (color == COLOR_SKIP)
1694 nskip++;
1698 got_object_commit_close(commit);
1699 commit = NULL;
1701 STAILQ_INSERT_TAIL(&painted, qid, entry);
1702 qid = NULL;
1703 npainted++;
1705 err = got_privsep_send_painted_commits(ibuf, &painted,
1706 &npainted, 1, 0);
1707 if (err)
1708 goto done;
1711 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1712 if (err)
1713 goto done;
1715 *nids = nqueued;
1716 done:
1717 if (commit)
1718 got_object_commit_close(commit);
1719 got_object_qid_free(qid);
1720 return err;
1723 static void
1724 commit_painting_free(struct got_object_idset **keep,
1725 struct got_object_idset **drop,
1726 struct got_object_idset **skip)
1728 if (*keep) {
1729 got_object_idset_free(*keep);
1730 *keep = NULL;
1732 if (*drop) {
1733 got_object_idset_free(*drop);
1734 *drop = NULL;
1736 if (*skip) {
1737 got_object_idset_free(*skip);
1738 *skip = NULL;
1742 static const struct got_error *
1743 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1744 struct got_object_idset **drop, struct got_object_idset **skip)
1746 const struct got_error *err = NULL;
1748 *keep = got_object_idset_alloc();
1749 if (*keep == NULL) {
1750 err = got_error_from_errno("got_object_idset_alloc");
1751 goto done;
1753 *drop = got_object_idset_alloc();
1754 if (*drop == NULL) {
1755 err = got_error_from_errno("got_object_idset_alloc");
1756 goto done;
1758 *skip = got_object_idset_alloc();
1759 if (*skip == NULL) {
1760 err = got_error_from_errno("got_object_idset_alloc");
1761 goto done;
1764 err = recv_object_ids(*keep, ibuf);
1765 if (err)
1766 goto done;
1767 err = recv_object_ids(*drop, ibuf);
1768 if (err)
1769 goto done;
1770 err = recv_object_ids(*skip, ibuf);
1771 if (err)
1772 goto done;
1774 done:
1775 if (err)
1776 commit_painting_free(keep, drop, skip);
1778 return err;
1781 static const struct got_error *
1782 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1783 struct got_pack *pack, struct got_packidx *packidx,
1784 struct got_object_cache *objcache, struct got_object_idset *keep,
1785 struct got_object_idset *drop, struct got_object_idset *skip)
1787 const struct got_error *err = NULL;
1788 struct got_imsg_commit_painting_request ireq;
1789 struct got_object_id id;
1790 size_t datalen;
1791 struct got_object_id_queue ids;
1792 int nids = 0;
1794 STAILQ_INIT(&ids);
1796 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1797 if (datalen != sizeof(ireq))
1798 return got_error(GOT_ERR_PRIVSEP_LEN);
1799 memcpy(&ireq, imsg->data, sizeof(ireq));
1800 memcpy(&id, &ireq.id, sizeof(id));
1802 err = queue_commit_id(&ids, &id, ireq.color);
1803 if (err)
1804 return err;
1805 nids = 1;
1807 err = paint_commits(&ids, &nids, keep, drop, skip,
1808 pack, packidx, ibuf, objcache);
1809 if (err)
1810 goto done;
1812 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1813 if (err)
1814 goto done;
1816 err = got_privsep_send_painting_commits_done(ibuf);
1817 done:
1818 got_object_id_queue_free(&ids);
1819 return err;
1822 static const struct got_error *
1823 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1825 const struct got_error *err = NULL;
1826 struct imsg imsg;
1827 struct got_imsg_pack ipack;
1828 size_t datalen;
1829 struct got_pack *pack;
1831 *packp = NULL;
1833 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1834 if (err)
1835 return err;
1837 pack = calloc(1, sizeof(*pack));
1838 if (pack == NULL) {
1839 err = got_error_from_errno("calloc");
1840 goto done;
1843 if (imsg.hdr.type != GOT_IMSG_PACK) {
1844 err = got_error(GOT_ERR_PRIVSEP_MSG);
1845 goto done;
1848 if (imsg.fd == -1) {
1849 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1850 goto done;
1853 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1854 if (datalen != sizeof(ipack)) {
1855 err = got_error(GOT_ERR_PRIVSEP_LEN);
1856 goto done;
1858 memcpy(&ipack, imsg.data, sizeof(ipack));
1860 pack->filesize = ipack.filesize;
1861 pack->fd = dup(imsg.fd);
1862 if (pack->fd == -1) {
1863 err = got_error_from_errno("dup");
1864 goto done;
1866 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1867 err = got_error_from_errno("lseek");
1868 goto done;
1870 pack->path_packfile = strdup(ipack.path_packfile);
1871 if (pack->path_packfile == NULL) {
1872 err = got_error_from_errno("strdup");
1873 goto done;
1876 err = got_delta_cache_alloc(&pack->delta_cache);
1877 if (err)
1878 goto done;
1880 #ifndef GOT_PACK_NO_MMAP
1881 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1882 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1883 pack->fd, 0);
1884 if (pack->map == MAP_FAILED)
1885 pack->map = NULL; /* fall back to read(2) */
1887 #endif
1888 done:
1889 if (err) {
1890 if (imsg.fd != -1)
1891 close(imsg.fd);
1892 free(pack);
1893 } else
1894 *packp = pack;
1895 imsg_free(&imsg);
1896 return err;
1899 int
1900 main(int argc, char *argv[])
1902 const struct got_error *err = NULL;
1903 struct imsgbuf ibuf;
1904 struct imsg imsg;
1905 struct got_packidx *packidx = NULL;
1906 struct got_pack *pack = NULL;
1907 struct got_object_cache objcache;
1908 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1909 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1910 struct got_parsed_tree_entry *entries = NULL;
1911 size_t nentries = 0, nentries_alloc = 0;
1913 //static int attached;
1914 //while (!attached) sleep(1);
1916 signal(SIGINT, catch_sigint);
1918 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1920 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1921 if (err) {
1922 err = got_error_from_errno("got_object_cache_init");
1923 got_privsep_send_error(&ibuf, err);
1924 return 1;
1927 #ifndef PROFILE
1928 /* revoke access to most system calls */
1929 if (pledge("stdio recvfd", NULL) == -1) {
1930 err = got_error_from_errno("pledge");
1931 got_privsep_send_error(&ibuf, err);
1932 return 1;
1935 /* revoke fs access */
1936 if (landlock_no_fs() == -1) {
1937 err = got_error_from_errno("landlock_no_fs");
1938 got_privsep_send_error(&ibuf, err);
1939 return 1;
1941 if (cap_enter() == -1) {
1942 err = got_error_from_errno("cap_enter");
1943 got_privsep_send_error(&ibuf, err);
1944 return 1;
1946 #endif
1948 err = receive_packidx(&packidx, &ibuf);
1949 if (err) {
1950 got_privsep_send_error(&ibuf, err);
1951 return 1;
1954 err = receive_pack(&pack, &ibuf);
1955 if (err) {
1956 got_privsep_send_error(&ibuf, err);
1957 return 1;
1960 for (;;) {
1961 imsg.fd = -1;
1963 if (sigint_received) {
1964 err = got_error(GOT_ERR_CANCELLED);
1965 break;
1968 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1969 if (err) {
1970 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1971 err = NULL;
1972 break;
1975 if (imsg.hdr.type == GOT_IMSG_STOP)
1976 break;
1978 switch (imsg.hdr.type) {
1979 case GOT_IMSG_TMPFD:
1980 if (basefile == NULL) {
1981 err = receive_tempfile(&basefile, "w+",
1982 &imsg, &ibuf);
1983 } else if (accumfile == NULL) {
1984 err = receive_tempfile(&accumfile, "w+",
1985 &imsg, &ibuf);
1986 } else
1987 err = got_error(GOT_ERR_PRIVSEP_MSG);
1988 break;
1989 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1990 err = object_request(&imsg, &ibuf, pack, packidx,
1991 &objcache);
1992 break;
1993 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1994 if (basefile == NULL || accumfile == NULL) {
1995 err = got_error(GOT_ERR_PRIVSEP_MSG);
1996 break;
1998 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1999 &objcache, basefile, accumfile);
2000 break;
2001 case GOT_IMSG_RAW_DELTA_OUTFD:
2002 if (delta_outfile != NULL) {
2003 err = got_error(GOT_ERR_PRIVSEP_MSG);
2004 break;
2006 err = receive_tempfile(&delta_outfile, "w",
2007 &imsg, &ibuf);
2008 break;
2009 case GOT_IMSG_RAW_DELTA_REQUEST:
2010 if (delta_outfile == NULL) {
2011 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2012 break;
2014 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2015 pack, packidx);
2016 break;
2017 case GOT_IMSG_DELTA_REUSE_REQUEST:
2018 err = delta_reuse_request(&imsg, &ibuf, pack, packidx);
2019 break;
2020 case GOT_IMSG_COMMIT_REQUEST:
2021 err = commit_request(&imsg, &ibuf, pack, packidx,
2022 &objcache);
2023 break;
2024 case GOT_IMSG_TREE_REQUEST:
2025 err = tree_request(&imsg, &ibuf, pack, packidx,
2026 &objcache, &entries, &nentries, &nentries_alloc);
2027 break;
2028 case GOT_IMSG_BLOB_REQUEST:
2029 if (basefile == NULL || accumfile == NULL) {
2030 err = got_error(GOT_ERR_PRIVSEP_MSG);
2031 break;
2033 err = blob_request(&imsg, &ibuf, pack, packidx,
2034 &objcache, basefile, accumfile);
2035 break;
2036 case GOT_IMSG_TAG_REQUEST:
2037 err = tag_request(&imsg, &ibuf, pack, packidx,
2038 &objcache);
2039 break;
2040 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2041 err = commit_traversal_request(&imsg, &ibuf, pack,
2042 packidx, &objcache);
2043 break;
2044 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2045 err = enumeration_request(&imsg, &ibuf, pack,
2046 packidx, &objcache);
2047 break;
2048 case GOT_IMSG_COMMIT_PAINTING_INIT:
2049 commit_painting_free(&keep, &drop, &skip);
2050 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2051 break;
2052 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2053 if (keep == NULL || drop == NULL || skip == NULL) {
2054 err = got_error(GOT_ERR_PRIVSEP_MSG);
2055 break;
2057 err = commit_painting_request(&imsg, &ibuf, pack,
2058 packidx, &objcache, keep, drop, skip);
2059 break;
2060 case GOT_IMSG_COMMIT_PAINTING_DONE:
2061 commit_painting_free(&keep, &drop, &skip);
2062 break;
2063 default:
2064 err = got_error(GOT_ERR_PRIVSEP_MSG);
2065 break;
2068 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
2069 err = got_error_from_errno("close");
2070 imsg_free(&imsg);
2071 if (err)
2072 break;
2075 free(entries);
2076 commit_painting_free(&keep, &drop, &skip);
2077 if (packidx)
2078 got_packidx_close(packidx);
2079 if (pack)
2080 got_pack_close(pack);
2081 got_object_cache_close(&objcache);
2082 imsg_clear(&ibuf);
2083 if (basefile && fclose(basefile) == EOF && err == NULL)
2084 err = got_error_from_errno("fclose");
2085 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2086 err = got_error_from_errno("fclose");
2087 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2088 err = got_error_from_errno("fclose");
2089 if (err) {
2090 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2091 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2092 got_privsep_send_error(&ibuf, err);
2095 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2096 err = got_error_from_errno("close");
2097 return err ? 1 : 0;