Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/stat.h>
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
21 #include <sys/time.h>
22 #include <sys/mman.h>
24 #include <inttypes.h>
25 #include <limits.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <imsg.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <sha1.h>
33 #include <sha2.h>
34 #include <unistd.h>
35 #include <zlib.h>
37 #include "got_error.h"
38 #include "got_object.h"
39 #include "got_path.h"
41 #include "got_lib_delta.h"
42 #include "got_lib_delta_cache.h"
43 #include "got_lib_object.h"
44 #include "got_lib_object_cache.h"
45 #include "got_lib_object_parse.h"
46 #include "got_lib_object_idset.h"
47 #include "got_lib_privsep.h"
48 #include "got_lib_pack.h"
50 #ifndef nitems
51 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
52 #endif
54 static volatile sig_atomic_t sigint_received;
56 static void
57 catch_sigint(int signo)
58 {
59 sigint_received = 1;
60 }
62 static const struct got_error *
63 open_object(struct got_object **obj, struct got_pack *pack,
64 struct got_packidx *packidx, int idx, struct got_object_id *id,
65 struct got_object_cache *objcache)
66 {
67 const struct got_error *err;
69 err = got_packfile_open_object(obj, pack, packidx, idx, id);
70 if (err)
71 return err;
72 (*obj)->refcnt++;
74 err = got_object_cache_add(objcache, id, *obj);
75 if (err) {
76 if (err->code == GOT_ERR_OBJ_EXISTS ||
77 err->code == GOT_ERR_OBJ_TOO_LARGE)
78 err = NULL;
79 return err;
80 }
81 (*obj)->refcnt++;
82 return NULL;
83 }
85 static const struct got_error *
86 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
87 struct got_packidx *packidx, struct got_object_cache *objcache)
88 {
89 const struct got_error *err = NULL;
90 struct got_imsg_packed_object iobj;
91 struct got_object *obj;
92 struct got_object_id id;
93 size_t datalen;
95 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
96 if (datalen != sizeof(iobj))
97 return got_error(GOT_ERR_PRIVSEP_LEN);
98 memcpy(&iobj, imsg->data, sizeof(iobj));
99 memcpy(&id, &iobj.id, sizeof(id));
101 obj = got_object_cache_get(objcache, &id);
102 if (obj) {
103 obj->refcnt++;
104 } else {
105 err = open_object(&obj, pack, packidx, iobj.idx, &id,
106 objcache);
107 if (err)
108 goto done;
111 err = got_privsep_send_obj(ibuf, obj);
112 done:
113 got_object_close(obj);
114 return err;
117 static const struct got_error *
118 open_commit(struct got_commit_object **commit, struct got_pack *pack,
119 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
120 struct got_object_cache *objcache)
122 const struct got_error *err = NULL;
123 struct got_object *obj = NULL;
124 uint8_t *buf = NULL;
125 size_t len;
127 *commit = NULL;
129 obj = got_object_cache_get(objcache, id);
130 if (obj) {
131 obj->refcnt++;
132 } else {
133 err = open_object(&obj, pack, packidx, obj_idx, id,
134 objcache);
135 if (err)
136 return err;
139 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
140 if (err)
141 goto done;
143 obj->size = len;
145 err = got_object_parse_commit(commit, buf, len);
146 done:
147 got_object_close(obj);
148 free(buf);
149 return err;
152 static const struct got_error *
153 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
154 struct got_packidx *packidx, struct got_object_cache *objcache)
156 const struct got_error *err = NULL;
157 struct got_imsg_packed_object iobj;
158 struct got_commit_object *commit = NULL;
159 struct got_object_id id;
160 size_t datalen;
162 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
163 if (datalen != sizeof(iobj))
164 return got_error(GOT_ERR_PRIVSEP_LEN);
165 memcpy(&iobj, imsg->data, sizeof(iobj));
166 memcpy(&id, &iobj.id, sizeof(id));
168 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
169 if (err)
170 goto done;
172 err = got_privsep_send_commit(ibuf, commit);
173 done:
174 if (commit)
175 got_object_commit_close(commit);
176 if (err) {
177 if (err->code == GOT_ERR_PRIVSEP_PIPE)
178 err = NULL;
179 else
180 got_privsep_send_error(ibuf, err);
183 return err;
186 static const struct got_error *
187 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, size_t *nentries,
188 size_t *nentries_alloc, struct got_pack *pack, struct got_packidx *packidx,
189 int obj_idx, struct got_object_id *id, struct got_object_cache *objcache)
191 const struct got_error *err = NULL;
192 struct got_object *obj = NULL;
193 size_t len;
195 *buf = NULL;
196 *nentries = 0;
198 obj = got_object_cache_get(objcache, id);
199 if (obj) {
200 obj->refcnt++;
201 } else {
202 err = open_object(&obj, pack, packidx, obj_idx, id,
203 objcache);
204 if (err)
205 return err;
208 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
209 if (err)
210 goto done;
212 obj->size = len;
214 err = got_object_parse_tree(entries, nentries, nentries_alloc,
215 *buf, len);
216 done:
217 got_object_close(obj);
218 if (err) {
219 free(*buf);
220 *buf = NULL;
222 return err;
225 static const struct got_error *
226 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
227 struct got_packidx *packidx, struct got_object_cache *objcache,
228 struct got_parsed_tree_entry **entries, size_t *nentries,
229 size_t *nentries_alloc)
231 const struct got_error *err = NULL;
232 struct got_imsg_packed_object iobj;
233 uint8_t *buf = NULL;
234 struct got_object_id id;
235 size_t datalen;
237 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
238 if (datalen != sizeof(iobj))
239 return got_error(GOT_ERR_PRIVSEP_LEN);
240 memcpy(&iobj, imsg->data, sizeof(iobj));
241 memcpy(&id, &iobj.id, sizeof(id));
243 err = open_tree(&buf, entries, nentries, nentries_alloc,
244 pack, packidx, iobj.idx, &id, objcache);
245 if (err)
246 return err;
248 err = got_privsep_send_tree(ibuf, *entries, *nentries);
249 free(buf);
250 if (err) {
251 if (err->code == GOT_ERR_PRIVSEP_PIPE)
252 err = NULL;
253 else
254 got_privsep_send_error(ibuf, err);
257 return err;
260 static const struct got_error *
261 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
263 const struct got_error *err;
264 struct imsg imsg;
265 size_t datalen;
267 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
268 if (err)
269 return err;
271 if (imsg.hdr.type != imsg_code) {
272 err = got_error(GOT_ERR_PRIVSEP_MSG);
273 goto done;
276 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
277 if (datalen != 0) {
278 err = got_error(GOT_ERR_PRIVSEP_LEN);
279 goto done;
281 if (imsg.fd == -1) {
282 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
283 goto done;
286 *f = fdopen(imsg.fd, "w+");
287 if (*f == NULL) {
288 err = got_error_from_errno("fdopen");
289 close(imsg.fd);
290 goto done;
292 done:
293 imsg_free(&imsg);
294 return err;
297 static const struct got_error *
298 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
299 struct imsgbuf *ibuf)
301 size_t datalen;
303 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
304 if (datalen != 0)
305 return got_error(GOT_ERR_PRIVSEP_LEN);
307 if (imsg->fd == -1)
308 return got_error(GOT_ERR_PRIVSEP_NO_FD);
310 *f = fdopen(imsg->fd, mode);
311 if (*f == NULL)
312 return got_error_from_errno("fdopen");
313 imsg->fd = -1;
315 return NULL;
318 static const struct got_error *
319 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
320 struct got_packidx *packidx, struct got_object_cache *objcache,
321 FILE *basefile, FILE *accumfile)
323 const struct got_error *err = NULL;
324 struct got_imsg_packed_object iobj;
325 struct got_object *obj = NULL;
326 FILE *outfile = NULL;
327 struct got_object_id id;
328 size_t datalen;
329 uint64_t blob_size;
330 uint8_t *buf = NULL;
332 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
333 if (datalen != sizeof(iobj))
334 return got_error(GOT_ERR_PRIVSEP_LEN);
335 memcpy(&iobj, imsg->data, sizeof(iobj));
336 memcpy(&id, &iobj.id, sizeof(id));
338 obj = got_object_cache_get(objcache, &id);
339 if (obj) {
340 obj->refcnt++;
341 } else {
342 err = open_object(&obj, pack, packidx, iobj.idx, &id,
343 objcache);
344 if (err)
345 return err;
348 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
349 if (err)
350 goto done;
352 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
353 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
354 if (err)
355 goto done;
356 } else
357 blob_size = obj->size;
359 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
360 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
361 obj, pack);
362 else
363 err = got_packfile_extract_object(pack, obj, outfile, basefile,
364 accumfile);
365 if (err)
366 goto done;
368 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
369 done:
370 free(buf);
371 if (outfile && fclose(outfile) == EOF && err == NULL)
372 err = got_error_from_errno("fclose");
373 got_object_close(obj);
374 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
375 got_privsep_send_error(ibuf, err);
377 return err;
380 static const struct got_error *
381 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
382 struct got_packidx *packidx, struct got_object_cache *objcache)
384 const struct got_error *err = NULL;
385 struct got_imsg_packed_object iobj;
386 struct got_object *obj = NULL;
387 struct got_tag_object *tag = NULL;
388 uint8_t *buf = NULL;
389 size_t len;
390 struct got_object_id id;
391 size_t datalen;
393 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
394 if (datalen != sizeof(iobj))
395 return got_error(GOT_ERR_PRIVSEP_LEN);
396 memcpy(&iobj, imsg->data, sizeof(iobj));
397 memcpy(&id, &iobj.id, sizeof(id));
399 obj = got_object_cache_get(objcache, &id);
400 if (obj) {
401 obj->refcnt++;
402 } else {
403 err = open_object(&obj, pack, packidx, iobj.idx, &id,
404 objcache);
405 if (err)
406 return err;
409 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
410 if (err)
411 goto done;
413 obj->size = len;
414 err = got_object_parse_tag(&tag, buf, len);
415 if (err)
416 goto done;
418 err = got_privsep_send_tag(ibuf, tag);
419 done:
420 free(buf);
421 got_object_close(obj);
422 if (tag)
423 got_object_tag_close(tag);
424 if (err) {
425 if (err->code == GOT_ERR_PRIVSEP_PIPE)
426 err = NULL;
427 else
428 got_privsep_send_error(ibuf, err);
431 return err;
434 static struct got_parsed_tree_entry *
435 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
436 const char *name, size_t len)
438 struct got_parsed_tree_entry *pte;
439 int cmp, i;
441 /* Note that tree entries are sorted in strncmp() order. */
442 for (i = 0; i < nentries; i++) {
443 pte = &entries[i];
444 cmp = strncmp(pte->name, name, len);
445 if (cmp < 0)
446 continue;
447 if (cmp > 0)
448 break;
449 if (pte->name[len] == '\0')
450 return pte;
452 return NULL;
455 static const struct got_error *
456 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
457 struct got_parsed_tree_entry **entries1, size_t *nentries1,
458 size_t *nentries_alloc1,
459 struct got_parsed_tree_entry **entries2, size_t *nentries2,
460 size_t *nentries_alloc2,
461 const char *path, struct got_pack *pack, struct got_packidx *packidx,
462 struct imsgbuf *ibuf, struct got_object_cache *objcache)
464 const struct got_error *err = NULL;
465 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
466 const char *seg, *s;
467 size_t seglen;
469 *changed = 0;
471 /* We not do support comparing the root path. */
472 if (got_path_is_root_dir(path))
473 return got_error_path(path, GOT_ERR_BAD_PATH);
475 s = path;
476 while (*s == '/')
477 s++;
478 seg = s;
479 seglen = 0;
480 while (*s) {
481 if (*s != '/') {
482 s++;
483 seglen++;
484 if (*s)
485 continue;
488 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
489 if (pte1 == NULL) {
490 err = got_error(GOT_ERR_NO_OBJ);
491 break;
494 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
495 if (pte2 == NULL) {
496 *changed = 1;
497 break;
500 if (pte1->mode != pte2->mode) {
501 *changed = 1;
502 break;
505 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
506 *changed = 0;
507 break;
510 if (*s == '\0') { /* final path element */
511 *changed = 1;
512 break;
515 seg = s + 1;
516 s++;
517 seglen = 0;
518 if (*s) {
519 struct got_object_id id1, id2;
520 int idx;
522 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
523 idx = got_packidx_get_object_idx(packidx, &id1);
524 if (idx == -1) {
525 err = got_error_no_obj(&id1);
526 break;
528 *nentries1 = 0;
529 free(*buf1);
530 *buf1 = NULL;
531 err = open_tree(buf1, entries1, nentries1,
532 nentries_alloc1, pack, packidx, idx, &id1,
533 objcache);
534 pte1 = NULL;
535 if (err)
536 break;
538 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
539 idx = got_packidx_get_object_idx(packidx, &id2);
540 if (idx == -1) {
541 err = got_error_no_obj(&id2);
542 break;
544 *nentries2 = 0;
545 free(*buf2);
546 *buf2 = NULL;
547 err = open_tree(buf2, entries2, nentries2,
548 nentries_alloc2, pack, packidx, idx, &id2,
549 objcache);
550 pte2 = NULL;
551 if (err)
552 break;
556 return err;
559 static const struct got_error *
560 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
561 struct imsgbuf *ibuf)
563 struct ibuf *wbuf;
564 size_t i;
566 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
567 sizeof(struct got_imsg_traversed_commits) +
568 ncommits * sizeof(commit_ids[0]));
569 if (wbuf == NULL)
570 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
572 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
573 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
575 for (i = 0; i < ncommits; i++) {
576 struct got_object_id *id = &commit_ids[i];
577 if (imsg_add(wbuf, id, sizeof(*id)) == -1) {
578 return got_error_from_errno(
579 "imsg_add TRAVERSED_COMMITS");
583 wbuf->fd = -1;
584 imsg_close(ibuf, wbuf);
586 return got_privsep_flush_imsg(ibuf);
589 static const struct got_error *
590 send_commit_traversal_done(struct imsgbuf *ibuf)
592 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
593 NULL, 0) == -1)
594 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
596 return got_privsep_flush_imsg(ibuf);
599 static const struct got_error *
600 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
601 struct got_pack *pack, struct got_packidx *packidx,
602 struct got_object_cache *objcache)
604 const struct got_error *err = NULL;
605 struct got_imsg_packed_object iobj;
606 struct got_object_qid *pid;
607 struct got_commit_object *commit = NULL, *pcommit = NULL;
608 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
609 size_t nentries = 0, nentries_alloc = 0;
610 size_t pnentries = 0, pnentries_alloc = 0;
611 struct got_object_id id;
612 size_t datalen, path_len;
613 char *path = NULL;
614 const int min_alloc = 64;
615 int changed = 0, ncommits = 0, nallocated = 0;
616 struct got_object_id *commit_ids = NULL;
618 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
619 if (datalen < sizeof(iobj))
620 return got_error(GOT_ERR_PRIVSEP_LEN);
621 memcpy(&iobj, imsg->data, sizeof(iobj));
622 memcpy(&id, &iobj.id, sizeof(id));
624 path_len = datalen - sizeof(iobj) - 1;
625 if (path_len < 0)
626 return got_error(GOT_ERR_PRIVSEP_LEN);
627 if (path_len > 0) {
628 path = imsg->data + sizeof(iobj);
629 if (path[path_len] != '\0')
630 return got_error(GOT_ERR_PRIVSEP_LEN);
633 nallocated = min_alloc;
634 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
635 if (commit_ids == NULL)
636 return got_error_from_errno("reallocarray");
638 do {
639 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
640 int idx;
642 if (sigint_received) {
643 err = got_error(GOT_ERR_CANCELLED);
644 goto done;
647 if (commit == NULL) {
648 idx = got_packidx_get_object_idx(packidx, &id);
649 if (idx == -1)
650 break;
651 err = open_commit(&commit, pack, packidx,
652 idx, &id, objcache);
653 if (err) {
654 if (err->code != GOT_ERR_NO_OBJ)
655 goto done;
656 err = NULL;
657 break;
661 if (sizeof(struct got_imsg_traversed_commits) +
662 ncommits * sizeof(commit_ids[0]) >= max_datalen) {
663 err = send_traversed_commits(commit_ids, ncommits,
664 ibuf);
665 if (err)
666 goto done;
667 ncommits = 0;
669 ncommits++;
670 if (ncommits > nallocated) {
671 struct got_object_id *new;
672 nallocated += min_alloc;
673 new = reallocarray(commit_ids, nallocated,
674 sizeof(*commit_ids));
675 if (new == NULL) {
676 err = got_error_from_errno("reallocarray");
677 goto done;
679 commit_ids = new;
681 memcpy(&commit_ids[ncommits - 1], &id, sizeof(id));
683 pid = STAILQ_FIRST(&commit->parent_ids);
684 if (pid == NULL)
685 break;
687 idx = got_packidx_get_object_idx(packidx, &pid->id);
688 if (idx == -1)
689 break;
691 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
692 objcache);
693 if (err) {
694 if (err->code != GOT_ERR_NO_OBJ)
695 goto done;
696 err = NULL;
697 break;
700 if (path[0] == '/' && path[1] == '\0') {
701 if (got_object_id_cmp(pcommit->tree_id,
702 commit->tree_id) != 0) {
703 changed = 1;
704 break;
706 } else {
707 int pidx;
708 uint8_t *buf = NULL, *pbuf = NULL;
710 idx = got_packidx_get_object_idx(packidx,
711 commit->tree_id);
712 if (idx == -1)
713 break;
714 pidx = got_packidx_get_object_idx(packidx,
715 pcommit->tree_id);
716 if (pidx == -1)
717 break;
719 err = open_tree(&buf, &entries, &nentries,
720 &nentries_alloc, pack, packidx, idx,
721 commit->tree_id, objcache);
722 if (err)
723 goto done;
724 err = open_tree(&pbuf, &pentries, &pnentries,
725 &pnentries_alloc, pack, packidx, pidx,
726 pcommit->tree_id, objcache);
727 if (err) {
728 free(buf);
729 goto done;
732 err = tree_path_changed(&changed, &buf, &pbuf,
733 &entries, &nentries, &nentries_alloc,
734 &pentries, &pnentries, &pnentries_alloc,
735 path, pack, packidx, ibuf, objcache);
737 nentries = 0;
738 free(buf);
739 pnentries = 0;
740 free(pbuf);
741 if (err) {
742 if (err->code != GOT_ERR_NO_OBJ)
743 goto done;
744 err = NULL;
745 break;
749 if (!changed) {
750 memcpy(&id, &pid->id, sizeof(id));
751 got_object_commit_close(commit);
752 commit = pcommit;
753 pcommit = NULL;
755 } while (!changed);
757 if (ncommits > 0) {
758 err = send_traversed_commits(commit_ids, ncommits, ibuf);
759 if (err)
760 goto done;
762 if (changed) {
763 err = got_privsep_send_commit(ibuf, commit);
764 if (err)
765 goto done;
768 err = send_commit_traversal_done(ibuf);
769 done:
770 free(commit_ids);
771 if (commit)
772 got_object_commit_close(commit);
773 if (pcommit)
774 got_object_commit_close(pcommit);
775 free(entries);
776 free(pentries);
777 if (err) {
778 if (err->code == GOT_ERR_PRIVSEP_PIPE)
779 err = NULL;
780 else
781 got_privsep_send_error(ibuf, err);
784 return err;
787 static const struct got_error *
788 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
789 struct got_pack *pack, struct got_packidx *packidx,
790 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
792 const struct got_error *err = NULL;
793 uint8_t *buf = NULL;
794 uint64_t size = 0;
795 FILE *outfile = NULL;
796 struct got_imsg_packed_object iobj;
797 struct got_object *obj;
798 struct got_object_id id;
799 size_t datalen;
801 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
802 if (datalen != sizeof(iobj))
803 return got_error(GOT_ERR_PRIVSEP_LEN);
804 memcpy(&iobj, imsg->data, sizeof(iobj));
805 memcpy(&id, &iobj.id, sizeof(id));
807 obj = got_object_cache_get(objcache, &id);
808 if (obj) {
809 obj->refcnt++;
810 } else {
811 err = open_object(&obj, pack, packidx, iobj.idx, &id,
812 objcache);
813 if (err)
814 return err;
817 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
818 if (err)
819 return err;
821 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
822 err = got_pack_get_max_delta_object_size(&size, obj, pack);
823 if (err)
824 goto done;
825 } else
826 size = obj->size;
828 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
829 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
830 obj, pack);
831 else
832 err = got_packfile_extract_object(pack, obj, outfile, basefile,
833 accumfile);
834 if (err)
835 goto done;
837 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
838 done:
839 free(buf);
840 if (outfile && fclose(outfile) == EOF && err == NULL)
841 err = got_error_from_errno("fclose");
842 got_object_close(obj);
843 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
844 got_privsep_send_error(ibuf, err);
846 return err;
849 static const struct got_error *
850 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
851 off_t base_offset)
853 const struct got_error *err;
854 int idx;
856 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
857 if (err)
858 return err;
859 if (idx == -1)
860 return got_error(GOT_ERR_BAD_PACKIDX);
862 return got_packidx_get_object_id(base_id, packidx, idx);
865 static const struct got_error *
866 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
867 FILE *delta_outfile, struct got_pack *pack,
868 struct got_packidx *packidx)
870 const struct got_error *err = NULL;
871 struct got_imsg_raw_delta_request req;
872 size_t datalen, delta_size, delta_compressed_size;
873 off_t delta_offset, delta_data_offset;
874 uint8_t *delta_buf = NULL;
875 struct got_object_id id, base_id;
876 off_t base_offset, delta_out_offset = 0;
877 uint64_t base_size = 0, result_size = 0;
878 size_t w;
880 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
881 if (datalen != sizeof(req))
882 return got_error(GOT_ERR_PRIVSEP_LEN);
883 memcpy(&req, imsg->data, sizeof(req));
884 memcpy(&id, &req.id, sizeof(id));
886 imsg->fd = -1;
888 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
889 &delta_compressed_size, &delta_offset, &delta_data_offset,
890 &base_offset, &base_id, &base_size, &result_size,
891 pack, packidx, req.idx);
892 if (err)
893 goto done;
895 /*
896 * If this is an offset delta we must determine the base
897 * object ID ourselves.
898 */
899 if (base_offset != 0) {
900 err = get_base_object_id(&base_id, packidx, base_offset);
901 if (err)
902 goto done;
905 delta_out_offset = ftello(delta_outfile);
906 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
907 if (w != delta_compressed_size) {
908 err = got_ferror(delta_outfile, GOT_ERR_IO);
909 goto done;
911 if (fflush(delta_outfile) == -1) {
912 err = got_error_from_errno("fflush");
913 goto done;
916 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
917 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
918 &base_id);
919 done:
920 free(delta_buf);
921 return err;
924 struct search_deltas_arg {
925 struct imsgbuf *ibuf;
926 struct got_packidx *packidx;
927 struct got_pack *pack;
928 struct got_object_idset *idset;
929 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
930 size_t ndeltas;
931 };
933 static const struct got_error *
934 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
936 const struct got_error *err;
937 struct search_deltas_arg *a = arg;
938 int obj_idx;
939 uint8_t *delta_buf = NULL;
940 uint64_t base_size, result_size;
941 size_t delta_size, delta_compressed_size;
942 off_t delta_offset, delta_data_offset, base_offset;
943 struct got_object_id base_id;
945 if (sigint_received)
946 return got_error(GOT_ERR_CANCELLED);
948 obj_idx = got_packidx_get_object_idx(a->packidx, id);
949 if (obj_idx == -1)
950 return NULL; /* object not present in our pack file */
952 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
953 &delta_compressed_size, &delta_offset, &delta_data_offset,
954 &base_offset, &base_id, &base_size, &result_size,
955 a->pack, a->packidx, obj_idx);
956 if (err) {
957 if (err->code == GOT_ERR_OBJ_TYPE)
958 return NULL; /* object not stored as a delta */
959 return err;
962 /*
963 * If this is an offset delta we must determine the base
964 * object ID ourselves.
965 */
966 if (base_offset != 0) {
967 err = get_base_object_id(&base_id, a->packidx, base_offset);
968 if (err)
969 goto done;
972 if (got_object_idset_contains(a->idset, &base_id)) {
973 struct got_imsg_reused_delta *delta;
975 delta = &a->deltas[a->ndeltas++];
976 memcpy(&delta->id, id, sizeof(delta->id));
977 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
978 delta->base_size = base_size;
979 delta->result_size = result_size;
980 delta->delta_size = delta_size;
981 delta->delta_compressed_size = delta_compressed_size;
982 delta->delta_offset = delta_data_offset;
984 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
985 err = got_privsep_send_reused_deltas(a->ibuf,
986 a->deltas, a->ndeltas);
987 if (err)
988 goto done;
989 a->ndeltas = 0;
992 done:
993 free(delta_buf);
994 return err;
997 static const struct got_error *
998 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1000 const struct got_error *err = NULL;
1001 int done = 0;
1002 struct got_object_id *ids;
1003 size_t nids, i;
1005 for (;;) {
1006 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1007 if (err || done)
1008 break;
1009 for (i = 0; i < nids; i++) {
1010 err = got_object_idset_add(idset, &ids[i], NULL);
1011 if (err) {
1012 free(ids);
1013 return err;
1016 free(ids);
1019 return err;
1022 static const struct got_error *
1023 recv_object_id_queue(struct got_object_id_queue *queue,
1024 struct got_object_idset *queued_ids, struct imsgbuf *ibuf)
1026 const struct got_error *err = NULL;
1027 int done = 0;
1028 struct got_object_qid *qid;
1029 struct got_object_id *ids;
1030 size_t nids, i;
1032 for (;;) {
1033 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1034 if (err || done)
1035 break;
1036 for (i = 0; i < nids; i++) {
1037 err = got_object_qid_alloc_partial(&qid);
1038 if (err)
1039 return err;
1040 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1041 STAILQ_INSERT_TAIL(queue, qid, entry);
1042 err = got_object_idset_add(queued_ids, &qid->id, NULL);
1043 if (err)
1044 return err;
1048 return err;
1051 static const struct got_error *
1052 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1053 struct got_pack *pack, struct got_packidx *packidx)
1055 const struct got_error *err = NULL;
1056 struct got_object_idset *idset;
1057 struct search_deltas_arg sda;
1059 idset = got_object_idset_alloc();
1060 if (idset == NULL)
1061 return got_error_from_errno("got_object_idset_alloc");
1063 err = recv_object_ids(idset, ibuf);
1064 if (err)
1065 return err;
1067 memset(&sda, 0, sizeof(sda));
1068 sda.ibuf = ibuf;
1069 sda.idset = idset;
1070 sda.pack = pack;
1071 sda.packidx = packidx;
1072 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1073 if (err)
1074 goto done;
1076 if (sda.ndeltas > 0) {
1077 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1078 sda.ndeltas);
1079 if (err)
1080 goto done;
1083 err = got_privsep_send_reused_deltas_done(ibuf);
1084 done:
1085 got_object_idset_free(idset);
1086 return err;
1089 static const struct got_error *
1090 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1092 const struct got_error *err = NULL;
1093 struct imsg imsg;
1094 struct got_imsg_packidx ipackidx;
1095 size_t datalen;
1096 struct got_packidx *p;
1098 *packidx = NULL;
1100 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1101 if (err)
1102 return err;
1104 p = calloc(1, sizeof(*p));
1105 if (p == NULL) {
1106 err = got_error_from_errno("calloc");
1107 goto done;
1110 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1111 err = got_error(GOT_ERR_PRIVSEP_MSG);
1112 goto done;
1115 if (imsg.fd == -1) {
1116 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1117 goto done;
1120 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1121 if (datalen != sizeof(ipackidx)) {
1122 err = got_error(GOT_ERR_PRIVSEP_LEN);
1123 goto done;
1125 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1127 p->len = ipackidx.len;
1128 p->fd = dup(imsg.fd);
1129 if (p->fd == -1) {
1130 err = got_error_from_errno("dup");
1131 goto done;
1133 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1134 err = got_error_from_errno("lseek");
1135 goto done;
1138 #ifndef GOT_PACK_NO_MMAP
1139 if (p->len > 0 && p->len <= SIZE_MAX) {
1140 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1141 if (p->map == MAP_FAILED)
1142 p->map = NULL; /* fall back to read(2) */
1144 #endif
1145 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1146 done:
1147 if (err) {
1148 if (imsg.fd != -1)
1149 close(imsg.fd);
1150 got_packidx_close(p);
1151 } else
1152 *packidx = p;
1153 imsg_free(&imsg);
1154 return err;
1157 static const struct got_error *
1158 send_tree_enumeration_done(struct imsgbuf *ibuf)
1160 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1161 NULL, 0) == -1)
1162 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1164 return got_privsep_flush_imsg(ibuf);
1167 struct enumerated_tree {
1168 struct got_object_id id;
1169 char *path;
1170 uint8_t *buf;
1171 struct got_parsed_tree_entry *entries;
1172 int nentries;
1175 static const struct got_error *
1176 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1177 struct got_object_id *tree_id,
1178 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1179 struct got_object_cache *objcache, struct got_object_idset *idset,
1180 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1182 const struct got_error *err = NULL;
1183 struct got_object_id_queue ids;
1184 struct got_object_qid *qid;
1185 uint8_t *buf = NULL;
1186 struct got_parsed_tree_entry *entries = NULL;
1187 size_t nentries = 0, nentries_alloc = 0, i;
1188 struct enumerated_tree *tree;
1190 *ntrees = 0;
1191 *have_all_entries = 1;
1192 STAILQ_INIT(&ids);
1194 err = got_object_qid_alloc_partial(&qid);
1195 if (err)
1196 return err;
1197 memcpy(&qid->id, tree_id, sizeof(*tree_id));
1198 qid->data = strdup(path);
1199 if (qid->data == NULL) {
1200 err = got_error_from_errno("strdup");
1201 goto done;
1203 STAILQ_INSERT_TAIL(&ids, qid, entry);
1204 qid = NULL;
1206 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1207 do {
1208 const char *path;
1209 int idx, i;
1211 if (sigint_received) {
1212 err = got_error(GOT_ERR_CANCELLED);
1213 goto done;
1216 qid = STAILQ_FIRST(&ids);
1217 STAILQ_REMOVE_HEAD(&ids, entry);
1218 path = qid->data;
1220 idx = got_packidx_get_object_idx(packidx, &qid->id);
1221 if (idx == -1) {
1222 *have_all_entries = 0;
1223 break;
1226 err = open_tree(&buf, &entries, &nentries, &nentries_alloc,
1227 pack, packidx, idx, &qid->id, objcache);
1228 if (err) {
1229 if (err->code != GOT_ERR_NO_OBJ)
1230 goto done;
1233 err = got_object_idset_add(idset, &qid->id, NULL);
1234 if (err)
1235 goto done;
1237 for (i = 0; i < nentries; i++) {
1238 struct got_object_qid *eqid = NULL;
1239 struct got_parsed_tree_entry *pte = &entries[i];
1240 char *p;
1242 if (!S_ISDIR(pte->mode))
1243 continue;
1245 err = got_object_qid_alloc_partial(&eqid);
1246 if (err)
1247 goto done;
1248 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1250 if (got_object_idset_contains(idset, &eqid->id)) {
1251 got_object_qid_free(eqid);
1252 continue;
1255 if (asprintf(&p, "%s%s%s", path,
1256 got_path_is_root_dir(path) ? "" : "/",
1257 pte->name) == -1) {
1258 err = got_error_from_errno("asprintf");
1259 got_object_qid_free(eqid);
1260 goto done;
1262 eqid->data = p;
1263 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1266 if (*ntrees >= *nalloc) {
1267 struct enumerated_tree *new;
1268 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1269 sizeof(*new));
1270 if (new == NULL) {
1271 err = got_error_from_errno("malloc");
1272 goto done;
1274 *trees = new;
1275 *nalloc += 16;
1277 tree = &(*trees)[*ntrees];
1278 (*ntrees)++;
1279 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1280 tree->path = qid->data;
1281 tree->buf = buf;
1282 buf = NULL;
1283 tree->entries = entries;
1284 entries = NULL;
1285 nentries_alloc = 0;
1286 tree->nentries = nentries;
1287 nentries = 0;
1289 got_object_qid_free(qid);
1290 qid = NULL;
1291 } while (!STAILQ_EMPTY(&ids));
1293 if (*have_all_entries) {
1294 int i;
1296 * We have managed to traverse all entries in the hierarchy.
1297 * Tell the main process what we have found.
1299 for (i = 0; i < *ntrees; i++) {
1300 tree = &(*trees)[i];
1301 err = got_privsep_send_enumerated_tree(totlen,
1302 ibuf, &tree->id, tree->path, tree->entries,
1303 tree->nentries);
1304 if (err)
1305 goto done;
1306 free(tree->buf);
1307 tree->buf = NULL;
1308 free(tree->path);
1309 tree->path = NULL;
1310 free(tree->entries);
1311 tree->entries = NULL;
1313 *ntrees = 0; /* don't loop again below to free memory */
1315 err = send_tree_enumeration_done(ibuf);
1316 } else {
1318 * We can only load fully packed tree hierarchies on
1319 * behalf of the main process, otherwise the main process
1320 * gets a wrong idea about which tree objects have
1321 * already been traversed.
1322 * Indicate a missing entry for the root of this tree.
1323 * The main process should continue by loading this
1324 * entire tree the slow way.
1326 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1327 tree_id, "/", NULL, -1);
1328 if (err)
1329 goto done;
1331 done:
1332 free(buf);
1333 free(entries);
1334 for (i = 0; i < *ntrees; i++) {
1335 tree = &(*trees)[i];
1336 free(tree->buf);
1337 tree->buf = NULL;
1338 free(tree->path);
1339 tree->path = NULL;
1340 free(tree->entries);
1341 tree->entries = NULL;
1343 if (qid)
1344 free(qid->data);
1345 got_object_qid_free(qid);
1346 got_object_id_queue_free(&ids);
1347 if (err) {
1348 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1349 err = NULL;
1350 else
1351 got_privsep_send_error(ibuf, err);
1354 return err;
1357 static const struct got_error *
1358 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1359 struct got_pack *pack, struct got_packidx *packidx,
1360 struct got_object_cache *objcache)
1362 const struct got_error *err = NULL;
1363 struct got_object_id_queue commit_ids;
1364 const struct got_object_id_queue *parents = NULL;
1365 struct got_object_qid *qid = NULL;
1366 struct got_object *obj = NULL;
1367 struct got_commit_object *commit = NULL;
1368 struct got_object_id *tree_id = NULL;
1369 size_t totlen = 0;
1370 struct got_object_idset *idset, *queued_ids = NULL;
1371 int i, idx, have_all_entries = 1;
1372 struct enumerated_tree *trees = NULL;
1373 size_t ntrees = 0, nalloc = 16;
1375 STAILQ_INIT(&commit_ids);
1377 trees = calloc(nalloc, sizeof(*trees));
1378 if (trees == NULL)
1379 return got_error_from_errno("calloc");
1381 idset = got_object_idset_alloc();
1382 if (idset == NULL) {
1383 err = got_error_from_errno("got_object_idset_alloc");
1384 goto done;
1387 queued_ids = got_object_idset_alloc();
1388 if (queued_ids == NULL) {
1389 err = got_error_from_errno("got_object_idset_alloc");
1390 goto done;
1393 err = recv_object_id_queue(&commit_ids, queued_ids, ibuf);
1394 if (err)
1395 goto done;
1397 if (STAILQ_EMPTY(&commit_ids)) {
1398 err = got_error(GOT_ERR_PRIVSEP_MSG);
1399 goto done;
1402 err = recv_object_ids(idset, ibuf);
1403 if (err)
1404 goto done;
1406 while (!STAILQ_EMPTY(&commit_ids)) {
1407 if (sigint_received) {
1408 err = got_error(GOT_ERR_CANCELLED);
1409 goto done;
1412 qid = STAILQ_FIRST(&commit_ids);
1413 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1415 if (got_object_idset_contains(idset, &qid->id)) {
1416 got_object_qid_free(qid);
1417 qid = NULL;
1418 continue;
1421 idx = got_packidx_get_object_idx(packidx, &qid->id);
1422 if (idx == -1) {
1423 have_all_entries = 0;
1424 break;
1427 err = open_object(&obj, pack, packidx, idx, &qid->id,
1428 objcache);
1429 if (err)
1430 goto done;
1431 if (obj->type == GOT_OBJ_TYPE_TAG) {
1432 struct got_tag_object *tag;
1433 uint8_t *buf;
1434 size_t len;
1435 err = got_packfile_extract_object_to_mem(&buf,
1436 &len, obj, pack);
1437 if (err)
1438 goto done;
1439 obj->size = len;
1440 err = got_object_parse_tag(&tag, buf, len);
1441 if (err) {
1442 free(buf);
1443 goto done;
1445 idx = got_packidx_get_object_idx(packidx, &tag->id);
1446 if (idx == -1) {
1447 have_all_entries = 0;
1448 break;
1450 err = open_commit(&commit, pack, packidx, idx,
1451 &tag->id, objcache);
1452 got_object_tag_close(tag);
1453 free(buf);
1454 if (err)
1455 goto done;
1456 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1457 err = open_commit(&commit, pack, packidx, idx,
1458 &qid->id, objcache);
1459 if (err)
1460 goto done;
1461 } else {
1462 err = got_error(GOT_ERR_OBJ_TYPE);
1463 goto done;
1465 got_object_close(obj);
1466 obj = NULL;
1468 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1469 got_object_commit_get_committer_time(commit));
1470 if (err)
1471 goto done;
1473 tree_id = got_object_commit_get_tree_id(commit);
1474 idx = got_packidx_get_object_idx(packidx, tree_id);
1475 if (idx == -1) {
1476 have_all_entries = 0;
1477 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1478 tree_id, "/", NULL, -1);
1479 if (err)
1480 goto done;
1481 break;
1484 if (got_object_idset_contains(idset, tree_id)) {
1485 got_object_qid_free(qid);
1486 qid = NULL;
1487 err = send_tree_enumeration_done(ibuf);
1488 if (err)
1489 goto done;
1490 continue;
1493 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1494 tree_id, "/", pack, packidx, objcache, idset,
1495 &trees, &nalloc, &ntrees);
1496 if (err)
1497 goto done;
1499 if (!have_all_entries)
1500 break;
1502 got_object_qid_free(qid);
1503 qid = NULL;
1505 parents = got_object_commit_get_parent_ids(commit);
1506 if (parents) {
1507 struct got_object_qid *pid;
1508 STAILQ_FOREACH(pid, parents, entry) {
1509 if (got_object_idset_contains(idset, &pid->id))
1510 continue;
1511 if (got_object_idset_contains(queued_ids, &pid->id))
1512 continue;
1513 err = got_object_qid_alloc_partial(&qid);
1514 if (err)
1515 goto done;
1516 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1517 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1518 qid = NULL;
1522 got_object_commit_close(commit);
1523 commit = NULL;
1526 if (have_all_entries) {
1527 err = got_privsep_send_object_enumeration_done(ibuf);
1528 if (err)
1529 goto done;
1530 } else {
1531 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1532 if (err)
1533 goto done;
1535 done:
1536 if (obj)
1537 got_object_close(obj);
1538 if (commit)
1539 got_object_commit_close(commit);
1540 got_object_qid_free(qid);
1541 got_object_id_queue_free(&commit_ids);
1542 if (idset)
1543 got_object_idset_free(idset);
1544 if (queued_ids)
1545 got_object_idset_free(queued_ids);
1546 for (i = 0; i < ntrees; i++) {
1547 struct enumerated_tree *tree = &trees[i];
1548 free(tree->buf);
1549 free(tree->path);
1550 free(tree->entries);
1552 free(trees);
1553 return err;
1556 enum findtwixt_color {
1557 COLOR_KEEP = 0,
1558 COLOR_DROP,
1559 COLOR_SKIP,
1560 COLOR_MAX,
1563 static const struct got_error *
1564 paint_commit(struct got_object_qid *qid, intptr_t color)
1566 if (color < 0 || color >= COLOR_MAX)
1567 return got_error(GOT_ERR_RANGE);
1569 qid->data = (void *)color;
1570 return NULL;
1573 static const struct got_error *
1574 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1575 intptr_t color)
1577 const struct got_error *err;
1578 struct got_object_qid *qid;
1580 err = got_object_qid_alloc_partial(&qid);
1581 if (err)
1582 return err;
1584 memcpy(&qid->id, id, sizeof(qid->id));
1585 STAILQ_INSERT_TAIL(ids, qid, entry);
1586 return paint_commit(qid, color);
1589 static const struct got_error *
1590 paint_commits(struct got_object_id_queue *ids, int *nids,
1591 struct got_object_idset *keep, struct got_object_idset *drop,
1592 struct got_object_idset *skip, struct got_pack *pack,
1593 struct got_packidx *packidx, struct imsgbuf *ibuf,
1594 struct got_object_cache *objcache)
1596 const struct got_error *err = NULL;
1597 struct got_commit_object *commit = NULL;
1598 struct got_object_id_queue painted;
1599 const struct got_object_id_queue *parents;
1600 struct got_object_qid *qid = NULL;
1601 int nqueued = *nids, nskip = 0, npainted = 0;
1603 STAILQ_INIT(&painted);
1605 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1606 int idx;
1607 intptr_t color;
1609 if (sigint_received) {
1610 err = got_error(GOT_ERR_CANCELLED);
1611 goto done;
1614 qid = STAILQ_FIRST(ids);
1615 idx = got_packidx_get_object_idx(packidx, &qid->id);
1616 if (idx == -1) {
1617 qid = NULL;
1618 break;
1621 STAILQ_REMOVE_HEAD(ids, entry);
1622 nqueued--;
1623 color = (intptr_t)qid->data;
1624 if (color == COLOR_SKIP)
1625 nskip--;
1627 if (got_object_idset_contains(skip, &qid->id)) {
1628 got_object_qid_free(qid);
1629 qid = NULL;
1630 continue;
1633 switch (color) {
1634 case COLOR_KEEP:
1635 if (got_object_idset_contains(keep, &qid->id)) {
1636 got_object_qid_free(qid);
1637 qid = NULL;
1638 continue;
1640 if (got_object_idset_contains(drop, &qid->id)) {
1641 err = paint_commit(qid, COLOR_SKIP);
1642 if (err)
1643 goto done;
1645 err = got_object_idset_add(keep, &qid->id, NULL);
1646 if (err)
1647 goto done;
1648 break;
1649 case COLOR_DROP:
1650 if (got_object_idset_contains(drop, &qid->id)) {
1651 got_object_qid_free(qid);
1652 qid = NULL;
1653 continue;
1655 if (got_object_idset_contains(keep, &qid->id)) {
1656 err = paint_commit(qid, COLOR_SKIP);
1657 if (err)
1658 goto done;
1660 err = got_object_idset_add(drop, &qid->id, NULL);
1661 if (err)
1662 goto done;
1663 break;
1664 case COLOR_SKIP:
1665 if (!got_object_idset_contains(skip, &qid->id)) {
1666 err = got_object_idset_add(skip, &qid->id,
1667 NULL);
1668 if (err)
1669 goto done;
1671 break;
1672 default:
1673 /* should not happen */
1674 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1675 "%s invalid commit color %"PRIdPTR, __func__,
1676 color);
1677 goto done;
1680 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1681 objcache);
1682 if (err)
1683 goto done;
1685 parents = got_object_commit_get_parent_ids(commit);
1686 if (parents) {
1687 struct got_object_qid *pid;
1688 color = (intptr_t)qid->data;
1689 STAILQ_FOREACH(pid, parents, entry) {
1690 err = queue_commit_id(ids, &pid->id, color);
1691 if (err)
1692 goto done;
1693 nqueued++;
1694 if (color == COLOR_SKIP)
1695 nskip++;
1699 got_object_commit_close(commit);
1700 commit = NULL;
1702 STAILQ_INSERT_TAIL(&painted, qid, entry);
1703 qid = NULL;
1704 npainted++;
1706 err = got_privsep_send_painted_commits(ibuf, &painted,
1707 &npainted, 1, 0);
1708 if (err)
1709 goto done;
1712 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1713 if (err)
1714 goto done;
1716 *nids = nqueued;
1717 done:
1718 if (commit)
1719 got_object_commit_close(commit);
1720 got_object_qid_free(qid);
1721 return err;
1724 static void
1725 commit_painting_free(struct got_object_idset **keep,
1726 struct got_object_idset **drop,
1727 struct got_object_idset **skip)
1729 if (*keep) {
1730 got_object_idset_free(*keep);
1731 *keep = NULL;
1733 if (*drop) {
1734 got_object_idset_free(*drop);
1735 *drop = NULL;
1737 if (*skip) {
1738 got_object_idset_free(*skip);
1739 *skip = NULL;
1743 static const struct got_error *
1744 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1745 struct got_object_idset **drop, struct got_object_idset **skip)
1747 const struct got_error *err = NULL;
1749 *keep = got_object_idset_alloc();
1750 if (*keep == NULL) {
1751 err = got_error_from_errno("got_object_idset_alloc");
1752 goto done;
1754 *drop = got_object_idset_alloc();
1755 if (*drop == NULL) {
1756 err = got_error_from_errno("got_object_idset_alloc");
1757 goto done;
1759 *skip = got_object_idset_alloc();
1760 if (*skip == NULL) {
1761 err = got_error_from_errno("got_object_idset_alloc");
1762 goto done;
1765 err = recv_object_ids(*keep, ibuf);
1766 if (err)
1767 goto done;
1768 err = recv_object_ids(*drop, ibuf);
1769 if (err)
1770 goto done;
1771 err = recv_object_ids(*skip, ibuf);
1772 if (err)
1773 goto done;
1775 done:
1776 if (err)
1777 commit_painting_free(keep, drop, skip);
1779 return err;
1782 static const struct got_error *
1783 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1784 struct got_pack *pack, struct got_packidx *packidx,
1785 struct got_object_cache *objcache, struct got_object_idset *keep,
1786 struct got_object_idset *drop, struct got_object_idset *skip)
1788 const struct got_error *err = NULL;
1789 struct got_imsg_commit_painting_request ireq;
1790 struct got_object_id id;
1791 size_t datalen;
1792 struct got_object_id_queue ids;
1793 int nids = 0;
1795 STAILQ_INIT(&ids);
1797 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1798 if (datalen != sizeof(ireq))
1799 return got_error(GOT_ERR_PRIVSEP_LEN);
1800 memcpy(&ireq, imsg->data, sizeof(ireq));
1801 memcpy(&id, &ireq.id, sizeof(id));
1803 err = queue_commit_id(&ids, &id, ireq.color);
1804 if (err)
1805 return err;
1806 nids = 1;
1808 err = paint_commits(&ids, &nids, keep, drop, skip,
1809 pack, packidx, ibuf, objcache);
1810 if (err)
1811 goto done;
1813 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1814 if (err)
1815 goto done;
1817 err = got_privsep_send_painting_commits_done(ibuf);
1818 done:
1819 got_object_id_queue_free(&ids);
1820 return err;
1823 static const struct got_error *
1824 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1826 const struct got_error *err = NULL;
1827 struct imsg imsg;
1828 struct got_imsg_pack ipack;
1829 size_t datalen;
1830 struct got_pack *pack;
1832 *packp = NULL;
1834 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1835 if (err)
1836 return err;
1838 pack = calloc(1, sizeof(*pack));
1839 if (pack == NULL) {
1840 err = got_error_from_errno("calloc");
1841 goto done;
1844 if (imsg.hdr.type != GOT_IMSG_PACK) {
1845 err = got_error(GOT_ERR_PRIVSEP_MSG);
1846 goto done;
1849 if (imsg.fd == -1) {
1850 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1851 goto done;
1854 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1855 if (datalen != sizeof(ipack)) {
1856 err = got_error(GOT_ERR_PRIVSEP_LEN);
1857 goto done;
1859 memcpy(&ipack, imsg.data, sizeof(ipack));
1861 pack->filesize = ipack.filesize;
1862 pack->fd = dup(imsg.fd);
1863 if (pack->fd == -1) {
1864 err = got_error_from_errno("dup");
1865 goto done;
1867 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1868 err = got_error_from_errno("lseek");
1869 goto done;
1871 pack->path_packfile = strdup(ipack.path_packfile);
1872 if (pack->path_packfile == NULL) {
1873 err = got_error_from_errno("strdup");
1874 goto done;
1877 err = got_delta_cache_alloc(&pack->delta_cache);
1878 if (err)
1879 goto done;
1881 #ifndef GOT_PACK_NO_MMAP
1882 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1883 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1884 pack->fd, 0);
1885 if (pack->map == MAP_FAILED)
1886 pack->map = NULL; /* fall back to read(2) */
1888 #endif
1889 done:
1890 if (err) {
1891 if (imsg.fd != -1)
1892 close(imsg.fd);
1893 free(pack);
1894 } else
1895 *packp = pack;
1896 imsg_free(&imsg);
1897 return err;
1900 int
1901 main(int argc, char *argv[])
1903 const struct got_error *err = NULL;
1904 struct imsgbuf ibuf;
1905 struct imsg imsg;
1906 struct got_packidx *packidx = NULL;
1907 struct got_pack *pack = NULL;
1908 struct got_object_cache objcache;
1909 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1910 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1911 struct got_parsed_tree_entry *entries = NULL;
1912 size_t nentries = 0, nentries_alloc = 0;
1914 //static int attached;
1915 //while (!attached) sleep(1);
1917 signal(SIGINT, catch_sigint);
1919 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1921 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1922 if (err) {
1923 err = got_error_from_errno("got_object_cache_init");
1924 got_privsep_send_error(&ibuf, err);
1925 return 1;
1928 #ifndef PROFILE
1929 /* revoke access to most system calls */
1930 if (pledge("stdio recvfd", NULL) == -1) {
1931 err = got_error_from_errno("pledge");
1932 got_privsep_send_error(&ibuf, err);
1933 return 1;
1935 #endif
1937 err = receive_packidx(&packidx, &ibuf);
1938 if (err) {
1939 got_privsep_send_error(&ibuf, err);
1940 return 1;
1943 err = receive_pack(&pack, &ibuf);
1944 if (err) {
1945 got_privsep_send_error(&ibuf, err);
1946 return 1;
1949 for (;;) {
1950 imsg.fd = -1;
1952 if (sigint_received) {
1953 err = got_error(GOT_ERR_CANCELLED);
1954 break;
1957 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1958 if (err) {
1959 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1960 err = NULL;
1961 break;
1964 if (imsg.hdr.type == GOT_IMSG_STOP)
1965 break;
1967 switch (imsg.hdr.type) {
1968 case GOT_IMSG_TMPFD:
1969 if (basefile == NULL) {
1970 err = receive_tempfile(&basefile, "w+",
1971 &imsg, &ibuf);
1972 } else if (accumfile == NULL) {
1973 err = receive_tempfile(&accumfile, "w+",
1974 &imsg, &ibuf);
1975 } else
1976 err = got_error(GOT_ERR_PRIVSEP_MSG);
1977 break;
1978 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1979 err = object_request(&imsg, &ibuf, pack, packidx,
1980 &objcache);
1981 break;
1982 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1983 if (basefile == NULL || accumfile == NULL) {
1984 err = got_error(GOT_ERR_PRIVSEP_MSG);
1985 break;
1987 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1988 &objcache, basefile, accumfile);
1989 break;
1990 case GOT_IMSG_RAW_DELTA_OUTFD:
1991 if (delta_outfile != NULL) {
1992 err = got_error(GOT_ERR_PRIVSEP_MSG);
1993 break;
1995 err = receive_tempfile(&delta_outfile, "w",
1996 &imsg, &ibuf);
1997 break;
1998 case GOT_IMSG_RAW_DELTA_REQUEST:
1999 if (delta_outfile == NULL) {
2000 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2001 break;
2003 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2004 pack, packidx);
2005 break;
2006 case GOT_IMSG_DELTA_REUSE_REQUEST:
2007 err = delta_reuse_request(&imsg, &ibuf, pack, packidx);
2008 break;
2009 case GOT_IMSG_COMMIT_REQUEST:
2010 err = commit_request(&imsg, &ibuf, pack, packidx,
2011 &objcache);
2012 break;
2013 case GOT_IMSG_TREE_REQUEST:
2014 err = tree_request(&imsg, &ibuf, pack, packidx,
2015 &objcache, &entries, &nentries, &nentries_alloc);
2016 break;
2017 case GOT_IMSG_BLOB_REQUEST:
2018 if (basefile == NULL || accumfile == NULL) {
2019 err = got_error(GOT_ERR_PRIVSEP_MSG);
2020 break;
2022 err = blob_request(&imsg, &ibuf, pack, packidx,
2023 &objcache, basefile, accumfile);
2024 break;
2025 case GOT_IMSG_TAG_REQUEST:
2026 err = tag_request(&imsg, &ibuf, pack, packidx,
2027 &objcache);
2028 break;
2029 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2030 err = commit_traversal_request(&imsg, &ibuf, pack,
2031 packidx, &objcache);
2032 break;
2033 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2034 err = enumeration_request(&imsg, &ibuf, pack,
2035 packidx, &objcache);
2036 break;
2037 case GOT_IMSG_COMMIT_PAINTING_INIT:
2038 commit_painting_free(&keep, &drop, &skip);
2039 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2040 break;
2041 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2042 if (keep == NULL || drop == NULL || skip == NULL) {
2043 err = got_error(GOT_ERR_PRIVSEP_MSG);
2044 break;
2046 err = commit_painting_request(&imsg, &ibuf, pack,
2047 packidx, &objcache, keep, drop, skip);
2048 break;
2049 case GOT_IMSG_COMMIT_PAINTING_DONE:
2050 commit_painting_free(&keep, &drop, &skip);
2051 break;
2052 default:
2053 err = got_error(GOT_ERR_PRIVSEP_MSG);
2054 break;
2057 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
2058 err = got_error_from_errno("close");
2059 imsg_free(&imsg);
2060 if (err)
2061 break;
2064 free(entries);
2065 commit_painting_free(&keep, &drop, &skip);
2066 if (packidx)
2067 got_packidx_close(packidx);
2068 if (pack)
2069 got_pack_close(pack);
2070 got_object_cache_close(&objcache);
2071 imsg_clear(&ibuf);
2072 if (basefile && fclose(basefile) == EOF && err == NULL)
2073 err = got_error_from_errno("fclose");
2074 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2075 err = got_error_from_errno("fclose");
2076 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2077 err = got_error_from_errno("fclose");
2078 if (err) {
2079 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2080 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2081 got_privsep_send_error(&ibuf, err);
2084 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2085 err = got_error_from_errno("close");
2086 return err ? 1 : 0;