Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/stat.h>
18 #include <sys/types.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/mman.h>
23 #include <limits.h>
24 #include <signal.h>
25 #include <stdint.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <unistd.h>
30 #include <zlib.h>
32 #include "got_compat.h"
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_object_idset.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_pack.h"
47 #ifndef nitems
48 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
49 #endif
51 static volatile sig_atomic_t sigint_received;
53 static void
54 catch_sigint(int signo)
55 {
56 sigint_received = 1;
57 }
59 static const struct got_error *
60 open_object(struct got_object **obj, struct got_pack *pack,
61 struct got_packidx *packidx, int idx, struct got_object_id *id,
62 struct got_object_cache *objcache)
63 {
64 const struct got_error *err;
66 err = got_packfile_open_object(obj, pack, packidx, idx, id);
67 if (err)
68 return err;
69 (*obj)->refcnt++;
71 err = got_object_cache_add(objcache, id, *obj);
72 if (err) {
73 if (err->code == GOT_ERR_OBJ_EXISTS ||
74 err->code == GOT_ERR_OBJ_TOO_LARGE)
75 err = NULL;
76 return err;
77 }
78 (*obj)->refcnt++;
79 return NULL;
80 }
82 static const struct got_error *
83 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
84 struct got_packidx *packidx, struct got_object_cache *objcache)
85 {
86 const struct got_error *err = NULL;
87 struct got_imsg_packed_object iobj;
88 struct got_object *obj;
89 struct got_object_id id;
90 size_t datalen;
92 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
93 if (datalen != sizeof(iobj))
94 return got_error(GOT_ERR_PRIVSEP_LEN);
95 memcpy(&iobj, imsg->data, sizeof(iobj));
96 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
98 obj = got_object_cache_get(objcache, &id);
99 if (obj) {
100 obj->refcnt++;
101 } else {
102 err = open_object(&obj, pack, packidx, iobj.idx, &id,
103 objcache);
104 if (err)
105 goto done;
108 err = got_privsep_send_obj(ibuf, obj);
109 done:
110 got_object_close(obj);
111 return err;
114 static const struct got_error *
115 open_commit(struct got_commit_object **commit, struct got_pack *pack,
116 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
117 struct got_object_cache *objcache)
119 const struct got_error *err = NULL;
120 struct got_object *obj = NULL;
121 uint8_t *buf = NULL;
122 size_t len;
124 *commit = NULL;
126 obj = got_object_cache_get(objcache, id);
127 if (obj) {
128 obj->refcnt++;
129 } else {
130 err = open_object(&obj, pack, packidx, obj_idx, id,
131 objcache);
132 if (err)
133 return err;
136 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
137 if (err)
138 goto done;
140 obj->size = len;
142 err = got_object_parse_commit(commit, buf, len);
143 done:
144 got_object_close(obj);
145 free(buf);
146 return err;
149 static const struct got_error *
150 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
151 struct got_packidx *packidx, struct got_object_cache *objcache)
153 const struct got_error *err = NULL;
154 struct got_imsg_packed_object iobj;
155 struct got_commit_object *commit = NULL;
156 struct got_object_id id;
157 size_t datalen;
159 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
160 if (datalen != sizeof(iobj))
161 return got_error(GOT_ERR_PRIVSEP_LEN);
162 memcpy(&iobj, imsg->data, sizeof(iobj));
163 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
165 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
166 if (err)
167 goto done;
169 err = got_privsep_send_commit(ibuf, commit);
170 done:
171 if (commit)
172 got_object_commit_close(commit);
173 if (err) {
174 if (err->code == GOT_ERR_PRIVSEP_PIPE)
175 err = NULL;
176 else
177 got_privsep_send_error(ibuf, err);
180 return err;
183 static const struct got_error *
184 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, int *nentries,
185 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
186 struct got_object_id *id, struct got_object_cache *objcache)
188 const struct got_error *err = NULL;
189 struct got_object *obj = NULL;
190 size_t len;
192 *buf = NULL;
193 *nentries = 0;
195 obj = got_object_cache_get(objcache, id);
196 if (obj) {
197 obj->refcnt++;
198 } else {
199 err = open_object(&obj, pack, packidx, obj_idx, id,
200 objcache);
201 if (err)
202 return err;
205 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
206 if (err)
207 goto done;
209 obj->size = len;
211 err = got_object_parse_tree(entries, nentries, *buf, len);
212 done:
213 got_object_close(obj);
214 if (err) {
215 free(*buf);
216 *buf = NULL;
218 return err;
221 static const struct got_error *
222 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
223 struct got_packidx *packidx, struct got_object_cache *objcache)
225 const struct got_error *err = NULL;
226 struct got_imsg_packed_object iobj;
227 struct got_parsed_tree_entry *entries = NULL;
228 int nentries = 0;
229 uint8_t *buf = NULL;
230 struct got_object_id id;
231 size_t datalen;
233 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
234 if (datalen != sizeof(iobj))
235 return got_error(GOT_ERR_PRIVSEP_LEN);
236 memcpy(&iobj, imsg->data, sizeof(iobj));
237 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
239 err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
240 &id, objcache);
241 if (err)
242 return err;
244 err = got_privsep_send_tree(ibuf, entries, nentries);
245 free(entries);
246 free(buf);
247 if (err) {
248 if (err->code == GOT_ERR_PRIVSEP_PIPE)
249 err = NULL;
250 else
251 got_privsep_send_error(ibuf, err);
254 return err;
257 static const struct got_error *
258 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
260 const struct got_error *err;
261 struct imsg imsg;
262 size_t datalen;
264 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
265 if (err)
266 return err;
268 if (imsg.hdr.type != imsg_code) {
269 err = got_error(GOT_ERR_PRIVSEP_MSG);
270 goto done;
273 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
274 if (datalen != 0) {
275 err = got_error(GOT_ERR_PRIVSEP_LEN);
276 goto done;
278 if (imsg.fd == -1) {
279 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
280 goto done;
283 *f = fdopen(imsg.fd, "w+");
284 if (*f == NULL) {
285 err = got_error_from_errno("fdopen");
286 close(imsg.fd);
287 goto done;
289 done:
290 imsg_free(&imsg);
291 return err;
294 static const struct got_error *
295 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
296 struct imsgbuf *ibuf)
298 size_t datalen;
300 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
301 if (datalen != 0)
302 return got_error(GOT_ERR_PRIVSEP_LEN);
304 if (imsg->fd == -1)
305 return got_error(GOT_ERR_PRIVSEP_NO_FD);
307 *f = fdopen(imsg->fd, mode);
308 if (*f == NULL)
309 return got_error_from_errno("fdopen");
310 imsg->fd = -1;
312 return NULL;
315 static const struct got_error *
316 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
317 struct got_packidx *packidx, struct got_object_cache *objcache,
318 FILE *basefile, FILE *accumfile)
320 const struct got_error *err = NULL;
321 struct got_imsg_packed_object iobj;
322 struct got_object *obj = NULL;
323 FILE *outfile = NULL;
324 struct got_object_id id;
325 size_t datalen;
326 uint64_t blob_size;
327 uint8_t *buf = NULL;
329 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
330 if (datalen != sizeof(iobj))
331 return got_error(GOT_ERR_PRIVSEP_LEN);
332 memcpy(&iobj, imsg->data, sizeof(iobj));
333 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
335 obj = got_object_cache_get(objcache, &id);
336 if (obj) {
337 obj->refcnt++;
338 } else {
339 err = open_object(&obj, pack, packidx, iobj.idx, &id,
340 objcache);
341 if (err)
342 return err;
345 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
346 if (err)
347 goto done;
349 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
350 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
351 if (err)
352 goto done;
353 } else
354 blob_size = obj->size;
356 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
357 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
358 obj, pack);
359 else
360 err = got_packfile_extract_object(pack, obj, outfile, basefile,
361 accumfile);
362 if (err)
363 goto done;
365 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
366 done:
367 free(buf);
368 if (outfile && fclose(outfile) == EOF && err == NULL)
369 err = got_error_from_errno("fclose");
370 got_object_close(obj);
371 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
372 got_privsep_send_error(ibuf, err);
374 return err;
377 static const struct got_error *
378 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
379 struct got_packidx *packidx, struct got_object_cache *objcache)
381 const struct got_error *err = NULL;
382 struct got_imsg_packed_object iobj;
383 struct got_object *obj = NULL;
384 struct got_tag_object *tag = NULL;
385 uint8_t *buf = NULL;
386 size_t len;
387 struct got_object_id id;
388 size_t datalen;
390 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
391 if (datalen != sizeof(iobj))
392 return got_error(GOT_ERR_PRIVSEP_LEN);
393 memcpy(&iobj, imsg->data, sizeof(iobj));
394 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
396 obj = got_object_cache_get(objcache, &id);
397 if (obj) {
398 obj->refcnt++;
399 } else {
400 err = open_object(&obj, pack, packidx, iobj.idx, &id,
401 objcache);
402 if (err)
403 return err;
406 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
407 if (err)
408 goto done;
410 obj->size = len;
411 err = got_object_parse_tag(&tag, buf, len);
412 if (err)
413 goto done;
415 err = got_privsep_send_tag(ibuf, tag);
416 done:
417 free(buf);
418 got_object_close(obj);
419 if (tag)
420 got_object_tag_close(tag);
421 if (err) {
422 if (err->code == GOT_ERR_PRIVSEP_PIPE)
423 err = NULL;
424 else
425 got_privsep_send_error(ibuf, err);
428 return err;
431 static struct got_parsed_tree_entry *
432 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
433 const char *name, size_t len)
435 struct got_parsed_tree_entry *pte;
436 int cmp, i;
438 /* Note that tree entries are sorted in strncmp() order. */
439 for (i = 0; i < nentries; i++) {
440 pte = &entries[i];
441 cmp = strncmp(pte->name, name, len);
442 if (cmp < 0)
443 continue;
444 if (cmp > 0)
445 break;
446 if (pte->name[len] == '\0')
447 return pte;
449 return NULL;
452 static const struct got_error *
453 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
454 struct got_parsed_tree_entry **entries1, int *nentries1,
455 struct got_parsed_tree_entry **entries2, int *nentries2,
456 const char *path, struct got_pack *pack, struct got_packidx *packidx,
457 struct imsgbuf *ibuf, struct got_object_cache *objcache)
459 const struct got_error *err = NULL;
460 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
461 const char *seg, *s;
462 size_t seglen;
464 *changed = 0;
466 /* We not do support comparing the root path. */
467 if (got_path_is_root_dir(path))
468 return got_error_path(path, GOT_ERR_BAD_PATH);
470 s = path;
471 while (*s == '/')
472 s++;
473 seg = s;
474 seglen = 0;
475 while (*s) {
476 if (*s != '/') {
477 s++;
478 seglen++;
479 if (*s)
480 continue;
483 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
484 if (pte1 == NULL) {
485 err = got_error(GOT_ERR_NO_OBJ);
486 break;
489 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
490 if (pte2 == NULL) {
491 *changed = 1;
492 break;
495 if (pte1->mode != pte2->mode) {
496 *changed = 1;
497 break;
500 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
501 *changed = 0;
502 break;
505 if (*s == '\0') { /* final path element */
506 *changed = 1;
507 break;
510 seg = s + 1;
511 s++;
512 seglen = 0;
513 if (*s) {
514 struct got_object_id id1, id2;
515 int idx;
517 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
518 idx = got_packidx_get_object_idx(packidx, &id1);
519 if (idx == -1) {
520 err = got_error_no_obj(&id1);
521 break;
523 free(*entries1);
524 *nentries1 = 0;
525 free(*buf1);
526 *buf1 = NULL;
527 err = open_tree(buf1, entries1, nentries1, pack,
528 packidx, idx, &id1, objcache);
529 pte1 = NULL;
530 if (err)
531 break;
533 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
534 idx = got_packidx_get_object_idx(packidx, &id2);
535 if (idx == -1) {
536 err = got_error_no_obj(&id2);
537 break;
539 free(*entries2);
540 *nentries2 = 0;
541 free(*buf2);
542 *buf2 = NULL;
543 err = open_tree(buf2, entries2, nentries2, pack,
544 packidx, idx, &id2, objcache);
545 pte2 = NULL;
546 if (err)
547 break;
551 return err;
554 static const struct got_error *
555 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
556 struct imsgbuf *ibuf)
558 struct ibuf *wbuf;
559 size_t i;
561 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
562 sizeof(struct got_imsg_traversed_commits) +
563 ncommits * SHA1_DIGEST_LENGTH);
564 if (wbuf == NULL)
565 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
567 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
568 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
570 for (i = 0; i < ncommits; i++) {
571 struct got_object_id *id = &commit_ids[i];
572 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
573 return got_error_from_errno(
574 "imsg_add TRAVERSED_COMMITS");
578 wbuf->fd = -1;
579 imsg_close(ibuf, wbuf);
581 return got_privsep_flush_imsg(ibuf);
584 static const struct got_error *
585 send_commit_traversal_done(struct imsgbuf *ibuf)
587 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
588 NULL, 0) == -1)
589 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
591 return got_privsep_flush_imsg(ibuf);
594 static const struct got_error *
595 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
596 struct got_pack *pack, struct got_packidx *packidx,
597 struct got_object_cache *objcache)
599 const struct got_error *err = NULL;
600 struct got_imsg_packed_object iobj;
601 struct got_object_qid *pid;
602 struct got_commit_object *commit = NULL, *pcommit = NULL;
603 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
604 int nentries = 0, pnentries = 0;
605 struct got_object_id id;
606 size_t datalen, path_len;
607 char *path = NULL;
608 const int min_alloc = 64;
609 int changed = 0, ncommits = 0, nallocated = 0;
610 struct got_object_id *commit_ids = NULL;
612 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
613 if (datalen < sizeof(iobj))
614 return got_error(GOT_ERR_PRIVSEP_LEN);
615 memcpy(&iobj, imsg->data, sizeof(iobj));
616 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
618 path_len = datalen - sizeof(iobj) - 1;
619 if (path_len < 0)
620 return got_error(GOT_ERR_PRIVSEP_LEN);
621 if (path_len > 0) {
622 path = imsg->data + sizeof(iobj);
623 if (path[path_len] != '\0')
624 return got_error(GOT_ERR_PRIVSEP_LEN);
627 nallocated = min_alloc;
628 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
629 if (commit_ids == NULL)
630 return got_error_from_errno("reallocarray");
632 do {
633 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
634 int idx;
636 if (sigint_received) {
637 err = got_error(GOT_ERR_CANCELLED);
638 goto done;
641 if (commit == NULL) {
642 idx = got_packidx_get_object_idx(packidx, &id);
643 if (idx == -1)
644 break;
645 err = open_commit(&commit, pack, packidx,
646 idx, &id, objcache);
647 if (err) {
648 if (err->code != GOT_ERR_NO_OBJ)
649 goto done;
650 err = NULL;
651 break;
655 if (sizeof(struct got_imsg_traversed_commits) +
656 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
657 err = send_traversed_commits(commit_ids, ncommits,
658 ibuf);
659 if (err)
660 goto done;
661 ncommits = 0;
663 ncommits++;
664 if (ncommits > nallocated) {
665 struct got_object_id *new;
666 nallocated += min_alloc;
667 new = reallocarray(commit_ids, nallocated,
668 sizeof(*commit_ids));
669 if (new == NULL) {
670 err = got_error_from_errno("reallocarray");
671 goto done;
673 commit_ids = new;
675 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
676 SHA1_DIGEST_LENGTH);
678 pid = STAILQ_FIRST(&commit->parent_ids);
679 if (pid == NULL)
680 break;
682 idx = got_packidx_get_object_idx(packidx, &pid->id);
683 if (idx == -1)
684 break;
686 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
687 objcache);
688 if (err) {
689 if (err->code != GOT_ERR_NO_OBJ)
690 goto done;
691 err = NULL;
692 break;
695 if (path[0] == '/' && path[1] == '\0') {
696 if (got_object_id_cmp(pcommit->tree_id,
697 commit->tree_id) != 0) {
698 changed = 1;
699 break;
701 } else {
702 int pidx;
703 uint8_t *buf = NULL, *pbuf = NULL;
705 idx = got_packidx_get_object_idx(packidx,
706 commit->tree_id);
707 if (idx == -1)
708 break;
709 pidx = got_packidx_get_object_idx(packidx,
710 pcommit->tree_id);
711 if (pidx == -1)
712 break;
714 err = open_tree(&buf, &entries, &nentries, pack,
715 packidx, idx, commit->tree_id, objcache);
716 if (err)
717 goto done;
718 err = open_tree(&pbuf, &pentries, &pnentries, pack,
719 packidx, pidx, pcommit->tree_id, objcache);
720 if (err) {
721 free(buf);
722 goto done;
725 err = tree_path_changed(&changed, &buf, &pbuf,
726 &entries, &nentries, &pentries, &pnentries, path,
727 pack, packidx, ibuf, objcache);
729 free(entries);
730 entries = NULL;
731 nentries = 0;
732 free(buf);
733 free(pentries);
734 pentries = NULL;
735 pnentries = 0;
736 free(pbuf);
737 if (err) {
738 if (err->code != GOT_ERR_NO_OBJ)
739 goto done;
740 err = NULL;
741 break;
745 if (!changed) {
746 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
747 got_object_commit_close(commit);
748 commit = pcommit;
749 pcommit = NULL;
751 } while (!changed);
753 if (ncommits > 0) {
754 err = send_traversed_commits(commit_ids, ncommits, ibuf);
755 if (err)
756 goto done;
758 if (changed) {
759 err = got_privsep_send_commit(ibuf, commit);
760 if (err)
761 goto done;
764 err = send_commit_traversal_done(ibuf);
765 done:
766 free(commit_ids);
767 if (commit)
768 got_object_commit_close(commit);
769 if (pcommit)
770 got_object_commit_close(pcommit);
771 free(entries);
772 free(pentries);
773 if (err) {
774 if (err->code == GOT_ERR_PRIVSEP_PIPE)
775 err = NULL;
776 else
777 got_privsep_send_error(ibuf, err);
780 return err;
783 static const struct got_error *
784 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
785 struct got_pack *pack, struct got_packidx *packidx,
786 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
788 const struct got_error *err = NULL;
789 uint8_t *buf = NULL;
790 uint64_t size = 0;
791 FILE *outfile = NULL;
792 struct got_imsg_packed_object iobj;
793 struct got_object *obj;
794 struct got_object_id id;
795 size_t datalen;
797 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
798 if (datalen != sizeof(iobj))
799 return got_error(GOT_ERR_PRIVSEP_LEN);
800 memcpy(&iobj, imsg->data, sizeof(iobj));
801 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
803 obj = got_object_cache_get(objcache, &id);
804 if (obj) {
805 obj->refcnt++;
806 } else {
807 err = open_object(&obj, pack, packidx, iobj.idx, &id,
808 objcache);
809 if (err)
810 return err;
813 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
814 if (err)
815 return err;
817 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
818 err = got_pack_get_max_delta_object_size(&size, obj, pack);
819 if (err)
820 goto done;
821 } else
822 size = obj->size;
824 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
825 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
826 obj, pack);
827 else
828 err = got_packfile_extract_object(pack, obj, outfile, basefile,
829 accumfile);
830 if (err)
831 goto done;
833 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
834 done:
835 free(buf);
836 if (outfile && fclose(outfile) == EOF && err == NULL)
837 err = got_error_from_errno("fclose");
838 got_object_close(obj);
839 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
840 got_privsep_send_error(ibuf, err);
842 return err;
845 static const struct got_error *
846 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
847 off_t base_offset)
849 const struct got_error *err;
850 int idx;
852 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
853 if (err)
854 return err;
855 if (idx == -1)
856 return got_error(GOT_ERR_BAD_PACKIDX);
858 return got_packidx_get_object_id(base_id, packidx, idx);
861 static const struct got_error *
862 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
863 FILE *delta_outfile, struct got_pack *pack,
864 struct got_packidx *packidx)
866 const struct got_error *err = NULL;
867 struct got_imsg_raw_delta_request req;
868 size_t datalen, delta_size, delta_compressed_size;
869 off_t delta_offset;
870 uint8_t *delta_buf = NULL;
871 struct got_object_id id, base_id;
872 off_t base_offset, delta_out_offset = 0;
873 uint64_t base_size = 0, result_size = 0;
874 size_t w;
876 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
877 if (datalen != sizeof(req))
878 return got_error(GOT_ERR_PRIVSEP_LEN);
879 memcpy(&req, imsg->data, sizeof(req));
880 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
882 imsg->fd = -1;
884 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
885 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
886 &base_size, &result_size, pack, packidx, req.idx);
887 if (err)
888 goto done;
890 /*
891 * If this is an offset delta we must determine the base
892 * object ID ourselves.
893 */
894 if (base_offset != 0) {
895 err = get_base_object_id(&base_id, packidx, base_offset);
896 if (err)
897 goto done;
900 delta_out_offset = ftello(delta_outfile);
901 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
902 if (w != delta_compressed_size) {
903 err = got_ferror(delta_outfile, GOT_ERR_IO);
904 goto done;
906 if (fflush(delta_outfile) == -1) {
907 err = got_error_from_errno("fflush");
908 goto done;
911 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
912 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
913 &base_id);
914 done:
915 free(delta_buf);
916 return err;
919 struct search_deltas_arg {
920 struct imsgbuf *ibuf;
921 struct got_packidx *packidx;
922 struct got_pack *pack;
923 struct got_object_idset *idset;
924 FILE *delta_outfile;
925 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
926 size_t ndeltas;
927 };
929 static const struct got_error *
930 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
932 const struct got_error *err;
933 struct search_deltas_arg *a = arg;
934 int obj_idx;
935 uint8_t *delta_buf = NULL;
936 uint64_t base_size, result_size;
937 size_t delta_size, delta_compressed_size;
938 off_t delta_offset, base_offset;
939 struct got_object_id base_id;
941 if (sigint_received)
942 return got_error(GOT_ERR_CANCELLED);
944 obj_idx = got_packidx_get_object_idx(a->packidx, id);
945 if (obj_idx == -1)
946 return NULL; /* object not present in our pack file */
948 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
949 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
950 &base_size, &result_size, a->pack, a->packidx, obj_idx);
951 if (err) {
952 if (err->code == GOT_ERR_OBJ_TYPE)
953 return NULL; /* object not stored as a delta */
954 return err;
957 /*
958 * If this is an offset delta we must determine the base
959 * object ID ourselves.
960 */
961 if (base_offset != 0) {
962 err = get_base_object_id(&base_id, a->packidx, base_offset);
963 if (err)
964 goto done;
967 if (got_object_idset_contains(a->idset, &base_id)) {
968 struct got_imsg_reused_delta *delta;
969 off_t delta_out_offset = ftello(a->delta_outfile);
970 size_t w;
972 w = fwrite(delta_buf, 1, delta_compressed_size,
973 a->delta_outfile);
974 if (w != delta_compressed_size) {
975 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
976 goto done;
979 delta = &a->deltas[a->ndeltas++];
980 memcpy(&delta->id, id, sizeof(delta->id));
981 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
982 delta->base_size = base_size;
983 delta->result_size = result_size;
984 delta->delta_size = delta_size;
985 delta->delta_compressed_size = delta_compressed_size;
986 delta->delta_offset = delta_offset;
987 delta->delta_out_offset = delta_out_offset;
989 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
990 err = got_privsep_send_reused_deltas(a->ibuf,
991 a->deltas, a->ndeltas);
992 if (err)
993 goto done;
994 a->ndeltas = 0;
997 done:
998 free(delta_buf);
999 return err;
1002 static const struct got_error *
1003 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1005 const struct got_error *err = NULL;
1006 int done = 0;
1007 struct got_object_id *ids;
1008 size_t nids, i;
1010 for (;;) {
1011 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1012 if (err || done)
1013 break;
1014 for (i = 0; i < nids; i++) {
1015 err = got_object_idset_add(idset, &ids[i], NULL);
1016 if (err) {
1017 free(ids);
1018 return err;
1021 free(ids);
1024 return err;
1027 static const struct got_error *
1028 recv_object_id_queue(struct got_object_id_queue *queue, struct imsgbuf *ibuf)
1030 const struct got_error *err = NULL;
1031 int done = 0;
1032 struct got_object_qid *qid;
1033 struct got_object_id *ids;
1034 size_t nids, i;
1036 for (;;) {
1037 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1038 if (err || done)
1039 break;
1040 for (i = 0; i < nids; i++) {
1041 err = got_object_qid_alloc_partial(&qid);
1042 if (err)
1043 return err;
1044 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1045 STAILQ_INSERT_TAIL(queue, qid, entry);
1049 return err;
1052 static const struct got_error *
1053 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1054 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1056 const struct got_error *err = NULL;
1057 struct got_object_idset *idset;
1058 struct search_deltas_arg sda;
1060 idset = got_object_idset_alloc();
1061 if (idset == NULL)
1062 return got_error_from_errno("got_object_idset_alloc");
1064 err = recv_object_ids(idset, ibuf);
1065 if (err)
1066 return err;
1068 memset(&sda, 0, sizeof(sda));
1069 sda.ibuf = ibuf;
1070 sda.idset = idset;
1071 sda.pack = pack;
1072 sda.packidx = packidx;
1073 sda.delta_outfile = delta_outfile;
1074 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1075 if (err)
1076 goto done;
1078 if (sda.ndeltas > 0) {
1079 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1080 sda.ndeltas);
1081 if (err)
1082 goto done;
1085 if (fflush(delta_outfile) == -1) {
1086 err = got_error_from_errno("fflush");
1087 goto done;
1090 err = got_privsep_send_reused_deltas_done(ibuf);
1091 done:
1092 got_object_idset_free(idset);
1093 return err;
1096 static const struct got_error *
1097 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1099 const struct got_error *err = NULL;
1100 struct imsg imsg;
1101 struct got_imsg_packidx ipackidx;
1102 size_t datalen;
1103 struct got_packidx *p;
1105 *packidx = NULL;
1107 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1108 if (err)
1109 return err;
1111 p = calloc(1, sizeof(*p));
1112 if (p == NULL) {
1113 err = got_error_from_errno("calloc");
1114 goto done;
1117 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1118 err = got_error(GOT_ERR_PRIVSEP_MSG);
1119 goto done;
1122 if (imsg.fd == -1) {
1123 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1124 goto done;
1127 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1128 if (datalen != sizeof(ipackidx)) {
1129 err = got_error(GOT_ERR_PRIVSEP_LEN);
1130 goto done;
1132 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1134 p->len = ipackidx.len;
1135 p->fd = dup(imsg.fd);
1136 if (p->fd == -1) {
1137 err = got_error_from_errno("dup");
1138 goto done;
1140 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1141 err = got_error_from_errno("lseek");
1142 goto done;
1145 #ifndef GOT_PACK_NO_MMAP
1146 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1147 if (p->map == MAP_FAILED)
1148 p->map = NULL; /* fall back to read(2) */
1149 #endif
1150 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1151 done:
1152 if (err) {
1153 if (imsg.fd != -1)
1154 close(imsg.fd);
1155 got_packidx_close(p);
1156 } else
1157 *packidx = p;
1158 imsg_free(&imsg);
1159 return err;
1162 static const struct got_error *
1163 send_tree_enumeration_done(struct imsgbuf *ibuf)
1165 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1166 NULL, 0) == -1)
1167 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1169 return got_privsep_flush_imsg(ibuf);
1172 struct enumerated_tree {
1173 struct got_object_id id;
1174 char *path;
1175 uint8_t *buf;
1176 struct got_parsed_tree_entry *entries;
1177 int nentries;
1180 static const struct got_error *
1181 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1182 struct got_object_id *tree_id,
1183 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1184 struct got_object_cache *objcache, struct got_object_idset *idset,
1185 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1187 const struct got_error *err = NULL;
1188 struct got_object_id_queue ids;
1189 struct got_object_qid *qid;
1190 uint8_t *buf = NULL;
1191 struct got_parsed_tree_entry *entries = NULL;
1192 int nentries = 0, i;
1193 struct enumerated_tree *tree;
1195 *ntrees = 0;
1196 *have_all_entries = 1;
1197 STAILQ_INIT(&ids);
1199 err = got_object_qid_alloc_partial(&qid);
1200 if (err)
1201 return err;
1202 memcpy(&qid->id.sha1, tree_id, SHA1_DIGEST_LENGTH);
1203 qid->data = strdup(path);
1204 if (qid->data == NULL) {
1205 err = got_error_from_errno("strdup");
1206 goto done;
1208 STAILQ_INSERT_TAIL(&ids, qid, entry);
1209 qid = NULL;
1211 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1212 do {
1213 const char *path;
1214 int idx, i;
1216 if (sigint_received) {
1217 err = got_error(GOT_ERR_CANCELLED);
1218 goto done;
1221 qid = STAILQ_FIRST(&ids);
1222 STAILQ_REMOVE_HEAD(&ids, entry);
1223 path = qid->data;
1225 idx = got_packidx_get_object_idx(packidx, &qid->id);
1226 if (idx == -1) {
1227 *have_all_entries = 0;
1228 break;
1231 err = open_tree(&buf, &entries, &nentries,
1232 pack, packidx, idx, &qid->id, objcache);
1233 if (err) {
1234 if (err->code != GOT_ERR_NO_OBJ)
1235 goto done;
1238 err = got_object_idset_add(idset, &qid->id, NULL);
1239 if (err)
1240 goto done;
1242 for (i = 0; i < nentries; i++) {
1243 struct got_object_qid *eqid = NULL;
1244 struct got_parsed_tree_entry *pte = &entries[i];
1245 char *p;
1247 if (!S_ISDIR(pte->mode))
1248 continue;
1250 err = got_object_qid_alloc_partial(&eqid);
1251 if (err)
1252 goto done;
1253 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1255 if (got_object_idset_contains(idset, &eqid->id)) {
1256 got_object_qid_free(eqid);
1257 continue;
1260 if (asprintf(&p, "%s%s%s", path,
1261 got_path_is_root_dir(path) ? "" : "/",
1262 pte->name) == -1) {
1263 err = got_error_from_errno("asprintf");
1264 got_object_qid_free(eqid);
1265 goto done;
1267 eqid->data = p;
1268 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1271 if (*ntrees >= *nalloc) {
1272 struct enumerated_tree *new;
1273 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1274 sizeof(*new));
1275 if (new == NULL) {
1276 err = got_error_from_errno("malloc");
1277 goto done;
1279 *trees = new;
1280 *nalloc += 16;
1282 tree = &(*trees)[*ntrees];
1283 (*ntrees)++;
1284 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1285 tree->path = qid->data;
1286 tree->buf = buf;
1287 buf = NULL;
1288 tree->entries = entries;
1289 entries = NULL;
1290 tree->nentries = nentries;
1292 got_object_qid_free(qid);
1293 qid = NULL;
1294 } while (!STAILQ_EMPTY(&ids));
1296 if (*have_all_entries) {
1297 int i;
1299 * We have managed to traverse all entries in the hierarchy.
1300 * Tell the main process what we have found.
1302 for (i = 0; i < *ntrees; i++) {
1303 tree = &(*trees)[i];
1304 err = got_privsep_send_enumerated_tree(totlen,
1305 ibuf, &tree->id, tree->path, tree->entries,
1306 tree->nentries);
1307 if (err)
1308 goto done;
1309 free(tree->buf);
1310 tree->buf = NULL;
1311 free(tree->path);
1312 tree->path = NULL;
1313 free(tree->entries);
1314 tree->entries = NULL;
1316 *ntrees = 0; /* don't loop again below to free memory */
1318 err = send_tree_enumeration_done(ibuf);
1319 } else {
1321 * We can only load fully packed tree hierarchies on
1322 * behalf of the main process, otherwise the main process
1323 * gets a wrong idea about which tree objects have
1324 * already been traversed.
1325 * Indicate a missing entry for the root of this tree.
1326 * The main process should continue by loading this
1327 * entire tree the slow way.
1329 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1330 tree_id, "/", NULL, -1);
1331 if (err)
1332 goto done;
1334 done:
1335 free(buf);
1336 free(entries);
1337 for (i = 0; i < *ntrees; i++) {
1338 tree = &(*trees)[i];
1339 free(tree->buf);
1340 tree->buf = NULL;
1341 free(tree->path);
1342 tree->path = NULL;
1343 free(tree->entries);
1344 tree->entries = NULL;
1346 if (qid)
1347 free(qid->data);
1348 got_object_qid_free(qid);
1349 got_object_id_queue_free(&ids);
1350 if (err) {
1351 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1352 err = NULL;
1353 else
1354 got_privsep_send_error(ibuf, err);
1357 return err;
1360 static const struct got_error *
1361 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1362 struct got_pack *pack, struct got_packidx *packidx,
1363 struct got_object_cache *objcache)
1365 const struct got_error *err = NULL;
1366 struct got_object_id_queue commit_ids;
1367 const struct got_object_id_queue *parents = NULL;
1368 struct got_object_qid *qid = NULL;
1369 struct got_object *obj = NULL;
1370 struct got_commit_object *commit = NULL;
1371 struct got_object_id *tree_id = NULL;
1372 size_t totlen = 0;
1373 struct got_object_idset *idset;
1374 int i, idx, have_all_entries = 1;
1375 struct enumerated_tree *trees = NULL;
1376 size_t ntrees = 0, nalloc = 16;
1378 STAILQ_INIT(&commit_ids);
1380 trees = calloc(nalloc, sizeof(*trees));
1381 if (trees == NULL)
1382 return got_error_from_errno("calloc");
1384 idset = got_object_idset_alloc();
1385 if (idset == NULL) {
1386 err = got_error_from_errno("got_object_idset_alloc");
1387 goto done;
1390 err = recv_object_id_queue(&commit_ids, ibuf);
1391 if (err)
1392 goto done;
1394 if (STAILQ_EMPTY(&commit_ids)) {
1395 err = got_error(GOT_ERR_PRIVSEP_MSG);
1396 goto done;
1399 err = recv_object_ids(idset, ibuf);
1400 if (err)
1401 goto done;
1403 while (!STAILQ_EMPTY(&commit_ids)) {
1404 if (sigint_received) {
1405 err = got_error(GOT_ERR_CANCELLED);
1406 goto done;
1409 qid = STAILQ_FIRST(&commit_ids);
1410 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1412 if (got_object_idset_contains(idset, &qid->id)) {
1413 got_object_qid_free(qid);
1414 qid = NULL;
1415 continue;
1418 idx = got_packidx_get_object_idx(packidx, &qid->id);
1419 if (idx == -1) {
1420 have_all_entries = 0;
1421 break;
1424 err = open_object(&obj, pack, packidx, idx, &qid->id,
1425 objcache);
1426 if (err)
1427 goto done;
1428 if (obj->type == GOT_OBJ_TYPE_TAG) {
1429 struct got_tag_object *tag;
1430 uint8_t *buf;
1431 size_t len;
1432 err = got_packfile_extract_object_to_mem(&buf,
1433 &len, obj, pack);
1434 if (err)
1435 goto done;
1436 obj->size = len;
1437 err = got_object_parse_tag(&tag, buf, len);
1438 if (err) {
1439 free(buf);
1440 goto done;
1442 idx = got_packidx_get_object_idx(packidx, &tag->id);
1443 if (idx == -1) {
1444 have_all_entries = 0;
1445 break;
1447 err = open_commit(&commit, pack, packidx, idx,
1448 &tag->id, objcache);
1449 got_object_tag_close(tag);
1450 free(buf);
1451 if (err)
1452 goto done;
1453 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1454 err = open_commit(&commit, pack, packidx, idx,
1455 &qid->id, objcache);
1456 if (err)
1457 goto done;
1458 } else {
1459 err = got_error(GOT_ERR_OBJ_TYPE);
1460 goto done;
1462 got_object_close(obj);
1463 obj = NULL;
1465 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1466 got_object_commit_get_committer_time(commit));
1467 if (err)
1468 goto done;
1470 tree_id = got_object_commit_get_tree_id(commit);
1471 idx = got_packidx_get_object_idx(packidx, tree_id);
1472 if (idx == -1) {
1473 have_all_entries = 0;
1474 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1475 tree_id, "/", NULL, -1);
1476 if (err)
1477 goto done;
1478 break;
1481 if (got_object_idset_contains(idset, tree_id)) {
1482 got_object_qid_free(qid);
1483 qid = NULL;
1484 continue;
1487 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1488 tree_id, "/", pack, packidx, objcache, idset,
1489 &trees, &nalloc, &ntrees);
1490 if (err)
1491 goto done;
1493 if (!have_all_entries)
1494 break;
1496 got_object_qid_free(qid);
1497 qid = NULL;
1499 parents = got_object_commit_get_parent_ids(commit);
1500 if (parents) {
1501 struct got_object_qid *pid;
1502 STAILQ_FOREACH(pid, parents, entry) {
1503 if (got_object_idset_contains(idset, &pid->id))
1504 continue;
1505 err = got_object_qid_alloc_partial(&qid);
1506 if (err)
1507 goto done;
1508 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1509 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1510 qid = NULL;
1514 got_object_commit_close(commit);
1515 commit = NULL;
1518 if (have_all_entries) {
1519 err = got_privsep_send_object_enumeration_done(ibuf);
1520 if (err)
1521 goto done;
1522 } else {
1523 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1524 if (err)
1525 goto done;
1527 done:
1528 if (obj)
1529 got_object_close(obj);
1530 if (commit)
1531 got_object_commit_close(commit);
1532 got_object_qid_free(qid);
1533 got_object_id_queue_free(&commit_ids);
1534 if (idset)
1535 got_object_idset_free(idset);
1536 for (i = 0; i < ntrees; i++) {
1537 struct enumerated_tree *tree = &trees[i];
1538 free(tree->buf);
1539 free(tree->path);
1540 free(tree->entries);
1542 free(trees);
1543 return err;
1546 enum findtwixt_color {
1547 COLOR_KEEP = 0,
1548 COLOR_DROP,
1549 COLOR_SKIP,
1550 COLOR_MAX,
1553 static const struct got_error *
1554 paint_commit(struct got_object_qid *qid, intptr_t color)
1556 if (color < 0 || color >= COLOR_MAX)
1557 return got_error(GOT_ERR_RANGE);
1559 qid->data = (void *)color;
1560 return NULL;
1563 static const struct got_error *
1564 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1565 intptr_t color)
1567 const struct got_error *err;
1568 struct got_object_qid *qid;
1570 err = got_object_qid_alloc_partial(&qid);
1571 if (err)
1572 return err;
1574 memcpy(&qid->id, id, sizeof(qid->id));
1575 STAILQ_INSERT_TAIL(ids, qid, entry);
1576 return paint_commit(qid, color);
1579 static const struct got_error *
1580 paint_commits(struct got_object_id_queue *ids, int *nids,
1581 struct got_object_idset *keep, struct got_object_idset *drop,
1582 struct got_object_idset *skip, struct got_pack *pack,
1583 struct got_packidx *packidx, struct imsgbuf *ibuf,
1584 struct got_object_cache *objcache)
1586 const struct got_error *err = NULL;
1587 struct got_commit_object *commit = NULL;
1588 struct got_object_id_queue painted;
1589 const struct got_object_id_queue *parents;
1590 struct got_object_qid *qid = NULL;
1591 int nqueued = *nids, nskip = 0, npainted = 0;
1593 STAILQ_INIT(&painted);
1595 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1596 int idx;
1597 intptr_t color;
1599 if (sigint_received) {
1600 err = got_error(GOT_ERR_CANCELLED);
1601 goto done;
1604 qid = STAILQ_FIRST(ids);
1605 idx = got_packidx_get_object_idx(packidx, &qid->id);
1606 if (idx == -1) {
1607 qid = NULL;
1608 break;
1611 STAILQ_REMOVE_HEAD(ids, entry);
1612 nqueued--;
1613 color = (intptr_t)qid->data;
1614 if (color == COLOR_SKIP)
1615 nskip--;
1617 if (got_object_idset_contains(skip, &qid->id)) {
1618 got_object_qid_free(qid);
1619 qid = NULL;
1620 continue;
1623 switch (color) {
1624 case COLOR_KEEP:
1625 if (got_object_idset_contains(keep, &qid->id)) {
1626 got_object_qid_free(qid);
1627 qid = NULL;
1628 continue;
1630 if (got_object_idset_contains(drop, &qid->id)) {
1631 err = paint_commit(qid, COLOR_SKIP);
1632 if (err)
1633 goto done;
1635 err = got_object_idset_add(keep, &qid->id, NULL);
1636 if (err)
1637 goto done;
1638 break;
1639 case COLOR_DROP:
1640 if (got_object_idset_contains(drop, &qid->id)) {
1641 got_object_qid_free(qid);
1642 qid = NULL;
1643 continue;
1645 if (got_object_idset_contains(keep, &qid->id)) {
1646 err = paint_commit(qid, COLOR_SKIP);
1647 if (err)
1648 goto done;
1650 err = got_object_idset_add(drop, &qid->id, NULL);
1651 if (err)
1652 goto done;
1653 break;
1654 case COLOR_SKIP:
1655 if (!got_object_idset_contains(skip, &qid->id)) {
1656 err = got_object_idset_add(skip, &qid->id,
1657 NULL);
1658 if (err)
1659 goto done;
1661 break;
1662 default:
1663 /* should not happen */
1664 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1665 "%s invalid commit color %d", __func__, color);
1666 goto done;
1669 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1670 objcache);
1671 if (err)
1672 goto done;
1674 parents = got_object_commit_get_parent_ids(commit);
1675 if (parents) {
1676 struct got_object_qid *pid;
1677 color = (intptr_t)qid->data;
1678 STAILQ_FOREACH(pid, parents, entry) {
1679 err = queue_commit_id(ids, &pid->id, color);
1680 if (err)
1681 goto done;
1682 nqueued++;
1683 if (color == COLOR_SKIP)
1684 nskip++;
1688 got_object_commit_close(commit);
1689 commit = NULL;
1691 STAILQ_INSERT_TAIL(&painted, qid, entry);
1692 qid = NULL;
1693 npainted++;
1695 err = got_privsep_send_painted_commits(ibuf, &painted,
1696 &npainted, 1, 0);
1697 if (err)
1698 goto done;
1701 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1702 if (err)
1703 goto done;
1705 *nids = nqueued;
1706 done:
1707 if (commit)
1708 got_object_commit_close(commit);
1709 got_object_qid_free(qid);
1710 return err;
1713 static void
1714 commit_painting_free(struct got_object_idset **keep,
1715 struct got_object_idset **drop,
1716 struct got_object_idset **skip)
1718 if (*keep) {
1719 got_object_idset_free(*keep);
1720 *keep = NULL;
1722 if (*drop) {
1723 got_object_idset_free(*drop);
1724 *drop = NULL;
1726 if (*skip) {
1727 got_object_idset_free(*skip);
1728 *skip = NULL;
1732 static const struct got_error *
1733 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1734 struct got_object_idset **drop, struct got_object_idset **skip)
1736 const struct got_error *err = NULL;
1738 *keep = got_object_idset_alloc();
1739 if (*keep == NULL) {
1740 err = got_error_from_errno("got_object_idset_alloc");
1741 goto done;
1743 *drop = got_object_idset_alloc();
1744 if (*drop == NULL) {
1745 err = got_error_from_errno("got_object_idset_alloc");
1746 goto done;
1748 *skip = got_object_idset_alloc();
1749 if (*skip == NULL) {
1750 err = got_error_from_errno("got_object_idset_alloc");
1751 goto done;
1754 err = recv_object_ids(*keep, ibuf);
1755 if (err)
1756 goto done;
1757 err = recv_object_ids(*drop, ibuf);
1758 if (err)
1759 goto done;
1760 err = recv_object_ids(*skip, ibuf);
1761 if (err)
1762 goto done;
1764 done:
1765 if (err)
1766 commit_painting_free(keep, drop, skip);
1768 return err;
1771 static const struct got_error *
1772 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1773 struct got_pack *pack, struct got_packidx *packidx,
1774 struct got_object_cache *objcache, struct got_object_idset *keep,
1775 struct got_object_idset *drop, struct got_object_idset *skip)
1777 const struct got_error *err = NULL;
1778 struct got_imsg_commit_painting_request ireq;
1779 struct got_object_id id;
1780 size_t datalen;
1781 struct got_object_id_queue ids;
1782 int nids = 0;
1784 STAILQ_INIT(&ids);
1786 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1787 if (datalen != sizeof(ireq))
1788 return got_error(GOT_ERR_PRIVSEP_LEN);
1789 memcpy(&ireq, imsg->data, sizeof(ireq));
1790 memcpy(id.sha1, ireq.id, SHA1_DIGEST_LENGTH);
1792 err = queue_commit_id(&ids, &id, ireq.color);
1793 if (err)
1794 return err;
1795 nids = 1;
1797 err = paint_commits(&ids, &nids, keep, drop, skip,
1798 pack, packidx, ibuf, objcache);
1799 if (err)
1800 goto done;
1802 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1803 if (err)
1804 goto done;
1806 err = got_privsep_send_painting_commits_done(ibuf);
1807 done:
1808 got_object_id_queue_free(&ids);
1809 return err;
1812 static const struct got_error *
1813 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1815 const struct got_error *err = NULL;
1816 struct imsg imsg;
1817 struct got_imsg_pack ipack;
1818 size_t datalen;
1819 struct got_pack *pack;
1821 *packp = NULL;
1823 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1824 if (err)
1825 return err;
1827 pack = calloc(1, sizeof(*pack));
1828 if (pack == NULL) {
1829 err = got_error_from_errno("calloc");
1830 goto done;
1833 if (imsg.hdr.type != GOT_IMSG_PACK) {
1834 err = got_error(GOT_ERR_PRIVSEP_MSG);
1835 goto done;
1838 if (imsg.fd == -1) {
1839 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1840 goto done;
1843 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1844 if (datalen != sizeof(ipack)) {
1845 err = got_error(GOT_ERR_PRIVSEP_LEN);
1846 goto done;
1848 memcpy(&ipack, imsg.data, sizeof(ipack));
1850 pack->filesize = ipack.filesize;
1851 pack->fd = dup(imsg.fd);
1852 if (pack->fd == -1) {
1853 err = got_error_from_errno("dup");
1854 goto done;
1856 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1857 err = got_error_from_errno("lseek");
1858 goto done;
1860 pack->path_packfile = strdup(ipack.path_packfile);
1861 if (pack->path_packfile == NULL) {
1862 err = got_error_from_errno("strdup");
1863 goto done;
1866 err = got_delta_cache_alloc(&pack->delta_cache);
1867 if (err)
1868 goto done;
1870 #ifndef GOT_PACK_NO_MMAP
1871 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1872 pack->fd, 0);
1873 if (pack->map == MAP_FAILED)
1874 pack->map = NULL; /* fall back to read(2) */
1875 #endif
1876 done:
1877 if (err) {
1878 if (imsg.fd != -1)
1879 close(imsg.fd);
1880 free(pack);
1881 } else
1882 *packp = pack;
1883 imsg_free(&imsg);
1884 return err;
1887 int
1888 main(int argc, char *argv[])
1890 const struct got_error *err = NULL;
1891 struct imsgbuf ibuf;
1892 struct imsg imsg;
1893 struct got_packidx *packidx = NULL;
1894 struct got_pack *pack = NULL;
1895 struct got_object_cache objcache;
1896 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1897 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1899 //static int attached;
1900 //while (!attached) sleep(1);
1902 signal(SIGINT, catch_sigint);
1904 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1906 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1907 if (err) {
1908 err = got_error_from_errno("got_object_cache_init");
1909 got_privsep_send_error(&ibuf, err);
1910 return 1;
1913 #ifndef PROFILE
1914 /* revoke access to most system calls */
1915 if (pledge("stdio recvfd", NULL) == -1) {
1916 err = got_error_from_errno("pledge");
1917 got_privsep_send_error(&ibuf, err);
1918 return 1;
1921 /* revoke fs access */
1922 if (landlock_no_fs() == -1) {
1923 err = got_error_from_errno("landlock_no_fs");
1924 got_privsep_send_error(&ibuf, err);
1925 return 1;
1927 if (cap_enter() == -1) {
1928 err = got_error_from_errno("cap_enter");
1929 got_privsep_send_error(&ibuf, err);
1930 return 1;
1932 #endif
1934 err = receive_packidx(&packidx, &ibuf);
1935 if (err) {
1936 got_privsep_send_error(&ibuf, err);
1937 return 1;
1940 err = receive_pack(&pack, &ibuf);
1941 if (err) {
1942 got_privsep_send_error(&ibuf, err);
1943 return 1;
1946 for (;;) {
1947 imsg.fd = -1;
1949 if (sigint_received) {
1950 err = got_error(GOT_ERR_CANCELLED);
1951 break;
1954 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1955 if (err) {
1956 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1957 err = NULL;
1958 break;
1961 if (imsg.hdr.type == GOT_IMSG_STOP)
1962 break;
1964 switch (imsg.hdr.type) {
1965 case GOT_IMSG_TMPFD:
1966 if (basefile == NULL) {
1967 err = receive_tempfile(&basefile, "w+",
1968 &imsg, &ibuf);
1969 } else if (accumfile == NULL) {
1970 err = receive_tempfile(&accumfile, "w+",
1971 &imsg, &ibuf);
1972 } else
1973 err = got_error(GOT_ERR_PRIVSEP_MSG);
1974 break;
1975 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1976 err = object_request(&imsg, &ibuf, pack, packidx,
1977 &objcache);
1978 break;
1979 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1980 if (basefile == NULL || accumfile == NULL) {
1981 err = got_error(GOT_ERR_PRIVSEP_MSG);
1982 break;
1984 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1985 &objcache, basefile, accumfile);
1986 break;
1987 case GOT_IMSG_RAW_DELTA_OUTFD:
1988 if (delta_outfile != NULL) {
1989 err = got_error(GOT_ERR_PRIVSEP_MSG);
1990 break;
1992 err = receive_tempfile(&delta_outfile, "w",
1993 &imsg, &ibuf);
1994 break;
1995 case GOT_IMSG_RAW_DELTA_REQUEST:
1996 if (delta_outfile == NULL) {
1997 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1998 break;
2000 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2001 pack, packidx);
2002 break;
2003 case GOT_IMSG_DELTA_REUSE_REQUEST:
2004 if (delta_outfile == NULL) {
2005 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2006 break;
2008 err = delta_reuse_request(&imsg, &ibuf,
2009 delta_outfile, pack, packidx);
2010 break;
2011 case GOT_IMSG_COMMIT_REQUEST:
2012 err = commit_request(&imsg, &ibuf, pack, packidx,
2013 &objcache);
2014 break;
2015 case GOT_IMSG_TREE_REQUEST:
2016 err = tree_request(&imsg, &ibuf, pack, packidx,
2017 &objcache);
2018 break;
2019 case GOT_IMSG_BLOB_REQUEST:
2020 if (basefile == NULL || accumfile == NULL) {
2021 err = got_error(GOT_ERR_PRIVSEP_MSG);
2022 break;
2024 err = blob_request(&imsg, &ibuf, pack, packidx,
2025 &objcache, basefile, accumfile);
2026 break;
2027 case GOT_IMSG_TAG_REQUEST:
2028 err = tag_request(&imsg, &ibuf, pack, packidx,
2029 &objcache);
2030 break;
2031 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2032 err = commit_traversal_request(&imsg, &ibuf, pack,
2033 packidx, &objcache);
2034 break;
2035 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2036 err = enumeration_request(&imsg, &ibuf, pack,
2037 packidx, &objcache);
2038 break;
2039 case GOT_IMSG_COMMIT_PAINTING_INIT:
2040 commit_painting_free(&keep, &drop, &skip);
2041 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2042 break;
2043 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2044 if (keep == NULL || drop == NULL || skip == NULL) {
2045 err = got_error(GOT_ERR_PRIVSEP_MSG);
2046 break;
2048 err = commit_painting_request(&imsg, &ibuf, pack,
2049 packidx, &objcache, keep, drop, skip);
2050 break;
2051 case GOT_IMSG_COMMIT_PAINTING_DONE:
2052 commit_painting_free(&keep, &drop, &skip);
2053 break;
2054 default:
2055 err = got_error(GOT_ERR_PRIVSEP_MSG);
2056 break;
2059 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
2060 err = got_error_from_errno("close");
2061 imsg_free(&imsg);
2062 if (err)
2063 break;
2066 commit_painting_free(&keep, &drop, &skip);
2067 if (packidx)
2068 got_packidx_close(packidx);
2069 if (pack)
2070 got_pack_close(pack);
2071 got_object_cache_close(&objcache);
2072 imsg_clear(&ibuf);
2073 if (basefile && fclose(basefile) == EOF && err == NULL)
2074 err = got_error_from_errno("fclose");
2075 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2076 err = got_error_from_errno("fclose");
2077 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2078 err = got_error_from_errno("fclose");
2079 if (err) {
2080 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2081 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2082 got_privsep_send_error(&ibuf, err);
2085 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2086 err = got_error_from_errno("close");
2087 return err ? 1 : 0;