2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 #include <sys/types.h>
32 #include "got_compat.h"
34 #include "got_error.h"
35 #include "got_object.h"
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_object_idset.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_pack.h"
47 static volatile sig_atomic_t sigint_received;
50 catch_sigint(int signo)
55 static const struct got_error *
56 open_object(struct got_object **obj, struct got_pack *pack,
57 struct got_packidx *packidx, int idx, struct got_object_id *id,
58 struct got_object_cache *objcache)
60 const struct got_error *err;
62 err = got_packfile_open_object(obj, pack, packidx, idx, id);
67 err = got_object_cache_add(objcache, id, *obj);
69 if (err->code == GOT_ERR_OBJ_EXISTS ||
70 err->code == GOT_ERR_OBJ_TOO_LARGE)
78 static const struct got_error *
79 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
80 struct got_packidx *packidx, struct got_object_cache *objcache)
82 const struct got_error *err = NULL;
83 struct got_imsg_packed_object iobj;
84 struct got_object *obj;
85 struct got_object_id id;
88 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
89 if (datalen != sizeof(iobj))
90 return got_error(GOT_ERR_PRIVSEP_LEN);
91 memcpy(&iobj, imsg->data, sizeof(iobj));
92 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
94 obj = got_object_cache_get(objcache, &id);
98 err = open_object(&obj, pack, packidx, iobj.idx, &id,
104 err = got_privsep_send_obj(ibuf, obj);
106 got_object_close(obj);
110 static const struct got_error *
111 open_commit(struct got_commit_object **commit, struct got_pack *pack,
112 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
113 struct got_object_cache *objcache)
115 const struct got_error *err = NULL;
116 struct got_object *obj = NULL;
122 obj = got_object_cache_get(objcache, id);
126 err = open_object(&obj, pack, packidx, obj_idx, id,
132 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
138 err = got_object_parse_commit(commit, buf, len);
140 got_object_close(obj);
145 static const struct got_error *
146 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
147 struct got_packidx *packidx, struct got_object_cache *objcache)
149 const struct got_error *err = NULL;
150 struct got_imsg_packed_object iobj;
151 struct got_commit_object *commit = NULL;
152 struct got_object_id id;
155 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
156 if (datalen != sizeof(iobj))
157 return got_error(GOT_ERR_PRIVSEP_LEN);
158 memcpy(&iobj, imsg->data, sizeof(iobj));
159 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
161 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
165 err = got_privsep_send_commit(ibuf, commit);
168 got_object_commit_close(commit);
170 if (err->code == GOT_ERR_PRIVSEP_PIPE)
173 got_privsep_send_error(ibuf, err);
179 static const struct got_error *
180 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, int *nentries,
181 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
182 struct got_object_id *id, struct got_object_cache *objcache)
184 const struct got_error *err = NULL;
185 struct got_object *obj = NULL;
191 obj = got_object_cache_get(objcache, id);
195 err = open_object(&obj, pack, packidx, obj_idx, id,
201 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
207 err = got_object_parse_tree(entries, nentries, *buf, len);
209 got_object_close(obj);
217 static const struct got_error *
218 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
219 struct got_packidx *packidx, struct got_object_cache *objcache)
221 const struct got_error *err = NULL;
222 struct got_imsg_packed_object iobj;
223 struct got_parsed_tree_entry *entries = NULL;
226 struct got_object_id id;
229 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
230 if (datalen != sizeof(iobj))
231 return got_error(GOT_ERR_PRIVSEP_LEN);
232 memcpy(&iobj, imsg->data, sizeof(iobj));
233 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
235 err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
240 err = got_privsep_send_tree(ibuf, entries, nentries);
244 if (err->code == GOT_ERR_PRIVSEP_PIPE)
247 got_privsep_send_error(ibuf, err);
253 static const struct got_error *
254 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
256 const struct got_error *err;
260 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
264 if (imsg.hdr.type != imsg_code) {
265 err = got_error(GOT_ERR_PRIVSEP_MSG);
269 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
271 err = got_error(GOT_ERR_PRIVSEP_LEN);
275 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
279 *f = fdopen(imsg.fd, "w+");
281 err = got_error_from_errno("fdopen");
290 static const struct got_error *
291 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
292 struct imsgbuf *ibuf)
296 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
298 return got_error(GOT_ERR_PRIVSEP_LEN);
301 return got_error(GOT_ERR_PRIVSEP_NO_FD);
303 *f = fdopen(imsg->fd, mode);
305 return got_error_from_errno("fdopen");
311 static const struct got_error *
312 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
313 struct got_packidx *packidx, struct got_object_cache *objcache,
314 FILE *basefile, FILE *accumfile)
316 const struct got_error *err = NULL;
317 struct got_imsg_packed_object iobj;
318 struct got_object *obj = NULL;
319 FILE *outfile = NULL;
320 struct got_object_id id;
325 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
326 if (datalen != sizeof(iobj))
327 return got_error(GOT_ERR_PRIVSEP_LEN);
328 memcpy(&iobj, imsg->data, sizeof(iobj));
329 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
331 obj = got_object_cache_get(objcache, &id);
335 err = open_object(&obj, pack, packidx, iobj.idx, &id,
341 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
345 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
346 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
350 blob_size = obj->size;
352 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
353 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
356 err = got_packfile_extract_object(pack, obj, outfile, basefile,
361 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
364 if (outfile && fclose(outfile) == EOF && err == NULL)
365 err = got_error_from_errno("fclose");
366 got_object_close(obj);
367 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
368 got_privsep_send_error(ibuf, err);
373 static const struct got_error *
374 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
375 struct got_packidx *packidx, struct got_object_cache *objcache)
377 const struct got_error *err = NULL;
378 struct got_imsg_packed_object iobj;
379 struct got_object *obj = NULL;
380 struct got_tag_object *tag = NULL;
383 struct got_object_id id;
386 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
387 if (datalen != sizeof(iobj))
388 return got_error(GOT_ERR_PRIVSEP_LEN);
389 memcpy(&iobj, imsg->data, sizeof(iobj));
390 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
392 obj = got_object_cache_get(objcache, &id);
396 err = open_object(&obj, pack, packidx, iobj.idx, &id,
402 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
407 err = got_object_parse_tag(&tag, buf, len);
411 err = got_privsep_send_tag(ibuf, tag);
414 got_object_close(obj);
416 got_object_tag_close(tag);
418 if (err->code == GOT_ERR_PRIVSEP_PIPE)
421 got_privsep_send_error(ibuf, err);
427 static struct got_parsed_tree_entry *
428 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
429 const char *name, size_t len)
431 struct got_parsed_tree_entry *pte;
434 /* Note that tree entries are sorted in strncmp() order. */
435 for (i = 0; i < nentries; i++) {
437 cmp = strncmp(pte->name, name, len);
442 if (pte->name[len] == '\0')
448 static const struct got_error *
449 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
450 struct got_parsed_tree_entry **entries1, int *nentries1,
451 struct got_parsed_tree_entry **entries2, int *nentries2,
452 const char *path, struct got_pack *pack, struct got_packidx *packidx,
453 struct imsgbuf *ibuf, struct got_object_cache *objcache)
455 const struct got_error *err = NULL;
456 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
462 /* We not do support comparing the root path. */
463 if (got_path_is_root_dir(path))
464 return got_error_path(path, GOT_ERR_BAD_PATH);
479 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
481 err = got_error(GOT_ERR_NO_OBJ);
485 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
491 if (pte1->mode != pte2->mode) {
496 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
501 if (*s == '\0') { /* final path element */
510 struct got_object_id id1, id2;
513 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
514 idx = got_packidx_get_object_idx(packidx, &id1);
516 err = got_error_no_obj(&id1);
523 err = open_tree(buf1, entries1, nentries1, pack,
524 packidx, idx, &id1, objcache);
529 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
530 idx = got_packidx_get_object_idx(packidx, &id2);
532 err = got_error_no_obj(&id2);
539 err = open_tree(buf2, entries2, nentries2, pack,
540 packidx, idx, &id2, objcache);
550 static const struct got_error *
551 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
552 struct imsgbuf *ibuf)
557 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
558 sizeof(struct got_imsg_traversed_commits) +
559 ncommits * SHA1_DIGEST_LENGTH);
561 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
563 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
564 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
566 for (i = 0; i < ncommits; i++) {
567 struct got_object_id *id = &commit_ids[i];
568 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
569 return got_error_from_errno(
570 "imsg_add TRAVERSED_COMMITS");
575 imsg_close(ibuf, wbuf);
577 return got_privsep_flush_imsg(ibuf);
580 static const struct got_error *
581 send_commit_traversal_done(struct imsgbuf *ibuf)
583 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
585 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
587 return got_privsep_flush_imsg(ibuf);
590 static const struct got_error *
591 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
592 struct got_pack *pack, struct got_packidx *packidx,
593 struct got_object_cache *objcache)
595 const struct got_error *err = NULL;
596 struct got_imsg_packed_object iobj;
597 struct got_object_qid *pid;
598 struct got_commit_object *commit = NULL, *pcommit = NULL;
599 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
600 int nentries = 0, pnentries = 0;
601 struct got_object_id id;
602 size_t datalen, path_len;
604 const int min_alloc = 64;
605 int changed = 0, ncommits = 0, nallocated = 0;
606 struct got_object_id *commit_ids = NULL;
608 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
609 if (datalen < sizeof(iobj))
610 return got_error(GOT_ERR_PRIVSEP_LEN);
611 memcpy(&iobj, imsg->data, sizeof(iobj));
612 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
614 path_len = datalen - sizeof(iobj) - 1;
616 return got_error(GOT_ERR_PRIVSEP_LEN);
618 path = imsg->data + sizeof(iobj);
619 if (path[path_len] != '\0')
620 return got_error(GOT_ERR_PRIVSEP_LEN);
623 nallocated = min_alloc;
624 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
625 if (commit_ids == NULL)
626 return got_error_from_errno("reallocarray");
629 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
632 if (sigint_received) {
633 err = got_error(GOT_ERR_CANCELLED);
637 if (commit == NULL) {
638 idx = got_packidx_get_object_idx(packidx, &id);
641 err = open_commit(&commit, pack, packidx,
644 if (err->code != GOT_ERR_NO_OBJ)
651 if (sizeof(struct got_imsg_traversed_commits) +
652 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
653 err = send_traversed_commits(commit_ids, ncommits,
660 if (ncommits > nallocated) {
661 struct got_object_id *new;
662 nallocated += min_alloc;
663 new = reallocarray(commit_ids, nallocated,
664 sizeof(*commit_ids));
666 err = got_error_from_errno("reallocarray");
671 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
674 pid = STAILQ_FIRST(&commit->parent_ids);
678 idx = got_packidx_get_object_idx(packidx, &pid->id);
682 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
685 if (err->code != GOT_ERR_NO_OBJ)
691 if (path[0] == '/' && path[1] == '\0') {
692 if (got_object_id_cmp(pcommit->tree_id,
693 commit->tree_id) != 0) {
699 uint8_t *buf = NULL, *pbuf = NULL;
701 idx = got_packidx_get_object_idx(packidx,
705 pidx = got_packidx_get_object_idx(packidx,
710 err = open_tree(&buf, &entries, &nentries, pack,
711 packidx, idx, commit->tree_id, objcache);
714 err = open_tree(&pbuf, &pentries, &pnentries, pack,
715 packidx, pidx, pcommit->tree_id, objcache);
721 err = tree_path_changed(&changed, &buf, &pbuf,
722 &entries, &nentries, &pentries, &pnentries, path,
723 pack, packidx, ibuf, objcache);
734 if (err->code != GOT_ERR_NO_OBJ)
742 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
743 got_object_commit_close(commit);
750 err = send_traversed_commits(commit_ids, ncommits, ibuf);
755 err = got_privsep_send_commit(ibuf, commit);
760 err = send_commit_traversal_done(ibuf);
764 got_object_commit_close(commit);
766 got_object_commit_close(pcommit);
770 if (err->code == GOT_ERR_PRIVSEP_PIPE)
773 got_privsep_send_error(ibuf, err);
779 static const struct got_error *
780 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
781 struct got_pack *pack, struct got_packidx *packidx,
782 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
784 const struct got_error *err = NULL;
787 FILE *outfile = NULL;
788 struct got_imsg_packed_object iobj;
789 struct got_object *obj;
790 struct got_object_id id;
793 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
794 if (datalen != sizeof(iobj))
795 return got_error(GOT_ERR_PRIVSEP_LEN);
796 memcpy(&iobj, imsg->data, sizeof(iobj));
797 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
799 obj = got_object_cache_get(objcache, &id);
803 err = open_object(&obj, pack, packidx, iobj.idx, &id,
809 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
813 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
814 err = got_pack_get_max_delta_object_size(&size, obj, pack);
820 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
821 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
824 err = got_packfile_extract_object(pack, obj, outfile, basefile,
829 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
832 if (outfile && fclose(outfile) == EOF && err == NULL)
833 err = got_error_from_errno("fclose");
834 got_object_close(obj);
835 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
836 got_privsep_send_error(ibuf, err);
841 static const struct got_error *
842 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
845 const struct got_error *err;
848 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
852 return got_error(GOT_ERR_BAD_PACKIDX);
854 return got_packidx_get_object_id(base_id, packidx, idx);
857 static const struct got_error *
858 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
859 FILE *delta_outfile, struct got_pack *pack,
860 struct got_packidx *packidx)
862 const struct got_error *err = NULL;
863 struct got_imsg_raw_delta_request req;
864 size_t datalen, delta_size, delta_compressed_size;
866 uint8_t *delta_buf = NULL;
867 struct got_object_id id, base_id;
868 off_t base_offset, delta_out_offset = 0;
869 uint64_t base_size = 0, result_size = 0;
872 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
873 if (datalen != sizeof(req))
874 return got_error(GOT_ERR_PRIVSEP_LEN);
875 memcpy(&req, imsg->data, sizeof(req));
876 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
880 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
881 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
882 &base_size, &result_size, pack, packidx, req.idx);
887 * If this is an offset delta we must determine the base
888 * object ID ourselves.
890 if (base_offset != 0) {
891 err = get_base_object_id(&base_id, packidx, base_offset);
896 delta_out_offset = ftello(delta_outfile);
897 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
898 if (w != delta_compressed_size) {
899 err = got_ferror(delta_outfile, GOT_ERR_IO);
902 if (fflush(delta_outfile) == -1) {
903 err = got_error_from_errno("fflush");
907 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
908 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
915 struct search_deltas_arg {
916 struct imsgbuf *ibuf;
917 struct got_packidx *packidx;
918 struct got_pack *pack;
919 struct got_object_idset *idset;
921 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
925 static const struct got_error *
926 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
928 const struct got_error *err;
929 struct search_deltas_arg *a = arg;
931 uint8_t *delta_buf = NULL;
932 uint64_t base_size, result_size;
933 size_t delta_size, delta_compressed_size;
934 off_t delta_offset, base_offset;
935 struct got_object_id base_id;
938 return got_error(GOT_ERR_CANCELLED);
940 obj_idx = got_packidx_get_object_idx(a->packidx, id);
942 return NULL; /* object not present in our pack file */
944 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
945 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
946 &base_size, &result_size, a->pack, a->packidx, obj_idx);
948 if (err->code == GOT_ERR_OBJ_TYPE)
949 return NULL; /* object not stored as a delta */
954 * If this is an offset delta we must determine the base
955 * object ID ourselves.
957 if (base_offset != 0) {
958 err = get_base_object_id(&base_id, a->packidx, base_offset);
963 if (got_object_idset_contains(a->idset, &base_id)) {
964 struct got_imsg_reused_delta *delta;
965 off_t delta_out_offset = ftello(a->delta_outfile);
968 w = fwrite(delta_buf, 1, delta_compressed_size,
970 if (w != delta_compressed_size) {
971 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
975 delta = &a->deltas[a->ndeltas++];
976 memcpy(&delta->id, id, sizeof(delta->id));
977 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
978 delta->base_size = base_size;
979 delta->result_size = result_size;
980 delta->delta_size = delta_size;
981 delta->delta_compressed_size = delta_compressed_size;
982 delta->delta_offset = delta_offset;
983 delta->delta_out_offset = delta_out_offset;
985 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
986 err = got_privsep_send_reused_deltas(a->ibuf,
987 a->deltas, a->ndeltas);
998 static const struct got_error *
999 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1001 const struct got_error *err = NULL;
1003 struct got_object_id *ids;
1007 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1010 for (i = 0; i < nids; i++) {
1011 err = got_object_idset_add(idset, &ids[i], NULL);
1023 static const struct got_error *
1024 recv_object_id_queue(struct got_object_id_queue *queue, struct imsgbuf *ibuf)
1026 const struct got_error *err = NULL;
1028 struct got_object_qid *qid;
1029 struct got_object_id *ids;
1033 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1036 for (i = 0; i < nids; i++) {
1037 err = got_object_qid_alloc_partial(&qid);
1040 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1041 STAILQ_INSERT_TAIL(queue, qid, entry);
1048 static const struct got_error *
1049 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1050 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1052 const struct got_error *err = NULL;
1053 struct got_object_idset *idset;
1054 struct search_deltas_arg sda;
1056 idset = got_object_idset_alloc();
1058 return got_error_from_errno("got_object_idset_alloc");
1060 err = recv_object_ids(idset, ibuf);
1064 memset(&sda, 0, sizeof(sda));
1068 sda.packidx = packidx;
1069 sda.delta_outfile = delta_outfile;
1070 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1074 if (sda.ndeltas > 0) {
1075 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1081 if (fflush(delta_outfile) == -1) {
1082 err = got_error_from_errno("fflush");
1086 err = got_privsep_send_reused_deltas_done(ibuf);
1088 got_object_idset_free(idset);
1092 static const struct got_error *
1093 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1095 const struct got_error *err = NULL;
1097 struct got_imsg_packidx ipackidx;
1099 struct got_packidx *p;
1103 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1107 p = calloc(1, sizeof(*p));
1109 err = got_error_from_errno("calloc");
1113 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1114 err = got_error(GOT_ERR_PRIVSEP_MSG);
1118 if (imsg.fd == -1) {
1119 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1123 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1124 if (datalen != sizeof(ipackidx)) {
1125 err = got_error(GOT_ERR_PRIVSEP_LEN);
1128 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1130 p->len = ipackidx.len;
1131 p->fd = dup(imsg.fd);
1133 err = got_error_from_errno("dup");
1136 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1137 err = got_error_from_errno("lseek");
1141 #ifndef GOT_PACK_NO_MMAP
1142 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1143 if (p->map == MAP_FAILED)
1144 p->map = NULL; /* fall back to read(2) */
1146 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1151 got_packidx_close(p);
1158 static const struct got_error *
1159 send_tree_enumeration_done(struct imsgbuf *ibuf)
1161 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1163 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1165 return got_privsep_flush_imsg(ibuf);
1168 struct enumerated_tree {
1169 struct got_object_id id;
1172 struct got_parsed_tree_entry *entries;
1176 static const struct got_error *
1177 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1178 struct got_object_id *tree_id,
1179 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1180 struct got_object_cache *objcache, struct got_object_idset *idset,
1181 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1183 const struct got_error *err = NULL;
1184 struct got_object_id_queue ids;
1185 struct got_object_qid *qid;
1186 uint8_t *buf = NULL;
1187 struct got_parsed_tree_entry *entries = NULL;
1188 int nentries = 0, i;
1189 struct enumerated_tree *tree;
1192 *have_all_entries = 1;
1195 err = got_object_qid_alloc_partial(&qid);
1198 memcpy(&qid->id.sha1, tree_id, SHA1_DIGEST_LENGTH);
1199 qid->data = strdup(path);
1200 if (qid->data == NULL) {
1201 err = got_error_from_errno("strdup");
1204 STAILQ_INSERT_TAIL(&ids, qid, entry);
1207 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1212 if (sigint_received) {
1213 err = got_error(GOT_ERR_CANCELLED);
1217 qid = STAILQ_FIRST(&ids);
1218 STAILQ_REMOVE_HEAD(&ids, entry);
1221 idx = got_packidx_get_object_idx(packidx, &qid->id);
1223 *have_all_entries = 0;
1227 err = open_tree(&buf, &entries, &nentries,
1228 pack, packidx, idx, &qid->id, objcache);
1230 if (err->code != GOT_ERR_NO_OBJ)
1234 err = got_object_idset_add(idset, &qid->id, NULL);
1238 for (i = 0; i < nentries; i++) {
1239 struct got_object_qid *eqid = NULL;
1240 struct got_parsed_tree_entry *pte = &entries[i];
1243 if (!S_ISDIR(pte->mode))
1246 err = got_object_qid_alloc_partial(&eqid);
1249 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1251 if (got_object_idset_contains(idset, &eqid->id)) {
1252 got_object_qid_free(eqid);
1256 if (asprintf(&p, "%s%s%s", path,
1257 got_path_is_root_dir(path) ? "" : "/",
1259 err = got_error_from_errno("asprintf");
1260 got_object_qid_free(eqid);
1264 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1267 if (*ntrees >= *nalloc) {
1268 struct enumerated_tree *new;
1269 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1272 err = got_error_from_errno("malloc");
1278 tree = &(*trees)[*ntrees];
1280 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1281 tree->path = qid->data;
1284 tree->entries = entries;
1286 tree->nentries = nentries;
1288 got_object_qid_free(qid);
1290 } while (!STAILQ_EMPTY(&ids));
1292 if (*have_all_entries) {
1295 * We have managed to traverse all entries in the hierarchy.
1296 * Tell the main process what we have found.
1298 for (i = 0; i < *ntrees; i++) {
1299 tree = &(*trees)[i];
1300 err = got_privsep_send_enumerated_tree(totlen,
1301 ibuf, &tree->id, tree->path, tree->entries,
1309 free(tree->entries);
1310 tree->entries = NULL;
1312 *ntrees = 0; /* don't loop again below to free memory */
1314 err = send_tree_enumeration_done(ibuf);
1317 * We can only load fully packed tree hierarchies on
1318 * behalf of the main process, otherwise the main process
1319 * gets a wrong idea about which tree objects have
1320 * already been traversed.
1321 * Indicate a missing entry for the root of this tree.
1322 * The main process should continue by loading this
1323 * entire tree the slow way.
1325 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1326 tree_id, "/", NULL, -1);
1333 for (i = 0; i < *ntrees; i++) {
1334 tree = &(*trees)[i];
1339 free(tree->entries);
1340 tree->entries = NULL;
1344 got_object_qid_free(qid);
1345 got_object_id_queue_free(&ids);
1347 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1350 got_privsep_send_error(ibuf, err);
1356 static const struct got_error *
1357 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1358 struct got_pack *pack, struct got_packidx *packidx,
1359 struct got_object_cache *objcache)
1361 const struct got_error *err = NULL;
1362 struct got_object_id_queue commit_ids;
1363 const struct got_object_id_queue *parents = NULL;
1364 struct got_object_qid *qid = NULL;
1365 struct got_object *obj = NULL;
1366 struct got_commit_object *commit = NULL;
1367 struct got_object_id *tree_id = NULL;
1369 struct got_object_idset *idset;
1370 int i, idx, have_all_entries = 1;
1371 struct enumerated_tree *trees = NULL;
1372 size_t ntrees = 0, nalloc = 16;
1374 STAILQ_INIT(&commit_ids);
1376 trees = calloc(1, nalloc);
1378 return got_error_from_errno("calloc");
1380 idset = got_object_idset_alloc();
1381 if (idset == NULL) {
1382 err = got_error_from_errno("got_object_idset_alloc");
1386 err = recv_object_id_queue(&commit_ids, ibuf);
1390 err = recv_object_ids(idset, ibuf);
1394 while (!STAILQ_EMPTY(&commit_ids)) {
1395 if (sigint_received) {
1396 err = got_error(GOT_ERR_CANCELLED);
1400 qid = STAILQ_FIRST(&commit_ids);
1401 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1403 if (got_object_idset_contains(idset, &qid->id)) {
1404 got_object_qid_free(qid);
1409 idx = got_packidx_get_object_idx(packidx, &qid->id);
1413 err = open_object(&obj, pack, packidx, idx, &qid->id,
1417 if (obj->type == GOT_OBJ_TYPE_TAG) {
1418 struct got_tag_object *tag;
1421 err = got_packfile_extract_object_to_mem(&buf,
1426 err = got_object_parse_tag(&tag, buf, len);
1431 idx = got_packidx_get_object_idx(packidx, &tag->id);
1434 err = open_commit(&commit, pack, packidx, idx,
1435 &tag->id, objcache);
1436 got_object_tag_close(tag);
1440 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1441 err = open_commit(&commit, pack, packidx, idx,
1442 &qid->id, objcache);
1446 err = got_error(GOT_ERR_OBJ_TYPE);
1449 got_object_close(obj);
1452 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1453 got_object_commit_get_committer_time(commit));
1457 tree_id = got_object_commit_get_tree_id(commit);
1458 idx = got_packidx_get_object_idx(packidx, tree_id);
1460 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1461 tree_id, "/", NULL, -1);
1467 if (got_object_idset_contains(idset, tree_id)) {
1468 got_object_qid_free(qid);
1473 err = enumerate_tree(&have_all_entries, ibuf, &totlen, tree_id, "/",
1474 pack, packidx, objcache, idset, &trees, &nalloc, &ntrees);
1478 if (!have_all_entries)
1481 got_object_qid_free(qid);
1484 parents = got_object_commit_get_parent_ids(commit);
1486 struct got_object_qid *pid;
1487 STAILQ_FOREACH(pid, parents, entry) {
1488 if (got_object_idset_contains(idset, &pid->id))
1490 err = got_object_qid_alloc_partial(&qid);
1493 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1494 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1499 got_object_commit_close(commit);
1503 if (have_all_entries) {
1504 err = got_privsep_send_object_enumeration_done(ibuf);
1510 got_object_close(obj);
1512 got_object_commit_close(commit);
1513 got_object_qid_free(qid);
1514 got_object_id_queue_free(&commit_ids);
1516 got_object_idset_free(idset);
1517 for (i = 0; i < ntrees; i++) {
1518 struct enumerated_tree *tree = &trees[i];
1521 free(tree->entries);
1527 static const struct got_error *
1528 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1530 const struct got_error *err = NULL;
1532 struct got_imsg_pack ipack;
1534 struct got_pack *pack;
1538 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1542 pack = calloc(1, sizeof(*pack));
1544 err = got_error_from_errno("calloc");
1548 if (imsg.hdr.type != GOT_IMSG_PACK) {
1549 err = got_error(GOT_ERR_PRIVSEP_MSG);
1553 if (imsg.fd == -1) {
1554 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1558 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1559 if (datalen != sizeof(ipack)) {
1560 err = got_error(GOT_ERR_PRIVSEP_LEN);
1563 memcpy(&ipack, imsg.data, sizeof(ipack));
1565 pack->filesize = ipack.filesize;
1566 pack->fd = dup(imsg.fd);
1567 if (pack->fd == -1) {
1568 err = got_error_from_errno("dup");
1571 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1572 err = got_error_from_errno("lseek");
1575 pack->path_packfile = strdup(ipack.path_packfile);
1576 if (pack->path_packfile == NULL) {
1577 err = got_error_from_errno("strdup");
1581 err = got_delta_cache_alloc(&pack->delta_cache);
1585 #ifndef GOT_PACK_NO_MMAP
1586 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1588 if (pack->map == MAP_FAILED)
1589 pack->map = NULL; /* fall back to read(2) */
1603 main(int argc, char *argv[])
1605 const struct got_error *err = NULL;
1606 struct imsgbuf ibuf;
1608 struct got_packidx *packidx = NULL;
1609 struct got_pack *pack = NULL;
1610 struct got_object_cache objcache;
1611 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1613 //static int attached;
1614 //while (!attached) sleep(1);
1616 signal(SIGINT, catch_sigint);
1618 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1620 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1622 err = got_error_from_errno("got_object_cache_init");
1623 got_privsep_send_error(&ibuf, err);
1628 /* revoke access to most system calls */
1629 if (pledge("stdio recvfd", NULL) == -1) {
1630 err = got_error_from_errno("pledge");
1631 got_privsep_send_error(&ibuf, err);
1635 /* revoke fs access */
1636 if (landlock_no_fs() == -1) {
1637 err = got_error_from_errno("landlock_no_fs");
1638 got_privsep_send_error(&ibuf, err);
1643 err = receive_packidx(&packidx, &ibuf);
1645 got_privsep_send_error(&ibuf, err);
1649 err = receive_pack(&pack, &ibuf);
1651 got_privsep_send_error(&ibuf, err);
1658 if (sigint_received) {
1659 err = got_error(GOT_ERR_CANCELLED);
1663 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1665 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1670 if (imsg.hdr.type == GOT_IMSG_STOP)
1673 switch (imsg.hdr.type) {
1674 case GOT_IMSG_TMPFD:
1675 if (basefile == NULL) {
1676 err = receive_tempfile(&basefile, "w+",
1678 } else if (accumfile == NULL) {
1679 err = receive_tempfile(&accumfile, "w+",
1682 err = got_error(GOT_ERR_PRIVSEP_MSG);
1684 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1685 err = object_request(&imsg, &ibuf, pack, packidx,
1688 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1689 if (basefile == NULL || accumfile == NULL) {
1690 err = got_error(GOT_ERR_PRIVSEP_MSG);
1693 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1694 &objcache, basefile, accumfile);
1696 case GOT_IMSG_RAW_DELTA_OUTFD:
1697 if (delta_outfile != NULL) {
1698 err = got_error(GOT_ERR_PRIVSEP_MSG);
1701 err = receive_tempfile(&delta_outfile, "w",
1704 case GOT_IMSG_RAW_DELTA_REQUEST:
1705 if (delta_outfile == NULL) {
1706 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1709 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1712 case GOT_IMSG_DELTA_REUSE_REQUEST:
1713 if (delta_outfile == NULL) {
1714 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1717 err = delta_reuse_request(&imsg, &ibuf,
1718 delta_outfile, pack, packidx);
1720 case GOT_IMSG_COMMIT_REQUEST:
1721 err = commit_request(&imsg, &ibuf, pack, packidx,
1724 case GOT_IMSG_TREE_REQUEST:
1725 err = tree_request(&imsg, &ibuf, pack, packidx,
1728 case GOT_IMSG_BLOB_REQUEST:
1729 if (basefile == NULL || accumfile == NULL) {
1730 err = got_error(GOT_ERR_PRIVSEP_MSG);
1733 err = blob_request(&imsg, &ibuf, pack, packidx,
1734 &objcache, basefile, accumfile);
1736 case GOT_IMSG_TAG_REQUEST:
1737 err = tag_request(&imsg, &ibuf, pack, packidx,
1740 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1741 err = commit_traversal_request(&imsg, &ibuf, pack,
1742 packidx, &objcache);
1744 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
1745 err = enumeration_request(&imsg, &ibuf, pack,
1746 packidx, &objcache);
1749 err = got_error(GOT_ERR_PRIVSEP_MSG);
1753 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1754 err = got_error_from_errno("close");
1761 got_packidx_close(packidx);
1763 got_pack_close(pack);
1764 got_object_cache_close(&objcache);
1766 if (basefile && fclose(basefile) == EOF && err == NULL)
1767 err = got_error_from_errno("fclose");
1768 if (accumfile && fclose(accumfile) == EOF && err == NULL)
1769 err = got_error_from_errno("fclose");
1770 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1771 err = got_error_from_errno("fclose");
1773 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1774 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1775 got_privsep_send_error(&ibuf, err);
1778 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1779 err = got_error_from_errno("close");