2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 #include <sys/types.h>
19 #include <sys/queue.h>
35 #include "got_error.h"
36 #include "got_object.h"
39 #include "got_lib_delta.h"
40 #include "got_lib_delta_cache.h"
41 #include "got_lib_object.h"
42 #include "got_lib_object_cache.h"
43 #include "got_lib_object_parse.h"
44 #include "got_lib_object_idset.h"
45 #include "got_lib_privsep.h"
46 #include "got_lib_pack.h"
48 static volatile sig_atomic_t sigint_received;
51 catch_sigint(int signo)
56 static const struct got_error *
57 open_object(struct got_object **obj, struct got_pack *pack,
58 struct got_packidx *packidx, int idx, struct got_object_id *id,
59 struct got_object_cache *objcache)
61 const struct got_error *err;
63 err = got_packfile_open_object(obj, pack, packidx, idx, id);
68 err = got_object_cache_add(objcache, id, *obj);
70 if (err->code == GOT_ERR_OBJ_EXISTS ||
71 err->code == GOT_ERR_OBJ_TOO_LARGE)
79 static const struct got_error *
80 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
81 struct got_packidx *packidx, struct got_object_cache *objcache)
83 const struct got_error *err = NULL;
84 struct got_imsg_packed_object iobj;
85 struct got_object *obj;
86 struct got_object_id id;
89 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
90 if (datalen != sizeof(iobj))
91 return got_error(GOT_ERR_PRIVSEP_LEN);
92 memcpy(&iobj, imsg->data, sizeof(iobj));
93 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
95 obj = got_object_cache_get(objcache, &id);
99 err = open_object(&obj, pack, packidx, iobj.idx, &id,
105 err = got_privsep_send_obj(ibuf, obj);
107 got_object_close(obj);
111 static const struct got_error *
112 open_commit(struct got_commit_object **commit, struct got_pack *pack,
113 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
114 struct got_object_cache *objcache)
116 const struct got_error *err = NULL;
117 struct got_object *obj = NULL;
123 obj = got_object_cache_get(objcache, id);
127 err = open_object(&obj, pack, packidx, obj_idx, id,
133 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
139 err = got_object_parse_commit(commit, buf, len);
141 got_object_close(obj);
146 static const struct got_error *
147 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
148 struct got_packidx *packidx, struct got_object_cache *objcache)
150 const struct got_error *err = NULL;
151 struct got_imsg_packed_object iobj;
152 struct got_commit_object *commit = NULL;
153 struct got_object_id id;
156 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
157 if (datalen != sizeof(iobj))
158 return got_error(GOT_ERR_PRIVSEP_LEN);
159 memcpy(&iobj, imsg->data, sizeof(iobj));
160 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
162 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
166 err = got_privsep_send_commit(ibuf, commit);
169 got_object_commit_close(commit);
171 if (err->code == GOT_ERR_PRIVSEP_PIPE)
174 got_privsep_send_error(ibuf, err);
180 static const struct got_error *
181 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, int *nentries,
182 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
183 struct got_object_id *id, struct got_object_cache *objcache)
185 const struct got_error *err = NULL;
186 struct got_object *obj = NULL;
192 obj = got_object_cache_get(objcache, id);
196 err = open_object(&obj, pack, packidx, obj_idx, id,
202 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
208 err = got_object_parse_tree(entries, nentries, *buf, len);
210 got_object_close(obj);
218 static const struct got_error *
219 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
220 struct got_packidx *packidx, struct got_object_cache *objcache)
222 const struct got_error *err = NULL;
223 struct got_imsg_packed_object iobj;
224 struct got_parsed_tree_entry *entries = NULL;
227 struct got_object_id id;
230 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
231 if (datalen != sizeof(iobj))
232 return got_error(GOT_ERR_PRIVSEP_LEN);
233 memcpy(&iobj, imsg->data, sizeof(iobj));
234 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
236 err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
241 err = got_privsep_send_tree(ibuf, entries, nentries);
245 if (err->code == GOT_ERR_PRIVSEP_PIPE)
248 got_privsep_send_error(ibuf, err);
254 static const struct got_error *
255 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
257 const struct got_error *err;
261 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
265 if (imsg.hdr.type != imsg_code) {
266 err = got_error(GOT_ERR_PRIVSEP_MSG);
270 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
272 err = got_error(GOT_ERR_PRIVSEP_LEN);
276 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
280 *f = fdopen(imsg.fd, "w+");
282 err = got_error_from_errno("fdopen");
291 static const struct got_error *
292 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
293 struct imsgbuf *ibuf)
297 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
299 return got_error(GOT_ERR_PRIVSEP_LEN);
302 return got_error(GOT_ERR_PRIVSEP_NO_FD);
304 *f = fdopen(imsg->fd, mode);
306 return got_error_from_errno("fdopen");
312 static const struct got_error *
313 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
314 struct got_packidx *packidx, struct got_object_cache *objcache,
315 FILE *basefile, FILE *accumfile)
317 const struct got_error *err = NULL;
318 struct got_imsg_packed_object iobj;
319 struct got_object *obj = NULL;
320 FILE *outfile = NULL;
321 struct got_object_id id;
326 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
327 if (datalen != sizeof(iobj))
328 return got_error(GOT_ERR_PRIVSEP_LEN);
329 memcpy(&iobj, imsg->data, sizeof(iobj));
330 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
332 obj = got_object_cache_get(objcache, &id);
336 err = open_object(&obj, pack, packidx, iobj.idx, &id,
342 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
346 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
347 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
351 blob_size = obj->size;
353 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
354 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
357 err = got_packfile_extract_object(pack, obj, outfile, basefile,
362 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
365 if (outfile && fclose(outfile) == EOF && err == NULL)
366 err = got_error_from_errno("fclose");
367 got_object_close(obj);
368 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
369 got_privsep_send_error(ibuf, err);
374 static const struct got_error *
375 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
376 struct got_packidx *packidx, struct got_object_cache *objcache)
378 const struct got_error *err = NULL;
379 struct got_imsg_packed_object iobj;
380 struct got_object *obj = NULL;
381 struct got_tag_object *tag = NULL;
384 struct got_object_id id;
387 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
388 if (datalen != sizeof(iobj))
389 return got_error(GOT_ERR_PRIVSEP_LEN);
390 memcpy(&iobj, imsg->data, sizeof(iobj));
391 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
393 obj = got_object_cache_get(objcache, &id);
397 err = open_object(&obj, pack, packidx, iobj.idx, &id,
403 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
408 err = got_object_parse_tag(&tag, buf, len);
412 err = got_privsep_send_tag(ibuf, tag);
415 got_object_close(obj);
417 got_object_tag_close(tag);
419 if (err->code == GOT_ERR_PRIVSEP_PIPE)
422 got_privsep_send_error(ibuf, err);
428 static struct got_parsed_tree_entry *
429 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
430 const char *name, size_t len)
432 struct got_parsed_tree_entry *pte;
435 /* Note that tree entries are sorted in strncmp() order. */
436 for (i = 0; i < nentries; i++) {
438 cmp = strncmp(pte->name, name, len);
443 if (pte->name[len] == '\0')
449 static const struct got_error *
450 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
451 struct got_parsed_tree_entry **entries1, int *nentries1,
452 struct got_parsed_tree_entry **entries2, int *nentries2,
453 const char *path, struct got_pack *pack, struct got_packidx *packidx,
454 struct imsgbuf *ibuf, struct got_object_cache *objcache)
456 const struct got_error *err = NULL;
457 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
463 /* We not do support comparing the root path. */
464 if (got_path_is_root_dir(path))
465 return got_error_path(path, GOT_ERR_BAD_PATH);
480 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
482 err = got_error(GOT_ERR_NO_OBJ);
486 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
492 if (pte1->mode != pte2->mode) {
497 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
502 if (*s == '\0') { /* final path element */
511 struct got_object_id id1, id2;
514 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
515 idx = got_packidx_get_object_idx(packidx, &id1);
517 err = got_error_no_obj(&id1);
524 err = open_tree(buf1, entries1, nentries1, pack,
525 packidx, idx, &id1, objcache);
530 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
531 idx = got_packidx_get_object_idx(packidx, &id2);
533 err = got_error_no_obj(&id2);
540 err = open_tree(buf2, entries2, nentries2, pack,
541 packidx, idx, &id2, objcache);
551 static const struct got_error *
552 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
553 struct imsgbuf *ibuf)
558 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
559 sizeof(struct got_imsg_traversed_commits) +
560 ncommits * SHA1_DIGEST_LENGTH);
562 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
564 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
565 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
567 for (i = 0; i < ncommits; i++) {
568 struct got_object_id *id = &commit_ids[i];
569 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
570 return got_error_from_errno(
571 "imsg_add TRAVERSED_COMMITS");
576 imsg_close(ibuf, wbuf);
578 return got_privsep_flush_imsg(ibuf);
581 static const struct got_error *
582 send_commit_traversal_done(struct imsgbuf *ibuf)
584 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
586 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
588 return got_privsep_flush_imsg(ibuf);
591 static const struct got_error *
592 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
593 struct got_pack *pack, struct got_packidx *packidx,
594 struct got_object_cache *objcache)
596 const struct got_error *err = NULL;
597 struct got_imsg_packed_object iobj;
598 struct got_object_qid *pid;
599 struct got_commit_object *commit = NULL, *pcommit = NULL;
600 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
601 int nentries = 0, pnentries = 0;
602 struct got_object_id id;
603 size_t datalen, path_len;
605 const int min_alloc = 64;
606 int changed = 0, ncommits = 0, nallocated = 0;
607 struct got_object_id *commit_ids = NULL;
609 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
610 if (datalen < sizeof(iobj))
611 return got_error(GOT_ERR_PRIVSEP_LEN);
612 memcpy(&iobj, imsg->data, sizeof(iobj));
613 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
615 path_len = datalen - sizeof(iobj) - 1;
617 return got_error(GOT_ERR_PRIVSEP_LEN);
619 path = imsg->data + sizeof(iobj);
620 if (path[path_len] != '\0')
621 return got_error(GOT_ERR_PRIVSEP_LEN);
624 nallocated = min_alloc;
625 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
626 if (commit_ids == NULL)
627 return got_error_from_errno("reallocarray");
630 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
633 if (sigint_received) {
634 err = got_error(GOT_ERR_CANCELLED);
638 if (commit == NULL) {
639 idx = got_packidx_get_object_idx(packidx, &id);
642 err = open_commit(&commit, pack, packidx,
645 if (err->code != GOT_ERR_NO_OBJ)
652 if (sizeof(struct got_imsg_traversed_commits) +
653 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
654 err = send_traversed_commits(commit_ids, ncommits,
661 if (ncommits > nallocated) {
662 struct got_object_id *new;
663 nallocated += min_alloc;
664 new = reallocarray(commit_ids, nallocated,
665 sizeof(*commit_ids));
667 err = got_error_from_errno("reallocarray");
672 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
675 pid = STAILQ_FIRST(&commit->parent_ids);
679 idx = got_packidx_get_object_idx(packidx, &pid->id);
683 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
686 if (err->code != GOT_ERR_NO_OBJ)
692 if (path[0] == '/' && path[1] == '\0') {
693 if (got_object_id_cmp(pcommit->tree_id,
694 commit->tree_id) != 0) {
700 uint8_t *buf = NULL, *pbuf = NULL;
702 idx = got_packidx_get_object_idx(packidx,
706 pidx = got_packidx_get_object_idx(packidx,
711 err = open_tree(&buf, &entries, &nentries, pack,
712 packidx, idx, commit->tree_id, objcache);
715 err = open_tree(&pbuf, &pentries, &pnentries, pack,
716 packidx, pidx, pcommit->tree_id, objcache);
722 err = tree_path_changed(&changed, &buf, &pbuf,
723 &entries, &nentries, &pentries, &pnentries, path,
724 pack, packidx, ibuf, objcache);
735 if (err->code != GOT_ERR_NO_OBJ)
743 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
744 got_object_commit_close(commit);
751 err = send_traversed_commits(commit_ids, ncommits, ibuf);
756 err = got_privsep_send_commit(ibuf, commit);
761 err = send_commit_traversal_done(ibuf);
765 got_object_commit_close(commit);
767 got_object_commit_close(pcommit);
771 if (err->code == GOT_ERR_PRIVSEP_PIPE)
774 got_privsep_send_error(ibuf, err);
780 static const struct got_error *
781 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
782 struct got_pack *pack, struct got_packidx *packidx,
783 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
785 const struct got_error *err = NULL;
788 FILE *outfile = NULL;
789 struct got_imsg_packed_object iobj;
790 struct got_object *obj;
791 struct got_object_id id;
794 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
795 if (datalen != sizeof(iobj))
796 return got_error(GOT_ERR_PRIVSEP_LEN);
797 memcpy(&iobj, imsg->data, sizeof(iobj));
798 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
800 obj = got_object_cache_get(objcache, &id);
804 err = open_object(&obj, pack, packidx, iobj.idx, &id,
810 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
814 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
815 err = got_pack_get_max_delta_object_size(&size, obj, pack);
821 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
822 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
825 err = got_packfile_extract_object(pack, obj, outfile, basefile,
830 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
833 if (outfile && fclose(outfile) == EOF && err == NULL)
834 err = got_error_from_errno("fclose");
835 got_object_close(obj);
836 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
837 got_privsep_send_error(ibuf, err);
842 static const struct got_error *
843 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
846 const struct got_error *err;
849 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
853 return got_error(GOT_ERR_BAD_PACKIDX);
855 return got_packidx_get_object_id(base_id, packidx, idx);
858 static const struct got_error *
859 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
860 FILE *delta_outfile, struct got_pack *pack,
861 struct got_packidx *packidx)
863 const struct got_error *err = NULL;
864 struct got_imsg_raw_delta_request req;
865 size_t datalen, delta_size, delta_compressed_size;
867 uint8_t *delta_buf = NULL;
868 struct got_object_id id, base_id;
869 off_t base_offset, delta_out_offset = 0;
870 uint64_t base_size = 0, result_size = 0;
873 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
874 if (datalen != sizeof(req))
875 return got_error(GOT_ERR_PRIVSEP_LEN);
876 memcpy(&req, imsg->data, sizeof(req));
877 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
881 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
882 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
883 &base_size, &result_size, pack, packidx, req.idx);
888 * If this is an offset delta we must determine the base
889 * object ID ourselves.
891 if (base_offset != 0) {
892 err = get_base_object_id(&base_id, packidx, base_offset);
897 delta_out_offset = ftello(delta_outfile);
898 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
899 if (w != delta_compressed_size) {
900 err = got_ferror(delta_outfile, GOT_ERR_IO);
903 if (fflush(delta_outfile) == -1) {
904 err = got_error_from_errno("fflush");
908 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
909 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
916 struct search_deltas_arg {
917 struct imsgbuf *ibuf;
918 struct got_packidx *packidx;
919 struct got_pack *pack;
920 struct got_object_idset *idset;
922 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
926 static const struct got_error *
927 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
929 const struct got_error *err;
930 struct search_deltas_arg *a = arg;
932 uint8_t *delta_buf = NULL;
933 uint64_t base_size, result_size;
934 size_t delta_size, delta_compressed_size;
935 off_t delta_offset, base_offset;
936 struct got_object_id base_id;
939 return got_error(GOT_ERR_CANCELLED);
941 obj_idx = got_packidx_get_object_idx(a->packidx, id);
943 return NULL; /* object not present in our pack file */
945 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
946 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
947 &base_size, &result_size, a->pack, a->packidx, obj_idx);
949 if (err->code == GOT_ERR_OBJ_TYPE)
950 return NULL; /* object not stored as a delta */
955 * If this is an offset delta we must determine the base
956 * object ID ourselves.
958 if (base_offset != 0) {
959 err = get_base_object_id(&base_id, a->packidx, base_offset);
964 if (got_object_idset_contains(a->idset, &base_id)) {
965 struct got_imsg_reused_delta *delta;
966 off_t delta_out_offset = ftello(a->delta_outfile);
969 w = fwrite(delta_buf, 1, delta_compressed_size,
971 if (w != delta_compressed_size) {
972 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
976 delta = &a->deltas[a->ndeltas++];
977 memcpy(&delta->id, id, sizeof(delta->id));
978 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
979 delta->base_size = base_size;
980 delta->result_size = result_size;
981 delta->delta_size = delta_size;
982 delta->delta_compressed_size = delta_compressed_size;
983 delta->delta_offset = delta_offset;
984 delta->delta_out_offset = delta_out_offset;
986 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
987 err = got_privsep_send_reused_deltas(a->ibuf,
988 a->deltas, a->ndeltas);
999 static const struct got_error *
1000 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1002 const struct got_error *err = NULL;
1004 struct got_object_id *ids;
1008 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1011 for (i = 0; i < nids; i++) {
1012 err = got_object_idset_add(idset, &ids[i], NULL);
1024 static const struct got_error *
1025 recv_object_id_queue(struct got_object_id_queue *queue, struct imsgbuf *ibuf)
1027 const struct got_error *err = NULL;
1029 struct got_object_qid *qid;
1030 struct got_object_id *ids;
1034 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1037 for (i = 0; i < nids; i++) {
1038 err = got_object_qid_alloc_partial(&qid);
1041 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1042 STAILQ_INSERT_TAIL(queue, qid, entry);
1049 static const struct got_error *
1050 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1051 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1053 const struct got_error *err = NULL;
1054 struct got_object_idset *idset;
1055 struct search_deltas_arg sda;
1057 idset = got_object_idset_alloc();
1059 return got_error_from_errno("got_object_idset_alloc");
1061 err = recv_object_ids(idset, ibuf);
1065 memset(&sda, 0, sizeof(sda));
1069 sda.packidx = packidx;
1070 sda.delta_outfile = delta_outfile;
1071 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1075 if (sda.ndeltas > 0) {
1076 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1082 if (fflush(delta_outfile) == -1) {
1083 err = got_error_from_errno("fflush");
1087 err = got_privsep_send_reused_deltas_done(ibuf);
1089 got_object_idset_free(idset);
1093 static const struct got_error *
1094 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1096 const struct got_error *err = NULL;
1098 struct got_imsg_packidx ipackidx;
1100 struct got_packidx *p;
1104 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1108 p = calloc(1, sizeof(*p));
1110 err = got_error_from_errno("calloc");
1114 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1115 err = got_error(GOT_ERR_PRIVSEP_MSG);
1119 if (imsg.fd == -1) {
1120 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1124 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1125 if (datalen != sizeof(ipackidx)) {
1126 err = got_error(GOT_ERR_PRIVSEP_LEN);
1129 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1131 p->len = ipackidx.len;
1132 p->fd = dup(imsg.fd);
1134 err = got_error_from_errno("dup");
1137 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1138 err = got_error_from_errno("lseek");
1142 #ifndef GOT_PACK_NO_MMAP
1143 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1144 if (p->map == MAP_FAILED)
1145 p->map = NULL; /* fall back to read(2) */
1147 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1152 got_packidx_close(p);
1159 static const struct got_error *
1160 send_tree_enumeration_done(struct imsgbuf *ibuf)
1162 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1164 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1166 return got_privsep_flush_imsg(ibuf);
1169 struct enumerated_tree {
1170 struct got_object_id id;
1173 struct got_parsed_tree_entry *entries;
1177 static const struct got_error *
1178 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1179 struct got_object_id *tree_id,
1180 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1181 struct got_object_cache *objcache, struct got_object_idset *idset,
1182 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1184 const struct got_error *err = NULL;
1185 struct got_object_id_queue ids;
1186 struct got_object_qid *qid;
1187 uint8_t *buf = NULL;
1188 struct got_parsed_tree_entry *entries = NULL;
1189 int nentries = 0, i;
1190 struct enumerated_tree *tree;
1193 *have_all_entries = 1;
1196 err = got_object_qid_alloc_partial(&qid);
1199 memcpy(&qid->id.sha1, tree_id, SHA1_DIGEST_LENGTH);
1200 qid->data = strdup(path);
1201 if (qid->data == NULL) {
1202 err = got_error_from_errno("strdup");
1205 STAILQ_INSERT_TAIL(&ids, qid, entry);
1208 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1213 if (sigint_received) {
1214 err = got_error(GOT_ERR_CANCELLED);
1218 qid = STAILQ_FIRST(&ids);
1219 STAILQ_REMOVE_HEAD(&ids, entry);
1222 idx = got_packidx_get_object_idx(packidx, &qid->id);
1224 *have_all_entries = 0;
1228 err = open_tree(&buf, &entries, &nentries,
1229 pack, packidx, idx, &qid->id, objcache);
1231 if (err->code != GOT_ERR_NO_OBJ)
1235 err = got_object_idset_add(idset, &qid->id, NULL);
1239 for (i = 0; i < nentries; i++) {
1240 struct got_object_qid *eqid = NULL;
1241 struct got_parsed_tree_entry *pte = &entries[i];
1244 if (!S_ISDIR(pte->mode))
1247 err = got_object_qid_alloc_partial(&eqid);
1250 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1252 if (got_object_idset_contains(idset, &eqid->id)) {
1253 got_object_qid_free(eqid);
1257 if (asprintf(&p, "%s%s%s", path,
1258 got_path_is_root_dir(path) ? "" : "/",
1260 err = got_error_from_errno("asprintf");
1261 got_object_qid_free(eqid);
1265 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1268 if (*ntrees >= *nalloc) {
1269 struct enumerated_tree *new;
1270 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1273 err = got_error_from_errno("malloc");
1279 tree = &(*trees)[*ntrees];
1281 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1282 tree->path = qid->data;
1285 tree->entries = entries;
1287 tree->nentries = nentries;
1289 got_object_qid_free(qid);
1291 } while (!STAILQ_EMPTY(&ids));
1293 if (*have_all_entries) {
1296 * We have managed to traverse all entries in the hierarchy.
1297 * Tell the main process what we have found.
1299 for (i = 0; i < *ntrees; i++) {
1300 tree = &(*trees)[i];
1301 err = got_privsep_send_enumerated_tree(totlen,
1302 ibuf, &tree->id, tree->path, tree->entries,
1310 free(tree->entries);
1311 tree->entries = NULL;
1313 *ntrees = 0; /* don't loop again below to free memory */
1315 err = send_tree_enumeration_done(ibuf);
1318 * We can only load fully packed tree hierarchies on
1319 * behalf of the main process, otherwise the main process
1320 * gets a wrong idea about which tree objects have
1321 * already been traversed.
1322 * Indicate a missing entry for the root of this tree.
1323 * The main process should continue by loading this
1324 * entire tree the slow way.
1326 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1327 tree_id, "/", NULL, -1);
1334 for (i = 0; i < *ntrees; i++) {
1335 tree = &(*trees)[i];
1340 free(tree->entries);
1341 tree->entries = NULL;
1345 got_object_qid_free(qid);
1346 got_object_id_queue_free(&ids);
1348 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1351 got_privsep_send_error(ibuf, err);
1357 static const struct got_error *
1358 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1359 struct got_pack *pack, struct got_packidx *packidx,
1360 struct got_object_cache *objcache)
1362 const struct got_error *err = NULL;
1363 struct got_object_id_queue commit_ids;
1364 const struct got_object_id_queue *parents = NULL;
1365 struct got_object_qid *qid = NULL;
1366 struct got_object *obj = NULL;
1367 struct got_commit_object *commit = NULL;
1368 struct got_object_id *tree_id = NULL;
1370 struct got_object_idset *idset;
1371 int i, idx, have_all_entries = 1;
1372 struct enumerated_tree *trees = NULL;
1373 size_t ntrees = 0, nalloc = 16;
1375 STAILQ_INIT(&commit_ids);
1377 trees = calloc(nalloc, sizeof(*trees));
1379 return got_error_from_errno("calloc");
1381 idset = got_object_idset_alloc();
1382 if (idset == NULL) {
1383 err = got_error_from_errno("got_object_idset_alloc");
1387 err = recv_object_id_queue(&commit_ids, ibuf);
1391 err = recv_object_ids(idset, ibuf);
1395 while (!STAILQ_EMPTY(&commit_ids)) {
1396 if (sigint_received) {
1397 err = got_error(GOT_ERR_CANCELLED);
1401 qid = STAILQ_FIRST(&commit_ids);
1402 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1404 if (got_object_idset_contains(idset, &qid->id)) {
1405 got_object_qid_free(qid);
1410 idx = got_packidx_get_object_idx(packidx, &qid->id);
1414 err = open_object(&obj, pack, packidx, idx, &qid->id,
1418 if (obj->type == GOT_OBJ_TYPE_TAG) {
1419 struct got_tag_object *tag;
1422 err = got_packfile_extract_object_to_mem(&buf,
1427 err = got_object_parse_tag(&tag, buf, len);
1432 idx = got_packidx_get_object_idx(packidx, &tag->id);
1435 err = open_commit(&commit, pack, packidx, idx,
1436 &tag->id, objcache);
1437 got_object_tag_close(tag);
1441 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1442 err = open_commit(&commit, pack, packidx, idx,
1443 &qid->id, objcache);
1447 err = got_error(GOT_ERR_OBJ_TYPE);
1450 got_object_close(obj);
1453 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1454 got_object_commit_get_committer_time(commit));
1458 tree_id = got_object_commit_get_tree_id(commit);
1459 idx = got_packidx_get_object_idx(packidx, tree_id);
1461 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1462 tree_id, "/", NULL, -1);
1468 if (got_object_idset_contains(idset, tree_id)) {
1469 got_object_qid_free(qid);
1474 err = enumerate_tree(&have_all_entries, ibuf, &totlen, tree_id, "/",
1475 pack, packidx, objcache, idset, &trees, &nalloc, &ntrees);
1479 if (!have_all_entries)
1482 got_object_qid_free(qid);
1485 parents = got_object_commit_get_parent_ids(commit);
1487 struct got_object_qid *pid;
1488 STAILQ_FOREACH(pid, parents, entry) {
1489 if (got_object_idset_contains(idset, &pid->id))
1491 err = got_object_qid_alloc_partial(&qid);
1494 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1495 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1500 got_object_commit_close(commit);
1504 if (have_all_entries) {
1505 err = got_privsep_send_object_enumeration_done(ibuf);
1511 got_object_close(obj);
1513 got_object_commit_close(commit);
1514 got_object_qid_free(qid);
1515 got_object_id_queue_free(&commit_ids);
1517 got_object_idset_free(idset);
1518 for (i = 0; i < ntrees; i++) {
1519 struct enumerated_tree *tree = &trees[i];
1522 free(tree->entries);
1528 static const struct got_error *
1529 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1531 const struct got_error *err = NULL;
1533 struct got_imsg_pack ipack;
1535 struct got_pack *pack;
1539 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1543 pack = calloc(1, sizeof(*pack));
1545 err = got_error_from_errno("calloc");
1549 if (imsg.hdr.type != GOT_IMSG_PACK) {
1550 err = got_error(GOT_ERR_PRIVSEP_MSG);
1554 if (imsg.fd == -1) {
1555 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1559 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1560 if (datalen != sizeof(ipack)) {
1561 err = got_error(GOT_ERR_PRIVSEP_LEN);
1564 memcpy(&ipack, imsg.data, sizeof(ipack));
1566 pack->filesize = ipack.filesize;
1567 pack->fd = dup(imsg.fd);
1568 if (pack->fd == -1) {
1569 err = got_error_from_errno("dup");
1572 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1573 err = got_error_from_errno("lseek");
1576 pack->path_packfile = strdup(ipack.path_packfile);
1577 if (pack->path_packfile == NULL) {
1578 err = got_error_from_errno("strdup");
1582 err = got_delta_cache_alloc(&pack->delta_cache);
1586 #ifndef GOT_PACK_NO_MMAP
1587 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1589 if (pack->map == MAP_FAILED)
1590 pack->map = NULL; /* fall back to read(2) */
1604 main(int argc, char *argv[])
1606 const struct got_error *err = NULL;
1607 struct imsgbuf ibuf;
1609 struct got_packidx *packidx = NULL;
1610 struct got_pack *pack = NULL;
1611 struct got_object_cache objcache;
1612 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1614 //static int attached;
1615 //while (!attached) sleep(1);
1617 signal(SIGINT, catch_sigint);
1619 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1621 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1623 err = got_error_from_errno("got_object_cache_init");
1624 got_privsep_send_error(&ibuf, err);
1629 /* revoke access to most system calls */
1630 if (pledge("stdio recvfd", NULL) == -1) {
1631 err = got_error_from_errno("pledge");
1632 got_privsep_send_error(&ibuf, err);
1637 err = receive_packidx(&packidx, &ibuf);
1639 got_privsep_send_error(&ibuf, err);
1643 err = receive_pack(&pack, &ibuf);
1645 got_privsep_send_error(&ibuf, err);
1652 if (sigint_received) {
1653 err = got_error(GOT_ERR_CANCELLED);
1657 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1659 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1664 if (imsg.hdr.type == GOT_IMSG_STOP)
1667 switch (imsg.hdr.type) {
1668 case GOT_IMSG_TMPFD:
1669 if (basefile == NULL) {
1670 err = receive_tempfile(&basefile, "w+",
1672 } else if (accumfile == NULL) {
1673 err = receive_tempfile(&accumfile, "w+",
1676 err = got_error(GOT_ERR_PRIVSEP_MSG);
1678 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1679 err = object_request(&imsg, &ibuf, pack, packidx,
1682 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1683 if (basefile == NULL || accumfile == NULL) {
1684 err = got_error(GOT_ERR_PRIVSEP_MSG);
1687 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1688 &objcache, basefile, accumfile);
1690 case GOT_IMSG_RAW_DELTA_OUTFD:
1691 if (delta_outfile != NULL) {
1692 err = got_error(GOT_ERR_PRIVSEP_MSG);
1695 err = receive_tempfile(&delta_outfile, "w",
1698 case GOT_IMSG_RAW_DELTA_REQUEST:
1699 if (delta_outfile == NULL) {
1700 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1703 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1706 case GOT_IMSG_DELTA_REUSE_REQUEST:
1707 if (delta_outfile == NULL) {
1708 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1711 err = delta_reuse_request(&imsg, &ibuf,
1712 delta_outfile, pack, packidx);
1714 case GOT_IMSG_COMMIT_REQUEST:
1715 err = commit_request(&imsg, &ibuf, pack, packidx,
1718 case GOT_IMSG_TREE_REQUEST:
1719 err = tree_request(&imsg, &ibuf, pack, packidx,
1722 case GOT_IMSG_BLOB_REQUEST:
1723 if (basefile == NULL || accumfile == NULL) {
1724 err = got_error(GOT_ERR_PRIVSEP_MSG);
1727 err = blob_request(&imsg, &ibuf, pack, packidx,
1728 &objcache, basefile, accumfile);
1730 case GOT_IMSG_TAG_REQUEST:
1731 err = tag_request(&imsg, &ibuf, pack, packidx,
1734 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1735 err = commit_traversal_request(&imsg, &ibuf, pack,
1736 packidx, &objcache);
1738 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
1739 err = enumeration_request(&imsg, &ibuf, pack,
1740 packidx, &objcache);
1743 err = got_error(GOT_ERR_PRIVSEP_MSG);
1747 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1748 err = got_error_from_errno("close");
1755 got_packidx_close(packidx);
1757 got_pack_close(pack);
1758 got_object_cache_close(&objcache);
1760 if (basefile && fclose(basefile) == EOF && err == NULL)
1761 err = got_error_from_errno("fclose");
1762 if (accumfile && fclose(accumfile) == EOF && err == NULL)
1763 err = got_error_from_errno("fclose");
1764 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1765 err = got_error_from_errno("fclose");
1767 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1768 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1769 got_privsep_send_error(&ibuf, err);
1772 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1773 err = got_error_from_errno("close");