Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/mman.h>
23 #include <limits.h>
24 #include <signal.h>
25 #include <stdint.h>
26 #include <imsg.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <sha1.h>
31 #include <unistd.h>
32 #include <zlib.h>
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_object_idset.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_pack.h"
47 static volatile sig_atomic_t sigint_received;
49 static void
50 catch_sigint(int signo)
51 {
52 sigint_received = 1;
53 }
55 static const struct got_error *
56 open_object(struct got_object **obj, struct got_pack *pack,
57 struct got_packidx *packidx, int idx, struct got_object_id *id,
58 struct got_object_cache *objcache)
59 {
60 const struct got_error *err;
62 err = got_packfile_open_object(obj, pack, packidx, idx, id);
63 if (err)
64 return err;
65 (*obj)->refcnt++;
67 err = got_object_cache_add(objcache, id, *obj);
68 if (err) {
69 if (err->code == GOT_ERR_OBJ_EXISTS ||
70 err->code == GOT_ERR_OBJ_TOO_LARGE)
71 err = NULL;
72 return err;
73 }
74 (*obj)->refcnt++;
75 return NULL;
76 }
78 static const struct got_error *
79 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
80 struct got_packidx *packidx, struct got_object_cache *objcache)
81 {
82 const struct got_error *err = NULL;
83 struct got_imsg_packed_object iobj;
84 struct got_object *obj;
85 struct got_object_id id;
86 size_t datalen;
88 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
89 if (datalen != sizeof(iobj))
90 return got_error(GOT_ERR_PRIVSEP_LEN);
91 memcpy(&iobj, imsg->data, sizeof(iobj));
92 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
94 obj = got_object_cache_get(objcache, &id);
95 if (obj) {
96 obj->refcnt++;
97 } else {
98 err = open_object(&obj, pack, packidx, iobj.idx, &id,
99 objcache);
100 if (err)
101 goto done;
104 err = got_privsep_send_obj(ibuf, obj);
105 done:
106 got_object_close(obj);
107 return err;
110 static const struct got_error *
111 open_commit(struct got_commit_object **commit, struct got_pack *pack,
112 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
113 struct got_object_cache *objcache)
115 const struct got_error *err = NULL;
116 struct got_object *obj = NULL;
117 uint8_t *buf = NULL;
118 size_t len;
120 *commit = NULL;
122 obj = got_object_cache_get(objcache, id);
123 if (obj) {
124 obj->refcnt++;
125 } else {
126 err = open_object(&obj, pack, packidx, obj_idx, id,
127 objcache);
128 if (err)
129 return err;
132 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
133 if (err)
134 goto done;
136 obj->size = len;
138 err = got_object_parse_commit(commit, buf, len);
139 done:
140 got_object_close(obj);
141 free(buf);
142 return err;
145 static const struct got_error *
146 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
147 struct got_packidx *packidx, struct got_object_cache *objcache)
149 const struct got_error *err = NULL;
150 struct got_imsg_packed_object iobj;
151 struct got_commit_object *commit = NULL;
152 struct got_object_id id;
153 size_t datalen;
155 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
156 if (datalen != sizeof(iobj))
157 return got_error(GOT_ERR_PRIVSEP_LEN);
158 memcpy(&iobj, imsg->data, sizeof(iobj));
159 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
161 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
162 if (err)
163 goto done;
165 err = got_privsep_send_commit(ibuf, commit);
166 done:
167 if (commit)
168 got_object_commit_close(commit);
169 if (err) {
170 if (err->code == GOT_ERR_PRIVSEP_PIPE)
171 err = NULL;
172 else
173 got_privsep_send_error(ibuf, err);
176 return err;
179 static const struct got_error *
180 open_tree(uint8_t **buf, struct got_pathlist_head *entries, int *nentries,
181 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
182 struct got_object_id *id, struct got_object_cache *objcache)
184 const struct got_error *err = NULL;
185 struct got_object *obj = NULL;
186 size_t len;
188 *buf = NULL;
189 *nentries = 0;
191 obj = got_object_cache_get(objcache, id);
192 if (obj) {
193 obj->refcnt++;
194 } else {
195 err = open_object(&obj, pack, packidx, obj_idx, id,
196 objcache);
197 if (err)
198 return err;
201 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
202 if (err)
203 goto done;
205 obj->size = len;
207 err = got_object_parse_tree(entries, nentries, *buf, len);
208 done:
209 got_object_close(obj);
210 if (err) {
211 free(*buf);
212 *buf = NULL;
214 return err;
217 static const struct got_error *
218 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
219 struct got_packidx *packidx, struct got_object_cache *objcache)
221 const struct got_error *err = NULL;
222 struct got_imsg_packed_object iobj;
223 struct got_pathlist_head entries;
224 int nentries = 0;
225 uint8_t *buf = NULL;
226 struct got_object_id id;
227 size_t datalen;
229 TAILQ_INIT(&entries);
231 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
232 if (datalen != sizeof(iobj))
233 return got_error(GOT_ERR_PRIVSEP_LEN);
234 memcpy(&iobj, imsg->data, sizeof(iobj));
235 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
237 err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
238 &id, objcache);
239 if (err)
240 return err;
242 err = got_privsep_send_tree(ibuf, &entries, nentries);
243 got_object_parsed_tree_entries_free(&entries);
244 free(buf);
245 if (err) {
246 if (err->code == GOT_ERR_PRIVSEP_PIPE)
247 err = NULL;
248 else
249 got_privsep_send_error(ibuf, err);
252 return err;
255 static const struct got_error *
256 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
258 const struct got_error *err;
259 struct imsg imsg;
260 size_t datalen;
262 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
263 if (err)
264 return err;
266 if (imsg.hdr.type != imsg_code) {
267 err = got_error(GOT_ERR_PRIVSEP_MSG);
268 goto done;
271 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
272 if (datalen != 0) {
273 err = got_error(GOT_ERR_PRIVSEP_LEN);
274 goto done;
276 if (imsg.fd == -1) {
277 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
278 goto done;
281 *f = fdopen(imsg.fd, "w+");
282 if (*f == NULL) {
283 err = got_error_from_errno("fdopen");
284 close(imsg.fd);
285 goto done;
287 done:
288 imsg_free(&imsg);
289 return err;
292 static const struct got_error *
293 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
294 struct imsgbuf *ibuf)
296 size_t datalen;
298 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
299 if (datalen != 0)
300 return got_error(GOT_ERR_PRIVSEP_LEN);
302 if (imsg->fd == -1)
303 return got_error(GOT_ERR_PRIVSEP_NO_FD);
305 *f = fdopen(imsg->fd, mode);
306 if (*f == NULL)
307 return got_error_from_errno("fdopen");
308 imsg->fd = -1;
310 return NULL;
313 static const struct got_error *
314 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
315 struct got_packidx *packidx, struct got_object_cache *objcache,
316 FILE *basefile, FILE *accumfile)
318 const struct got_error *err = NULL;
319 struct got_imsg_packed_object iobj;
320 struct got_object *obj = NULL;
321 FILE *outfile = NULL;
322 struct got_object_id id;
323 size_t datalen;
324 uint64_t blob_size;
325 uint8_t *buf = NULL;
327 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
328 if (datalen != sizeof(iobj))
329 return got_error(GOT_ERR_PRIVSEP_LEN);
330 memcpy(&iobj, imsg->data, sizeof(iobj));
331 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
333 obj = got_object_cache_get(objcache, &id);
334 if (obj) {
335 obj->refcnt++;
336 } else {
337 err = open_object(&obj, pack, packidx, iobj.idx, &id,
338 objcache);
339 if (err)
340 return err;
343 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
344 if (err)
345 goto done;
347 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
348 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
349 if (err)
350 goto done;
351 } else
352 blob_size = obj->size;
354 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
355 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
356 obj, pack);
357 else
358 err = got_packfile_extract_object(pack, obj, outfile, basefile,
359 accumfile);
360 if (err)
361 goto done;
363 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
364 done:
365 free(buf);
366 if (outfile && fclose(outfile) == EOF && err == NULL)
367 err = got_error_from_errno("fclose");
368 got_object_close(obj);
369 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
370 got_privsep_send_error(ibuf, err);
372 return err;
375 static const struct got_error *
376 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
377 struct got_packidx *packidx, struct got_object_cache *objcache)
379 const struct got_error *err = NULL;
380 struct got_imsg_packed_object iobj;
381 struct got_object *obj = NULL;
382 struct got_tag_object *tag = NULL;
383 uint8_t *buf = NULL;
384 size_t len;
385 struct got_object_id id;
386 size_t datalen;
388 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
389 if (datalen != sizeof(iobj))
390 return got_error(GOT_ERR_PRIVSEP_LEN);
391 memcpy(&iobj, imsg->data, sizeof(iobj));
392 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
394 obj = got_object_cache_get(objcache, &id);
395 if (obj) {
396 obj->refcnt++;
397 } else {
398 err = open_object(&obj, pack, packidx, iobj.idx, &id,
399 objcache);
400 if (err)
401 return err;
404 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
405 if (err)
406 goto done;
408 obj->size = len;
409 err = got_object_parse_tag(&tag, buf, len);
410 if (err)
411 goto done;
413 err = got_privsep_send_tag(ibuf, tag);
414 done:
415 free(buf);
416 got_object_close(obj);
417 if (tag)
418 got_object_tag_close(tag);
419 if (err) {
420 if (err->code == GOT_ERR_PRIVSEP_PIPE)
421 err = NULL;
422 else
423 got_privsep_send_error(ibuf, err);
426 return err;
429 static struct got_parsed_tree_entry *
430 find_entry_by_name(struct got_pathlist_head *entries, int nentries,
431 const char *name, size_t len)
433 struct got_pathlist_entry *pe;
435 /* Note that tree entries are sorted in strncmp() order. */
436 TAILQ_FOREACH(pe, entries, entry) {
437 int cmp = strncmp(pe->path, name, len);
438 if (cmp < 0)
439 continue;
440 if (cmp > 0)
441 break;
442 if (pe->path[len] == '\0')
443 return (struct got_parsed_tree_entry *)pe->data;
445 return NULL;
448 static const struct got_error *
449 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
450 struct got_pathlist_head *entries1, int *nentries1,
451 struct got_pathlist_head *entries2, int *nentries2,
452 const char *path, struct got_pack *pack, struct got_packidx *packidx,
453 struct imsgbuf *ibuf, struct got_object_cache *objcache)
455 const struct got_error *err = NULL;
456 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
457 const char *seg, *s;
458 size_t seglen;
460 *changed = 0;
462 /* We not do support comparing the root path. */
463 if (got_path_is_root_dir(path))
464 return got_error_path(path, GOT_ERR_BAD_PATH);
466 s = path;
467 while (*s == '/')
468 s++;
469 seg = s;
470 seglen = 0;
471 while (*s) {
472 if (*s != '/') {
473 s++;
474 seglen++;
475 if (*s)
476 continue;
479 pte1 = find_entry_by_name(entries1, *nentries1, seg, seglen);
480 if (pte1 == NULL) {
481 err = got_error(GOT_ERR_NO_OBJ);
482 break;
485 pte2 = find_entry_by_name(entries2, *nentries2, seg, seglen);
486 if (pte2 == NULL) {
487 *changed = 1;
488 break;
491 if (pte1->mode != pte2->mode) {
492 *changed = 1;
493 break;
496 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
497 *changed = 0;
498 break;
501 if (*s == '\0') { /* final path element */
502 *changed = 1;
503 break;
506 seg = s + 1;
507 s++;
508 seglen = 0;
509 if (*s) {
510 struct got_object_id id1, id2;
511 int idx;
513 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
514 idx = got_packidx_get_object_idx(packidx, &id1);
515 if (idx == -1) {
516 err = got_error_no_obj(&id1);
517 break;
519 got_object_parsed_tree_entries_free(entries1);
520 *nentries1 = 0;
521 free(*buf1);
522 *buf1 = NULL;
523 err = open_tree(buf1, entries1, nentries1, pack,
524 packidx, idx, &id1, objcache);
525 pte1 = NULL;
526 if (err)
527 break;
529 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
530 idx = got_packidx_get_object_idx(packidx, &id2);
531 if (idx == -1) {
532 err = got_error_no_obj(&id2);
533 break;
535 got_object_parsed_tree_entries_free(entries2);
536 *nentries2 = 0;
537 free(*buf2);
538 *buf2 = NULL;
539 err = open_tree(buf2, entries2, nentries2, pack,
540 packidx, idx, &id2, objcache);
541 pte2 = NULL;
542 if (err)
543 break;
547 return err;
550 static const struct got_error *
551 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
552 struct imsgbuf *ibuf)
554 const struct got_error *err;
555 struct ibuf *wbuf;
556 size_t i;
558 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
559 sizeof(struct got_imsg_traversed_commits) +
560 ncommits * SHA1_DIGEST_LENGTH);
561 if (wbuf == NULL)
562 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
564 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1) {
565 err = got_error_from_errno("imsg_add TRAVERSED_COMMITS");
566 ibuf_free(wbuf);
567 return err;
569 for (i = 0; i < ncommits; i++) {
570 struct got_object_id *id = &commit_ids[i];
571 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
572 err = got_error_from_errno(
573 "imsg_add TRAVERSED_COMMITS");
574 ibuf_free(wbuf);
575 return err;
579 wbuf->fd = -1;
580 imsg_close(ibuf, wbuf);
582 return got_privsep_flush_imsg(ibuf);
585 static const struct got_error *
586 send_commit_traversal_done(struct imsgbuf *ibuf)
588 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
589 NULL, 0) == -1)
590 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
592 return got_privsep_flush_imsg(ibuf);
596 static const struct got_error *
597 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
598 struct got_pack *pack, struct got_packidx *packidx,
599 struct got_object_cache *objcache)
601 const struct got_error *err = NULL;
602 struct got_imsg_packed_object iobj;
603 struct got_object_qid *pid;
604 struct got_commit_object *commit = NULL, *pcommit = NULL;
605 struct got_pathlist_head entries, pentries;
606 int nentries = 0, pnentries = 0;
607 struct got_object_id id;
608 size_t datalen, path_len;
609 char *path = NULL;
610 const int min_alloc = 64;
611 int changed = 0, ncommits = 0, nallocated = 0;
612 struct got_object_id *commit_ids = NULL;
614 TAILQ_INIT(&entries);
615 TAILQ_INIT(&pentries);
617 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
618 if (datalen < sizeof(iobj))
619 return got_error(GOT_ERR_PRIVSEP_LEN);
620 memcpy(&iobj, imsg->data, sizeof(iobj));
621 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
623 path_len = datalen - sizeof(iobj) - 1;
624 if (path_len < 0)
625 return got_error(GOT_ERR_PRIVSEP_LEN);
626 if (path_len > 0) {
627 path = imsg->data + sizeof(iobj);
628 if (path[path_len] != '\0')
629 return got_error(GOT_ERR_PRIVSEP_LEN);
632 nallocated = min_alloc;
633 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
634 if (commit_ids == NULL)
635 return got_error_from_errno("reallocarray");
637 do {
638 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
639 int idx;
641 if (sigint_received) {
642 err = got_error(GOT_ERR_CANCELLED);
643 goto done;
646 if (commit == NULL) {
647 idx = got_packidx_get_object_idx(packidx, &id);
648 if (idx == -1)
649 break;
650 err = open_commit(&commit, pack, packidx,
651 idx, &id, objcache);
652 if (err) {
653 if (err->code != GOT_ERR_NO_OBJ)
654 goto done;
655 err = NULL;
656 break;
660 if (sizeof(struct got_imsg_traversed_commits) +
661 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
662 err = send_traversed_commits(commit_ids, ncommits,
663 ibuf);
664 if (err)
665 goto done;
666 ncommits = 0;
668 ncommits++;
669 if (ncommits > nallocated) {
670 struct got_object_id *new;
671 nallocated += min_alloc;
672 new = reallocarray(commit_ids, nallocated,
673 sizeof(*commit_ids));
674 if (new == NULL) {
675 err = got_error_from_errno("reallocarray");
676 goto done;
678 commit_ids = new;
680 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
681 SHA1_DIGEST_LENGTH);
683 pid = STAILQ_FIRST(&commit->parent_ids);
684 if (pid == NULL)
685 break;
687 idx = got_packidx_get_object_idx(packidx, &pid->id);
688 if (idx == -1)
689 break;
691 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
692 objcache);
693 if (err) {
694 if (err->code != GOT_ERR_NO_OBJ)
695 goto done;
696 err = NULL;
697 break;
700 if (path[0] == '/' && path[1] == '\0') {
701 if (got_object_id_cmp(pcommit->tree_id,
702 commit->tree_id) != 0) {
703 changed = 1;
704 break;
706 } else {
707 int pidx;
708 uint8_t *buf = NULL, *pbuf = NULL;
710 idx = got_packidx_get_object_idx(packidx,
711 commit->tree_id);
712 if (idx == -1)
713 break;
714 pidx = got_packidx_get_object_idx(packidx,
715 pcommit->tree_id);
716 if (pidx == -1)
717 break;
719 err = open_tree(&buf, &entries, &nentries, pack,
720 packidx, idx, commit->tree_id, objcache);
721 if (err)
722 goto done;
723 err = open_tree(&pbuf, &pentries, &pnentries, pack,
724 packidx, pidx, pcommit->tree_id, objcache);
725 if (err) {
726 free(buf);
727 goto done;
730 err = tree_path_changed(&changed, &buf, &pbuf,
731 &entries, &nentries, &pentries, &pnentries, path,
732 pack, packidx, ibuf, objcache);
734 got_object_parsed_tree_entries_free(&entries);
735 nentries = 0;
736 free(buf);
737 got_object_parsed_tree_entries_free(&pentries);
738 pnentries = 0;
739 free(pbuf);
740 if (err) {
741 if (err->code != GOT_ERR_NO_OBJ)
742 goto done;
743 err = NULL;
744 break;
748 if (!changed) {
749 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
750 got_object_commit_close(commit);
751 commit = pcommit;
752 pcommit = NULL;
754 } while (!changed);
756 if (ncommits > 0) {
757 err = send_traversed_commits(commit_ids, ncommits, ibuf);
758 if (err)
759 goto done;
761 if (changed) {
762 err = got_privsep_send_commit(ibuf, commit);
763 if (err)
764 goto done;
767 err = send_commit_traversal_done(ibuf);
768 done:
769 free(commit_ids);
770 if (commit)
771 got_object_commit_close(commit);
772 if (pcommit)
773 got_object_commit_close(pcommit);
774 if (nentries != 0)
775 got_object_parsed_tree_entries_free(&entries);
776 if (pnentries != 0)
777 got_object_parsed_tree_entries_free(&pentries);
778 if (err) {
779 if (err->code == GOT_ERR_PRIVSEP_PIPE)
780 err = NULL;
781 else
782 got_privsep_send_error(ibuf, err);
785 return err;
788 static const struct got_error *
789 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
790 struct got_pack *pack, struct got_packidx *packidx,
791 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
793 const struct got_error *err = NULL;
794 uint8_t *buf = NULL;
795 uint64_t size = 0;
796 FILE *outfile = NULL;
797 struct got_imsg_packed_object iobj;
798 struct got_object *obj;
799 struct got_object_id id;
800 size_t datalen;
802 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
803 if (datalen != sizeof(iobj))
804 return got_error(GOT_ERR_PRIVSEP_LEN);
805 memcpy(&iobj, imsg->data, sizeof(iobj));
806 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
808 obj = got_object_cache_get(objcache, &id);
809 if (obj) {
810 obj->refcnt++;
811 } else {
812 err = open_object(&obj, pack, packidx, iobj.idx, &id,
813 objcache);
814 if (err)
815 return err;
818 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
819 if (err)
820 return err;
822 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
823 err = got_pack_get_max_delta_object_size(&size, obj, pack);
824 if (err)
825 goto done;
826 } else
827 size = obj->size;
829 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
830 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
831 obj, pack);
832 else
833 err = got_packfile_extract_object(pack, obj, outfile, basefile,
834 accumfile);
835 if (err)
836 goto done;
838 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
839 done:
840 free(buf);
841 if (outfile && fclose(outfile) == EOF && err == NULL)
842 err = got_error_from_errno("fclose");
843 got_object_close(obj);
844 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
845 got_privsep_send_error(ibuf, err);
847 return err;
850 static const struct got_error *
851 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
852 off_t base_offset)
854 const struct got_error *err;
855 int idx;
857 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
858 if (err)
859 return err;
860 if (idx == -1)
861 return got_error(GOT_ERR_BAD_PACKIDX);
863 return got_packidx_get_object_id(base_id, packidx, idx);
866 static const struct got_error *
867 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
868 FILE *delta_outfile, struct got_pack *pack,
869 struct got_packidx *packidx)
871 const struct got_error *err = NULL;
872 struct got_imsg_raw_delta_request req;
873 size_t datalen, delta_size, delta_compressed_size;
874 off_t delta_offset;
875 uint8_t *delta_buf = NULL;
876 struct got_object_id id, base_id;
877 off_t base_offset, delta_out_offset = 0;
878 uint64_t base_size = 0, result_size = 0;
879 size_t w;
881 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
882 if (datalen != sizeof(req))
883 return got_error(GOT_ERR_PRIVSEP_LEN);
884 memcpy(&req, imsg->data, sizeof(req));
885 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
887 imsg->fd = -1;
889 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
890 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
891 &base_size, &result_size, pack, packidx, req.idx);
892 if (err)
893 goto done;
895 /*
896 * If this is an offset delta we must determine the base
897 * object ID ourselves.
898 */
899 if (base_offset != 0) {
900 err = get_base_object_id(&base_id, packidx, base_offset);
901 if (err)
902 goto done;
905 delta_out_offset = ftello(delta_outfile);
906 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
907 if (w != delta_compressed_size) {
908 err = got_ferror(delta_outfile, GOT_ERR_IO);
909 goto done;
911 if (fflush(delta_outfile) == -1) {
912 err = got_error_from_errno("fflush");
913 goto done;
916 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
917 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
918 &base_id);
919 done:
920 free(delta_buf);
921 return err;
924 struct search_deltas_arg {
925 struct imsgbuf *ibuf;
926 struct got_packidx *packidx;
927 struct got_pack *pack;
928 struct got_object_idset *idset;
929 FILE *delta_outfile;
930 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
931 size_t ndeltas;
932 };
934 static const struct got_error *
935 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
937 const struct got_error *err;
938 struct search_deltas_arg *a = arg;
939 int obj_idx;
940 uint8_t *delta_buf = NULL;
941 uint64_t base_size, result_size;
942 size_t delta_size, delta_compressed_size;
943 off_t delta_offset, base_offset;
944 struct got_object_id base_id;
946 if (sigint_received)
947 return got_error(GOT_ERR_CANCELLED);
949 obj_idx = got_packidx_get_object_idx(a->packidx, id);
950 if (obj_idx == -1)
951 return NULL; /* object not present in our pack file */
953 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
954 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
955 &base_size, &result_size, a->pack, a->packidx, obj_idx);
956 if (err) {
957 if (err->code == GOT_ERR_OBJ_TYPE)
958 return NULL; /* object not stored as a delta */
959 return err;
962 /*
963 * If this is an offset delta we must determine the base
964 * object ID ourselves.
965 */
966 if (base_offset != 0) {
967 err = get_base_object_id(&base_id, a->packidx, base_offset);
968 if (err)
969 goto done;
972 if (got_object_idset_contains(a->idset, &base_id)) {
973 struct got_imsg_reused_delta *delta;
974 off_t delta_out_offset = ftello(a->delta_outfile);
975 size_t w;
977 w = fwrite(delta_buf, 1, delta_compressed_size,
978 a->delta_outfile);
979 if (w != delta_compressed_size) {
980 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
981 goto done;
984 delta = &a->deltas[a->ndeltas++];
985 memcpy(&delta->id, id, sizeof(delta->id));
986 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
987 delta->base_size = base_size;
988 delta->result_size = result_size;
989 delta->delta_size = delta_size;
990 delta->delta_compressed_size = delta_compressed_size;
991 delta->delta_offset = delta_offset;
992 delta->delta_out_offset = delta_out_offset;
994 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
995 err = got_privsep_send_reused_deltas(a->ibuf,
996 a->deltas, a->ndeltas);
997 if (err)
998 goto done;
999 a->ndeltas = 0;
1002 done:
1003 free(delta_buf);
1004 return err;
1007 static const struct got_error *
1008 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1010 const struct got_error *err = NULL;
1011 int done = 0;
1012 struct got_object_id *ids;
1013 size_t nids, i;
1015 for (;;) {
1016 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1017 if (err || done)
1018 break;
1019 for (i = 0; i < nids; i++) {
1020 err = got_object_idset_add(idset, &ids[i], NULL);
1021 if (err) {
1022 free(ids);
1023 return err;
1026 free(ids);
1029 return err;
1032 static const struct got_error *
1033 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1034 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1036 const struct got_error *err = NULL;
1037 struct got_object_idset *idset;
1038 struct search_deltas_arg sda;
1040 idset = got_object_idset_alloc();
1041 if (idset == NULL)
1042 return got_error_from_errno("got_object_idset_alloc");
1044 err = recv_object_ids(idset, ibuf);
1045 if (err)
1046 return err;
1048 memset(&sda, 0, sizeof(sda));
1049 sda.ibuf = ibuf;
1050 sda.idset = idset;
1051 sda.pack = pack;
1052 sda.packidx = packidx;
1053 sda.delta_outfile = delta_outfile;
1054 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1055 if (err)
1056 goto done;
1058 if (sda.ndeltas > 0) {
1059 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1060 sda.ndeltas);
1061 if (err)
1062 goto done;
1065 if (fflush(delta_outfile) == -1) {
1066 err = got_error_from_errno("fflush");
1067 goto done;
1070 err = got_privsep_send_reused_deltas_done(ibuf);
1071 done:
1072 got_object_idset_free(idset);
1073 return err;
1076 static const struct got_error *
1077 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1079 const struct got_error *err = NULL;
1080 struct imsg imsg;
1081 struct got_imsg_packidx ipackidx;
1082 size_t datalen;
1083 struct got_packidx *p;
1085 *packidx = NULL;
1087 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1088 if (err)
1089 return err;
1091 p = calloc(1, sizeof(*p));
1092 if (p == NULL) {
1093 err = got_error_from_errno("calloc");
1094 goto done;
1097 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1098 err = got_error(GOT_ERR_PRIVSEP_MSG);
1099 goto done;
1102 if (imsg.fd == -1) {
1103 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1104 goto done;
1107 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1108 if (datalen != sizeof(ipackidx)) {
1109 err = got_error(GOT_ERR_PRIVSEP_LEN);
1110 goto done;
1112 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1114 p->len = ipackidx.len;
1115 p->fd = dup(imsg.fd);
1116 if (p->fd == -1) {
1117 err = got_error_from_errno("dup");
1118 goto done;
1120 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1121 err = got_error_from_errno("lseek");
1122 goto done;
1125 #ifndef GOT_PACK_NO_MMAP
1126 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1127 if (p->map == MAP_FAILED)
1128 p->map = NULL; /* fall back to read(2) */
1129 #endif
1130 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1131 done:
1132 if (err) {
1133 if (imsg.fd != -1)
1134 close(imsg.fd);
1135 got_packidx_close(p);
1136 } else
1137 *packidx = p;
1138 imsg_free(&imsg);
1139 return err;
1142 static const struct got_error *
1143 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1145 const struct got_error *err = NULL;
1146 struct imsg imsg;
1147 struct got_imsg_pack ipack;
1148 size_t datalen;
1149 struct got_pack *pack;
1151 *packp = NULL;
1153 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1154 if (err)
1155 return err;
1157 pack = calloc(1, sizeof(*pack));
1158 if (pack == NULL) {
1159 err = got_error_from_errno("calloc");
1160 goto done;
1163 if (imsg.hdr.type != GOT_IMSG_PACK) {
1164 err = got_error(GOT_ERR_PRIVSEP_MSG);
1165 goto done;
1168 if (imsg.fd == -1) {
1169 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1170 goto done;
1173 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1174 if (datalen != sizeof(ipack)) {
1175 err = got_error(GOT_ERR_PRIVSEP_LEN);
1176 goto done;
1178 memcpy(&ipack, imsg.data, sizeof(ipack));
1180 pack->filesize = ipack.filesize;
1181 pack->fd = dup(imsg.fd);
1182 if (pack->fd == -1) {
1183 err = got_error_from_errno("dup");
1184 goto done;
1186 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1187 err = got_error_from_errno("lseek");
1188 goto done;
1190 pack->path_packfile = strdup(ipack.path_packfile);
1191 if (pack->path_packfile == NULL) {
1192 err = got_error_from_errno("strdup");
1193 goto done;
1196 pack->delta_cache = got_delta_cache_alloc(100,
1197 GOT_DELTA_RESULT_SIZE_CACHED_MAX);
1198 if (pack->delta_cache == NULL) {
1199 err = got_error_from_errno("got_delta_cache_alloc");
1200 goto done;
1203 #ifndef GOT_PACK_NO_MMAP
1204 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1205 pack->fd, 0);
1206 if (pack->map == MAP_FAILED)
1207 pack->map = NULL; /* fall back to read(2) */
1208 #endif
1209 done:
1210 if (err) {
1211 if (imsg.fd != -1)
1212 close(imsg.fd);
1213 free(pack);
1214 } else
1215 *packp = pack;
1216 imsg_free(&imsg);
1217 return err;
1220 int
1221 main(int argc, char *argv[])
1223 const struct got_error *err = NULL;
1224 struct imsgbuf ibuf;
1225 struct imsg imsg;
1226 struct got_packidx *packidx = NULL;
1227 struct got_pack *pack = NULL;
1228 struct got_object_cache objcache;
1229 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1231 //static int attached;
1232 //while (!attached) sleep(1);
1234 signal(SIGINT, catch_sigint);
1236 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1238 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1239 if (err) {
1240 err = got_error_from_errno("got_object_cache_init");
1241 got_privsep_send_error(&ibuf, err);
1242 return 1;
1245 #ifndef PROFILE
1246 /* revoke access to most system calls */
1247 if (pledge("stdio recvfd", NULL) == -1) {
1248 err = got_error_from_errno("pledge");
1249 got_privsep_send_error(&ibuf, err);
1250 return 1;
1252 #endif
1254 err = receive_packidx(&packidx, &ibuf);
1255 if (err) {
1256 got_privsep_send_error(&ibuf, err);
1257 return 1;
1260 err = receive_pack(&pack, &ibuf);
1261 if (err) {
1262 got_privsep_send_error(&ibuf, err);
1263 return 1;
1266 for (;;) {
1267 imsg.fd = -1;
1269 if (sigint_received) {
1270 err = got_error(GOT_ERR_CANCELLED);
1271 break;
1274 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1275 if (err) {
1276 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1277 err = NULL;
1278 break;
1281 if (imsg.hdr.type == GOT_IMSG_STOP)
1282 break;
1284 switch (imsg.hdr.type) {
1285 case GOT_IMSG_TMPFD:
1286 if (basefile == NULL) {
1287 err = receive_tempfile(&basefile, "w+",
1288 &imsg, &ibuf);
1289 } else if (accumfile == NULL) {
1290 err = receive_tempfile(&accumfile, "w+",
1291 &imsg, &ibuf);
1292 } else
1293 err = got_error(GOT_ERR_PRIVSEP_MSG);
1294 break;
1295 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1296 err = object_request(&imsg, &ibuf, pack, packidx,
1297 &objcache);
1298 break;
1299 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1300 if (basefile == NULL || accumfile == NULL) {
1301 err = got_error(GOT_ERR_PRIVSEP_MSG);
1302 break;
1304 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1305 &objcache, basefile, accumfile);
1306 break;
1307 case GOT_IMSG_RAW_DELTA_OUTFD:
1308 if (delta_outfile != NULL) {
1309 err = got_error(GOT_ERR_PRIVSEP_MSG);
1310 break;
1312 err = receive_tempfile(&delta_outfile, "w",
1313 &imsg, &ibuf);
1314 break;
1315 case GOT_IMSG_RAW_DELTA_REQUEST:
1316 if (delta_outfile == NULL) {
1317 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1318 break;
1320 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1321 pack, packidx);
1322 break;
1323 case GOT_IMSG_DELTA_REUSE_REQUEST:
1324 if (delta_outfile == NULL) {
1325 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1326 break;
1328 err = delta_reuse_request(&imsg, &ibuf,
1329 delta_outfile, pack, packidx);
1330 break;
1331 case GOT_IMSG_COMMIT_REQUEST:
1332 err = commit_request(&imsg, &ibuf, pack, packidx,
1333 &objcache);
1334 break;
1335 case GOT_IMSG_TREE_REQUEST:
1336 err = tree_request(&imsg, &ibuf, pack, packidx,
1337 &objcache);
1338 break;
1339 case GOT_IMSG_BLOB_REQUEST:
1340 if (basefile == NULL || accumfile == NULL) {
1341 err = got_error(GOT_ERR_PRIVSEP_MSG);
1342 break;
1344 err = blob_request(&imsg, &ibuf, pack, packidx,
1345 &objcache, basefile, accumfile);
1346 break;
1347 case GOT_IMSG_TAG_REQUEST:
1348 err = tag_request(&imsg, &ibuf, pack, packidx,
1349 &objcache);
1350 break;
1351 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1352 err = commit_traversal_request(&imsg, &ibuf, pack,
1353 packidx, &objcache);
1354 break;
1355 default:
1356 err = got_error(GOT_ERR_PRIVSEP_MSG);
1357 break;
1360 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1361 err = got_error_from_errno("close");
1362 imsg_free(&imsg);
1363 if (err)
1364 break;
1367 if (packidx)
1368 got_packidx_close(packidx);
1369 if (pack)
1370 got_pack_close(pack);
1371 got_object_cache_close(&objcache);
1372 imsg_clear(&ibuf);
1373 if (basefile && fclose(basefile) == EOF && err == NULL)
1374 err = got_error_from_errno("fclose");
1375 if (accumfile && fclose(accumfile) == EOF && err == NULL)
1376 err = got_error_from_errno("fclose");
1377 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1378 err = got_error_from_errno("fclose");
1379 if (err) {
1380 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1381 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1382 got_privsep_send_error(&ibuf, err);
1385 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1386 err = got_error_from_errno("close");
1387 return err ? 1 : 0;