Blob


1 /*
2 * Copyright (c) 2020 Ori Bernstein
3 * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
22 #include <limits.h>
23 #include <stdio.h>
24 #include <stdint.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <time.h>
28 #include <imsg.h>
29 #include <inttypes.h>
30 #include <unistd.h>
32 #include "got_error.h"
33 #include "got_cancel.h"
34 #include "got_object.h"
35 #include "got_reference.h"
36 #include "got_repository_admin.h"
37 #include "got_path.h"
39 #include "got_lib_delta.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_idset.h"
43 #include "got_lib_privsep.h"
44 #include "got_lib_ratelimit.h"
45 #include "got_lib_pack.h"
46 #include "got_lib_pack_create.h"
47 #include "got_lib_repository.h"
49 struct send_id_arg {
50 struct imsgbuf *ibuf;
51 struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
52 size_t nids;
53 };
55 static const struct got_error *
56 send_id(struct got_object_id *id, void *data, void *arg)
57 {
58 const struct got_error *err = NULL;
59 struct send_id_arg *a = arg;
61 a->ids[a->nids++] = id;
63 if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
64 err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
65 if (err)
66 return err;
67 a->nids = 0;
68 }
70 return NULL;
71 }
73 static const struct got_error *
74 send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
75 {
76 const struct got_error *err;
77 struct send_id_arg sia;
79 memset(&sia, 0, sizeof(sia));
80 sia.ibuf = ibuf;
81 err = got_object_idset_for_each(idset, send_id, &sia);
82 if (err)
83 return err;
85 if (sia.nids > 0) {
86 err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
87 if (err)
88 return err;
89 }
91 return got_privsep_send_object_idlist_done(ibuf);
92 }
94 static const struct got_error *
95 recv_reused_delta(struct got_imsg_reused_delta *delta,
96 struct got_object_idset *idset, struct got_pack_metavec *v)
97 {
98 struct got_pack_meta *m, *base;
100 if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
101 delta->delta_offset +
102 delta->delta_compressed_size < delta->delta_offset)
103 return got_error(GOT_ERR_BAD_PACKFILE);
105 m = got_object_idset_get(idset, &delta->id);
106 if (m == NULL)
107 return got_error(GOT_ERR_NO_OBJ);
109 base = got_object_idset_get(idset, &delta->base_id);
110 if (base == NULL)
111 return got_error(GOT_ERR_NO_OBJ);
113 m->delta_len = delta->delta_size;
114 m->delta_compressed_len = delta->delta_compressed_size;
115 m->delta_offset = delta->delta_out_offset;
116 m->prev = base;
117 m->size = delta->result_size;
118 m->reused_delta_offset = delta->delta_offset;
119 m->base_obj_id = got_object_id_dup(&delta->base_id);
120 if (m->base_obj_id == NULL)
121 return got_error_from_errno("got_object_id_dup");
123 return got_pack_add_meta(m, v);
126 static const struct got_error *
127 prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx,
128 int delta_outfd, struct got_repository *repo)
130 const struct got_error *err = NULL;
132 if (!pack->child_has_delta_outfd) {
133 int outfd_child;
134 outfd_child = dup(delta_outfd);
135 if (outfd_child == -1) {
136 err = got_error_from_errno("dup");
137 goto done;
139 err = got_privsep_send_raw_delta_outfd(
140 pack->privsep_child->ibuf, outfd_child);
141 if (err)
142 goto done;
143 pack->child_has_delta_outfd = 1;
146 err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf);
147 done:
148 return err;
151 const struct got_error *
152 got_pack_search_deltas(struct got_pack_metavec *v,
153 struct got_object_idset *idset, int delta_cache_fd,
154 int ncolored, int nfound, int ntrees, int ncommits,
155 struct got_repository *repo,
156 got_pack_progress_cb progress_cb, void *progress_arg,
157 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
159 const struct got_error *err = NULL;
160 struct got_packidx *packidx;
161 struct got_pack *pack;
162 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
163 size_t ndeltas, i;
165 err = got_pack_find_pack_for_reuse(&packidx, repo);
166 if (err)
167 return err;
169 if (packidx == NULL)
170 return NULL;
172 err = got_pack_cache_pack_for_packidx(&pack, packidx, repo);
173 if (err)
174 return err;
176 if (pack->privsep_child == NULL) {
177 err = got_pack_start_privsep_child(pack, packidx);
178 if (err)
179 return err;
182 err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo);
183 if (err)
184 return err;
186 err = send_idset(pack->privsep_child->ibuf, idset);
187 if (err)
188 return err;
190 for (;;) {
191 int done = 0;
193 if (cancel_cb) {
194 err = (*cancel_cb)(cancel_arg);
195 if (err)
196 break;
199 err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
200 pack->privsep_child->ibuf);
201 if (err || done)
202 break;
204 for (i = 0; i < ndeltas; i++) {
205 struct got_imsg_reused_delta *delta = &deltas[i];
206 err = recv_reused_delta(delta, idset, v);
207 if (err)
208 goto done;
211 err = got_pack_report_progress(progress_cb, progress_arg, rl,
212 ncolored, nfound, ntrees, 0L, ncommits,
213 got_object_idset_num_elements(idset), v->nmeta, 0);
214 if (err)
215 break;
217 done:
218 return err;
221 struct recv_painted_commit_arg {
222 int *ncolored;
223 int *nqueued;
224 int *nskip;
225 struct got_object_id_queue *ids;
226 struct got_object_idset *keep;
227 struct got_object_idset *drop;
228 struct got_object_idset *skip;
229 got_pack_progress_cb progress_cb;
230 void *progress_arg;
231 struct got_ratelimit *rl;
232 got_cancel_cb cancel_cb;
233 void *cancel_arg;
234 };
236 static const struct got_error *
237 recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
239 const struct got_error *err = NULL;
240 struct recv_painted_commit_arg *a = arg;
241 struct got_object_qid *qid, *tmp;
243 if (a->cancel_cb) {
244 err = a->cancel_cb(a->cancel_arg);
245 if (err)
246 return err;
249 switch (color) {
250 case COLOR_KEEP:
251 err = got_object_idset_add(a->keep, id, NULL);
252 if (err)
253 return err;
254 (*a->ncolored)++;
255 break;
256 case COLOR_DROP:
257 err = got_object_idset_add(a->drop, id, NULL);
258 if (err)
259 return err;
260 (*a->ncolored)++;
261 break;
262 case COLOR_SKIP:
263 err = got_object_idset_add(a->skip, id, NULL);
264 if (err)
265 return err;
266 break;
267 default:
268 /* should not happen */
269 return got_error_fmt(GOT_ERR_NOT_IMPL,
270 "%s invalid commit color %"PRIdPTR, __func__, color);
273 STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
274 if (got_object_id_cmp(&qid->id, id) != 0)
275 continue;
276 STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
277 color = (intptr_t)qid->data;
278 got_object_qid_free(qid);
279 (*a->nqueued)--;
280 if (color == COLOR_SKIP)
281 (*a->nskip)--;
282 break;
285 return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
286 *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
289 static const struct got_error *
290 paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
291 int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
292 struct got_object_id_queue *ids,
293 struct got_object_idset *keep, struct got_object_idset *drop,
294 struct got_object_idset *skip, struct got_repository *repo,
295 got_pack_progress_cb progress_cb, void *progress_arg,
296 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
298 const struct got_error *err = NULL;
299 struct got_object_id_queue next_ids;
300 struct got_object_qid *qid, *tmp;
301 struct recv_painted_commit_arg arg;
303 STAILQ_INIT(&next_ids);
305 err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
306 idx, id, color);
307 if (err)
308 return err;
310 arg.ncolored = ncolored;
311 arg.nqueued = nqueued;
312 arg.nskip = nskip;
313 arg.ids = ids;
314 arg.keep = keep;
315 arg.drop = drop;
316 arg.skip = skip;
317 arg.progress_cb = progress_cb;
318 arg.progress_arg = progress_arg;
319 arg.rl = rl;
320 arg.cancel_cb = cancel_cb;
321 arg.cancel_arg = cancel_arg;
322 err = got_privsep_recv_painted_commits(&next_ids,
323 recv_painted_commit, &arg, pack->privsep_child->ibuf);
324 if (err)
325 return err;
327 STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
328 struct got_object_qid *old_id;
329 intptr_t qcolor, ocolor;
330 STAILQ_FOREACH(old_id, ids, entry) {
331 if (got_object_id_cmp(&qid->id, &old_id->id))
332 continue;
333 qcolor = (intptr_t)qid->data;
334 ocolor = (intptr_t)old_id->data;
335 STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
336 got_object_qid_free(qid);
337 qid = NULL;
338 if (qcolor != ocolor) {
339 got_pack_paint_commit(old_id, qcolor);
340 if (ocolor == COLOR_SKIP)
341 (*nskip)--;
342 else if (qcolor == COLOR_SKIP)
343 (*nskip)++;
345 break;
348 while (!STAILQ_EMPTY(&next_ids)) {
349 qid = STAILQ_FIRST(&next_ids);
350 STAILQ_REMOVE_HEAD(&next_ids, entry);
351 got_pack_paint_commit(qid, color);
352 STAILQ_INSERT_TAIL(ids, qid, entry);
353 (*nqueued)++;
354 if (color == COLOR_SKIP)
355 (*nskip)++;
358 return err;
361 const struct got_error *
362 got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
363 struct got_object_idset *keep, struct got_object_idset *drop,
364 struct got_object_idset *skip, struct got_repository *repo,
365 got_pack_progress_cb progress_cb, void *progress_arg,
366 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
368 const struct got_error *err = NULL;
369 struct got_commit_object *commit = NULL;
370 struct got_packidx *packidx = NULL;
371 struct got_pack *pack = NULL;
372 const struct got_object_id_queue *parents;
373 struct got_object_qid *qid = NULL;
374 int nqueued = nids, nskip = 0;
375 int idx;
377 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
378 intptr_t color;
380 if (cancel_cb) {
381 err = cancel_cb(cancel_arg);
382 if (err)
383 break;
386 qid = STAILQ_FIRST(ids);
387 STAILQ_REMOVE_HEAD(ids, entry);
388 nqueued--;
389 color = (intptr_t)qid->data;
390 if (color == COLOR_SKIP)
391 nskip--;
393 if (got_object_idset_contains(skip, &qid->id)) {
394 got_object_qid_free(qid);
395 qid = NULL;
396 continue;
398 if (color == COLOR_KEEP &&
399 got_object_idset_contains(keep, &qid->id)) {
400 got_object_qid_free(qid);
401 qid = NULL;
402 continue;
404 if (color == COLOR_DROP &&
405 got_object_idset_contains(drop, &qid->id)) {
406 got_object_qid_free(qid);
407 qid = NULL;
408 continue;
411 /* Pinned pack may have moved to different cache slot. */
412 pack = got_repo_get_pinned_pack(repo);
414 if (packidx && pack) {
415 idx = got_packidx_get_object_idx(packidx, &qid->id);
416 if (idx != -1) {
417 err = paint_packed_commits(pack, &qid->id,
418 idx, color, ncolored, &nqueued, &nskip,
419 ids, keep, drop, skip, repo,
420 progress_cb, progress_arg, rl,
421 cancel_cb, cancel_arg);
422 if (err)
423 break;
424 got_object_qid_free(qid);
425 qid = NULL;
426 continue;
430 switch (color) {
431 case COLOR_KEEP:
432 if (got_object_idset_contains(drop, &qid->id)) {
433 err = got_pack_paint_commit(qid, COLOR_SKIP);
434 if (err)
435 goto done;
436 } else
437 (*ncolored)++;
438 err = got_object_idset_add(keep, &qid->id, NULL);
439 if (err)
440 goto done;
441 break;
442 case COLOR_DROP:
443 if (got_object_idset_contains(keep, &qid->id)) {
444 err = got_pack_paint_commit(qid, COLOR_SKIP);
445 if (err)
446 goto done;
447 } else
448 (*ncolored)++;
449 err = got_object_idset_add(drop, &qid->id, NULL);
450 if (err)
451 goto done;
452 break;
453 case COLOR_SKIP:
454 if (!got_object_idset_contains(skip, &qid->id)) {
455 err = got_object_idset_add(skip, &qid->id,
456 NULL);
457 if (err)
458 goto done;
460 break;
461 default:
462 /* should not happen */
463 err = got_error_fmt(GOT_ERR_NOT_IMPL,
464 "%s invalid commit color %"PRIdPTR, __func__,
465 color);
466 goto done;
469 err = got_pack_report_progress(progress_cb, progress_arg, rl,
470 *ncolored, 0, 0, 0L, 0, 0, 0, 0);
471 if (err)
472 break;
474 err = got_object_open_as_commit(&commit, repo, &qid->id);
475 if (err)
476 break;
478 parents = got_object_commit_get_parent_ids(commit);
479 if (parents) {
480 struct got_object_qid *pid;
481 color = (intptr_t)qid->data;
482 STAILQ_FOREACH(pid, parents, entry) {
483 err = got_pack_queue_commit_id(ids, &pid->id,
484 color, repo);
485 if (err)
486 break;
487 nqueued++;
488 if (color == COLOR_SKIP)
489 nskip++;
493 if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
494 if (packidx == NULL) {
495 err = got_pack_find_pack_for_commit_painting(
496 &packidx, ids, nqueued, repo);
497 if (err)
498 goto done;
500 if (packidx != NULL) {
501 err = got_pack_cache_pack_for_packidx(&pack,
502 packidx, repo);
503 if (err)
504 goto done;
505 if (pack->privsep_child == NULL) {
506 err = got_pack_start_privsep_child(
507 pack, packidx);
508 if (err)
509 goto done;
511 err = got_privsep_init_commit_painting(
512 pack->privsep_child->ibuf);
513 if (err)
514 goto done;
515 err = send_idset(pack->privsep_child->ibuf,
516 keep);
517 if (err)
518 goto done;
519 err = send_idset(pack->privsep_child->ibuf, drop);
520 if (err)
521 goto done;
522 err = send_idset(pack->privsep_child->ibuf, skip);
523 if (err)
524 goto done;
525 err = got_repo_pin_pack(repo, packidx, pack);
526 if (err)
527 goto done;
531 got_object_commit_close(commit);
532 commit = NULL;
534 got_object_qid_free(qid);
535 qid = NULL;
537 done:
538 if (pack) {
539 const struct got_error *pack_err;
540 pack_err = got_privsep_send_painting_commits_done(
541 pack->privsep_child->ibuf);
542 if (err == NULL)
543 err = pack_err;
545 if (commit)
546 got_object_commit_close(commit);
547 got_object_qid_free(qid);
548 got_repo_unpin_pack(repo);
549 return err;
552 struct load_packed_obj_arg {
553 /* output parameters: */
554 struct got_object_id *id;
555 char *dpath;
556 time_t mtime;
558 /* input parameters: */
559 uint32_t seed;
560 int want_meta;
561 struct got_object_idset *idset;
562 struct got_object_idset *idset_exclude;
563 int loose_obj_only;
564 int *ncolored;
565 int *nfound;
566 int *ntrees;
567 got_pack_progress_cb progress_cb;
568 void *progress_arg;
569 struct got_ratelimit *rl;
570 got_cancel_cb cancel_cb;
571 void *cancel_arg;
572 };
574 static const struct got_error *
575 load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
576 struct got_repository *repo)
578 struct load_packed_obj_arg *a = arg;
580 if (got_object_idset_contains(a->idset, id) ||
581 got_object_idset_contains(a->idset_exclude, id))
582 return NULL;
584 return got_pack_add_object(a->want_meta,
585 a->want_meta ? a->idset : a->idset_exclude,
586 id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
587 repo, a->ncolored, a->nfound, a->ntrees,
588 a->progress_cb, a->progress_arg, a->rl);
591 static const struct got_error *
592 load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
593 struct got_object_id *id, const char *dpath, struct got_repository *repo)
595 const struct got_error *err;
596 struct load_packed_obj_arg *a = arg;
597 const char *relpath;
599 /*
600 * When we receive a tree's ID and path but not the tree itself,
601 * this tree object was not found in the pack file. This is the
602 * last time we are being called for this optimized traversal.
603 * Return from here and switch to loading objects the slow way.
604 */
605 if (tree == NULL) {
606 free(a->id);
607 a->id = got_object_id_dup(id);
608 if (a->id == NULL) {
609 err = got_error_from_errno("got_object_id_dup");
610 free(a->dpath);
611 a->dpath = NULL;
612 return err;
615 free(a->dpath);
616 a->dpath = strdup(dpath);
617 if (a->dpath == NULL) {
618 err = got_error_from_errno("strdup");
619 free(a->id);
620 a->id = NULL;
621 return err;
624 a->mtime = mtime;
625 return NULL;
628 if (got_object_idset_contains(a->idset, id) ||
629 got_object_idset_contains(a->idset_exclude, id))
630 return NULL;
632 relpath = dpath;
633 while (relpath[0] == '/')
634 relpath++;
636 err = got_pack_add_object(a->want_meta,
637 a->want_meta ? a->idset : a->idset_exclude,
638 id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
639 a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
640 a->progress_cb, a->progress_arg, a->rl);
641 if (err)
642 return err;
644 return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
645 a->idset_exclude, tree, dpath, mtime, a->seed, repo,
646 a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
647 a->progress_cb, a->progress_arg, a->rl,
648 a->cancel_cb, a->cancel_arg);
651 const struct got_error *
652 got_pack_load_packed_object_ids(int *found_all_objects,
653 struct got_object_id **ours, int nours,
654 struct got_object_id **theirs, int ntheirs,
655 int want_meta, uint32_t seed, struct got_object_idset *idset,
656 struct got_object_idset *idset_exclude, int loose_obj_only,
657 struct got_repository *repo, struct got_packidx *packidx,
658 int *ncolored, int *nfound, int *ntrees,
659 got_pack_progress_cb progress_cb, void *progress_arg,
660 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
662 const struct got_error *err = NULL;
663 struct load_packed_obj_arg lpa;
665 memset(&lpa, 0, sizeof(lpa));
666 lpa.seed = seed;
667 lpa.want_meta = want_meta;
668 lpa.idset = idset;
669 lpa.idset_exclude = idset_exclude;
670 lpa.loose_obj_only = loose_obj_only;
671 lpa.ncolored = ncolored;
672 lpa.nfound = nfound;
673 lpa.ntrees = ntrees;
674 lpa.progress_cb = progress_cb;
675 lpa.progress_arg = progress_arg;
676 lpa.rl = rl;
677 lpa.cancel_cb = cancel_cb;
678 lpa.cancel_arg = cancel_arg;
680 /* Attempt to load objects via got-read-pack, as far as possible. */
681 err = got_object_enumerate(found_all_objects, load_packed_commit_id,
682 load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
683 packidx, repo);
684 if (err)
685 return err;
687 if (lpa.id == NULL)
688 return NULL;
690 /*
691 * An incomplete tree hierarchy was present in the pack file
692 * and caused loading to be aborted.
693 * Continue loading trees the slow way.
694 */
695 err = got_pack_load_tree(want_meta, idset, idset_exclude,
696 lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
697 ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
698 cancel_cb, cancel_arg);
699 free(lpa.id);
700 free(lpa.dpath);
701 return err;