2 * Copyright (c) 2020 Ori Bernstein
3 * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 #include <sys/types.h>
19 #include <sys/queue.h>
32 #include "got_compat.h"
33 #include "got_error.h"
34 #include "got_cancel.h"
35 #include "got_object.h"
36 #include "got_reference.h"
37 #include "got_repository_admin.h"
40 #include "got_lib_delta.h"
41 #include "got_lib_object.h"
42 #include "got_lib_object_cache.h"
43 #include "got_lib_object_idset.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_ratelimit.h"
46 #include "got_lib_pack.h"
47 #include "got_lib_pack_create.h"
48 #include "got_lib_repository.h"
52 struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
56 static const struct got_error *
57 send_id(struct got_object_id *id, void *data, void *arg)
59 const struct got_error *err = NULL;
60 struct send_id_arg *a = arg;
62 a->ids[a->nids++] = id;
64 if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
65 err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
74 static const struct got_error *
75 send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
77 const struct got_error *err;
78 struct send_id_arg sia;
80 memset(&sia, 0, sizeof(sia));
82 err = got_object_idset_for_each(idset, send_id, &sia);
87 err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
92 return got_privsep_send_object_idlist_done(ibuf);
95 static const struct got_error *
96 recv_reused_delta(struct got_imsg_reused_delta *delta,
97 struct got_object_idset *idset, struct got_pack_metavec *v)
99 struct got_pack_meta *m, *base;
101 if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
102 delta->delta_offset +
103 delta->delta_compressed_size < delta->delta_offset)
104 return got_error(GOT_ERR_BAD_PACKFILE);
106 m = got_object_idset_get(idset, &delta->id);
108 return got_error(GOT_ERR_NO_OBJ);
110 base = got_object_idset_get(idset, &delta->base_id);
112 return got_error(GOT_ERR_NO_OBJ);
114 m->delta_len = delta->delta_size;
115 m->delta_compressed_len = delta->delta_compressed_size;
118 m->size = delta->result_size;
119 m->reused_delta_offset = delta->delta_offset;
120 m->base_obj_id = got_object_id_dup(&delta->base_id);
121 if (m->base_obj_id == NULL)
122 return got_error_from_errno("got_object_id_dup");
124 return got_pack_add_meta(m, v);
127 const struct got_error *
128 got_pack_search_deltas(struct got_packidx **packidx, struct got_pack **pack,
129 struct got_pack_metavec *v, struct got_object_idset *idset,
130 int ncolored, int nfound, int ntrees, int ncommits,
131 struct got_repository *repo,
132 got_pack_progress_cb progress_cb, void *progress_arg,
133 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
135 const struct got_error *err = NULL;
136 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
142 err = got_pack_find_pack_for_reuse(packidx, repo);
146 if (*packidx == NULL)
149 err = got_pack_cache_pack_for_packidx(pack, *packidx, repo);
153 if ((*pack)->privsep_child == NULL) {
154 err = got_pack_start_privsep_child(*pack, *packidx);
159 err = got_privsep_send_delta_reuse_req((*pack)->privsep_child->ibuf);
163 err = send_idset((*pack)->privsep_child->ibuf, idset);
171 err = (*cancel_cb)(cancel_arg);
176 err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
177 (*pack)->privsep_child->ibuf);
181 for (i = 0; i < ndeltas; i++) {
182 struct got_imsg_reused_delta *delta = &deltas[i];
183 err = recv_reused_delta(delta, idset, v);
188 err = got_pack_report_progress(progress_cb, progress_arg, rl,
189 ncolored, nfound, ntrees, 0L, ncommits,
190 got_object_idset_num_elements(idset), v->nmeta, 0);
198 struct recv_painted_commit_arg {
202 struct got_object_id_queue *ids;
203 struct got_object_idset *keep;
204 struct got_object_idset *drop;
205 struct got_object_idset *skip;
206 got_pack_progress_cb progress_cb;
208 struct got_ratelimit *rl;
209 got_cancel_cb cancel_cb;
213 static const struct got_error *
214 recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
216 const struct got_error *err = NULL;
217 struct recv_painted_commit_arg *a = arg;
218 struct got_object_qid *qid, *tmp;
221 err = a->cancel_cb(a->cancel_arg);
228 err = got_object_idset_add(a->keep, id, NULL);
234 err = got_object_idset_add(a->drop, id, NULL);
240 err = got_object_idset_add(a->skip, id, NULL);
245 /* should not happen */
246 return got_error_fmt(GOT_ERR_NOT_IMPL,
247 "%s invalid commit color %"PRIdPTR, __func__, color);
250 STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
251 if (got_object_id_cmp(&qid->id, id) != 0)
253 STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
254 color = (intptr_t)qid->data;
255 got_object_qid_free(qid);
257 if (color == COLOR_SKIP)
262 return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
263 *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
266 static const struct got_error *
267 paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
268 int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
269 struct got_object_id_queue *ids,
270 struct got_object_idset *keep, struct got_object_idset *drop,
271 struct got_object_idset *skip, struct got_repository *repo,
272 got_pack_progress_cb progress_cb, void *progress_arg,
273 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
275 const struct got_error *err = NULL;
276 struct got_object_id_queue next_ids;
277 struct got_object_qid *qid, *tmp;
278 struct recv_painted_commit_arg arg;
280 STAILQ_INIT(&next_ids);
282 err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
287 arg.ncolored = ncolored;
288 arg.nqueued = nqueued;
294 arg.progress_cb = progress_cb;
295 arg.progress_arg = progress_arg;
297 arg.cancel_cb = cancel_cb;
298 arg.cancel_arg = cancel_arg;
299 err = got_privsep_recv_painted_commits(&next_ids,
300 recv_painted_commit, &arg, pack->privsep_child->ibuf);
304 STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
305 struct got_object_qid *old_id;
306 intptr_t qcolor, ocolor;
307 STAILQ_FOREACH(old_id, ids, entry) {
308 if (got_object_id_cmp(&qid->id, &old_id->id))
310 qcolor = (intptr_t)qid->data;
311 ocolor = (intptr_t)old_id->data;
312 STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
313 got_object_qid_free(qid);
315 if (qcolor != ocolor) {
316 got_pack_paint_commit(old_id, qcolor);
317 if (ocolor == COLOR_SKIP)
319 else if (qcolor == COLOR_SKIP)
325 while (!STAILQ_EMPTY(&next_ids)) {
326 qid = STAILQ_FIRST(&next_ids);
327 STAILQ_REMOVE_HEAD(&next_ids, entry);
328 got_pack_paint_commit(qid, color);
329 STAILQ_INSERT_TAIL(ids, qid, entry);
331 if (color == COLOR_SKIP)
338 const struct got_error *
339 got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
340 struct got_object_idset *keep, struct got_object_idset *drop,
341 struct got_object_idset *skip, struct got_repository *repo,
342 got_pack_progress_cb progress_cb, void *progress_arg,
343 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
345 const struct got_error *err = NULL;
346 struct got_commit_object *commit = NULL;
347 struct got_packidx *packidx = NULL;
348 struct got_pack *pack = NULL;
349 const struct got_object_id_queue *parents;
350 struct got_object_qid *qid = NULL;
351 int nqueued = nids, nskip = 0;
354 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
358 err = cancel_cb(cancel_arg);
363 qid = STAILQ_FIRST(ids);
364 STAILQ_REMOVE_HEAD(ids, entry);
366 color = (intptr_t)qid->data;
367 if (color == COLOR_SKIP)
370 if (got_object_idset_contains(skip, &qid->id)) {
371 got_object_qid_free(qid);
375 if (color == COLOR_KEEP &&
376 got_object_idset_contains(keep, &qid->id)) {
377 got_object_qid_free(qid);
381 if (color == COLOR_DROP &&
382 got_object_idset_contains(drop, &qid->id)) {
383 got_object_qid_free(qid);
388 /* Pinned pack may have moved to different cache slot. */
389 pack = got_repo_get_pinned_pack(repo);
391 if (packidx && pack) {
392 idx = got_packidx_get_object_idx(packidx, &qid->id);
394 err = paint_packed_commits(pack, &qid->id,
395 idx, color, ncolored, &nqueued, &nskip,
396 ids, keep, drop, skip, repo,
397 progress_cb, progress_arg, rl,
398 cancel_cb, cancel_arg);
401 got_object_qid_free(qid);
409 if (got_object_idset_contains(drop, &qid->id)) {
410 err = got_pack_paint_commit(qid, COLOR_SKIP);
415 err = got_object_idset_add(keep, &qid->id, NULL);
420 if (got_object_idset_contains(keep, &qid->id)) {
421 err = got_pack_paint_commit(qid, COLOR_SKIP);
426 err = got_object_idset_add(drop, &qid->id, NULL);
431 if (!got_object_idset_contains(skip, &qid->id)) {
432 err = got_object_idset_add(skip, &qid->id,
439 /* should not happen */
440 err = got_error_fmt(GOT_ERR_NOT_IMPL,
441 "%s invalid commit color %"PRIdPTR, __func__,
446 err = got_pack_report_progress(progress_cb, progress_arg, rl,
447 *ncolored, 0, 0, 0L, 0, 0, 0, 0);
451 err = got_object_open_as_commit(&commit, repo, &qid->id);
455 parents = got_object_commit_get_parent_ids(commit);
457 struct got_object_qid *pid;
458 color = (intptr_t)qid->data;
459 STAILQ_FOREACH(pid, parents, entry) {
460 err = got_pack_queue_commit_id(ids, &pid->id,
465 if (color == COLOR_SKIP)
470 if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
471 if (packidx == NULL) {
472 err = got_pack_find_pack_for_commit_painting(
473 &packidx, ids, nqueued, repo);
477 if (packidx != NULL) {
478 err = got_pack_cache_pack_for_packidx(&pack,
482 if (pack->privsep_child == NULL) {
483 err = got_pack_start_privsep_child(
488 err = got_privsep_init_commit_painting(
489 pack->privsep_child->ibuf);
492 err = send_idset(pack->privsep_child->ibuf,
496 err = send_idset(pack->privsep_child->ibuf, drop);
499 err = send_idset(pack->privsep_child->ibuf, skip);
502 err = got_repo_pin_pack(repo, packidx, pack);
508 got_object_commit_close(commit);
511 got_object_qid_free(qid);
516 const struct got_error *pack_err;
517 pack_err = got_privsep_send_painting_commits_done(
518 pack->privsep_child->ibuf);
523 got_object_commit_close(commit);
524 got_object_qid_free(qid);
525 got_repo_unpin_pack(repo);
529 struct load_packed_obj_arg {
530 /* output parameters: */
531 struct got_object_id *id;
535 /* input parameters: */
538 struct got_object_idset *idset;
539 struct got_object_idset *idset_exclude;
544 got_pack_progress_cb progress_cb;
546 struct got_ratelimit *rl;
547 got_cancel_cb cancel_cb;
551 static const struct got_error *
552 load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
553 struct got_repository *repo)
555 struct load_packed_obj_arg *a = arg;
557 if (got_object_idset_contains(a->idset, id) ||
558 got_object_idset_contains(a->idset_exclude, id))
561 return got_pack_add_object(a->want_meta,
562 a->want_meta ? a->idset : a->idset_exclude,
563 id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
564 repo, a->ncolored, a->nfound, a->ntrees,
565 a->progress_cb, a->progress_arg, a->rl);
568 static const struct got_error *
569 load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
570 struct got_object_id *id, const char *dpath, struct got_repository *repo)
572 const struct got_error *err;
573 struct load_packed_obj_arg *a = arg;
577 * When we receive a tree's ID and path but not the tree itself,
578 * this tree object was not found in the pack file. This is the
579 * last time we are being called for this optimized traversal.
580 * Return from here and switch to loading objects the slow way.
584 a->id = got_object_id_dup(id);
586 err = got_error_from_errno("got_object_id_dup");
593 a->dpath = strdup(dpath);
594 if (a->dpath == NULL) {
595 err = got_error_from_errno("strdup");
605 if (got_object_idset_contains(a->idset, id) ||
606 got_object_idset_contains(a->idset_exclude, id))
610 while (relpath[0] == '/')
613 err = got_pack_add_object(a->want_meta,
614 a->want_meta ? a->idset : a->idset_exclude,
615 id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
616 a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
617 a->progress_cb, a->progress_arg, a->rl);
621 return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
622 a->idset_exclude, tree, dpath, mtime, a->seed, repo,
623 a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
624 a->progress_cb, a->progress_arg, a->rl,
625 a->cancel_cb, a->cancel_arg);
628 const struct got_error *
629 got_pack_load_packed_object_ids(int *found_all_objects,
630 struct got_object_id **ours, int nours,
631 struct got_object_id **theirs, int ntheirs,
632 int want_meta, uint32_t seed, struct got_object_idset *idset,
633 struct got_object_idset *idset_exclude, int loose_obj_only,
634 struct got_repository *repo, struct got_packidx *packidx,
635 int *ncolored, int *nfound, int *ntrees,
636 got_pack_progress_cb progress_cb, void *progress_arg,
637 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
639 const struct got_error *err = NULL;
640 struct load_packed_obj_arg lpa;
642 memset(&lpa, 0, sizeof(lpa));
644 lpa.want_meta = want_meta;
646 lpa.idset_exclude = idset_exclude;
647 lpa.loose_obj_only = loose_obj_only;
648 lpa.ncolored = ncolored;
651 lpa.progress_cb = progress_cb;
652 lpa.progress_arg = progress_arg;
654 lpa.cancel_cb = cancel_cb;
655 lpa.cancel_arg = cancel_arg;
657 /* Attempt to load objects via got-read-pack, as far as possible. */
658 err = got_object_enumerate(found_all_objects, load_packed_commit_id,
659 load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
668 * An incomplete tree hierarchy was present in the pack file
669 * and caused loading to be aborted.
670 * Continue loading trees the slow way.
672 err = got_pack_load_tree(want_meta, idset, idset_exclude,
673 lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
674 ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
675 cancel_cb, cancel_arg);