2 301e83b3 2022-10-16 stsp * Copyright (c) 2020 Ori Bernstein
3 301e83b3 2022-10-16 stsp * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
5 301e83b3 2022-10-16 stsp * Permission to use, copy, modify, and distribute this software for any
6 301e83b3 2022-10-16 stsp * purpose with or without fee is hereby granted, provided that the above
7 301e83b3 2022-10-16 stsp * copyright notice and this permission notice appear in all copies.
9 301e83b3 2022-10-16 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 301e83b3 2022-10-16 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 301e83b3 2022-10-16 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 301e83b3 2022-10-16 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 301e83b3 2022-10-16 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 301e83b3 2022-10-16 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 301e83b3 2022-10-16 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 301e83b3 2022-10-16 stsp #include <sys/types.h>
19 301e83b3 2022-10-16 stsp #include <sys/queue.h>
20 301e83b3 2022-10-16 stsp #include <sys/tree.h>
21 301e83b3 2022-10-16 stsp #include <sys/uio.h>
23 301e83b3 2022-10-16 stsp #include <sha1.h>
24 301e83b3 2022-10-16 stsp #include <limits.h>
25 301e83b3 2022-10-16 stsp #include <stdio.h>
26 301e83b3 2022-10-16 stsp #include <stdint.h>
27 301e83b3 2022-10-16 stsp #include <stdlib.h>
28 301e83b3 2022-10-16 stsp #include <string.h>
29 301e83b3 2022-10-16 stsp #include <imsg.h>
30 301e83b3 2022-10-16 stsp #include <inttypes.h>
31 301e83b3 2022-10-16 stsp #include <unistd.h>
33 301e83b3 2022-10-16 stsp #include "got_error.h"
34 301e83b3 2022-10-16 stsp #include "got_cancel.h"
35 301e83b3 2022-10-16 stsp #include "got_object.h"
36 301e83b3 2022-10-16 stsp #include "got_reference.h"
37 301e83b3 2022-10-16 stsp #include "got_repository_admin.h"
38 301e83b3 2022-10-16 stsp #include "got_path.h"
40 301e83b3 2022-10-16 stsp #include "got_lib_delta.h"
41 301e83b3 2022-10-16 stsp #include "got_lib_object.h"
42 301e83b3 2022-10-16 stsp #include "got_lib_object_cache.h"
43 301e83b3 2022-10-16 stsp #include "got_lib_object_idset.h"
44 301e83b3 2022-10-16 stsp #include "got_lib_privsep.h"
45 301e83b3 2022-10-16 stsp #include "got_lib_pack.h"
46 301e83b3 2022-10-16 stsp #include "got_lib_pack_create.h"
47 301e83b3 2022-10-16 stsp #include "got_lib_repository.h"
49 301e83b3 2022-10-16 stsp struct send_id_arg {
50 301e83b3 2022-10-16 stsp struct imsgbuf *ibuf;
51 301e83b3 2022-10-16 stsp struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
52 301e83b3 2022-10-16 stsp size_t nids;
55 301e83b3 2022-10-16 stsp static const struct got_error *
56 301e83b3 2022-10-16 stsp send_id(struct got_object_id *id, void *data, void *arg)
58 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
59 301e83b3 2022-10-16 stsp struct send_id_arg *a = arg;
61 301e83b3 2022-10-16 stsp a->ids[a->nids++] = id;
63 301e83b3 2022-10-16 stsp if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
64 301e83b3 2022-10-16 stsp err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
67 301e83b3 2022-10-16 stsp a->nids = 0;
70 301e83b3 2022-10-16 stsp return NULL;
73 301e83b3 2022-10-16 stsp static const struct got_error *
74 301e83b3 2022-10-16 stsp send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
76 301e83b3 2022-10-16 stsp const struct got_error *err;
77 301e83b3 2022-10-16 stsp struct send_id_arg sia;
79 301e83b3 2022-10-16 stsp memset(&sia, 0, sizeof(sia));
80 301e83b3 2022-10-16 stsp sia.ibuf = ibuf;
81 301e83b3 2022-10-16 stsp err = got_object_idset_for_each(idset, send_id, &sia);
85 301e83b3 2022-10-16 stsp if (sia.nids > 0) {
86 301e83b3 2022-10-16 stsp err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
91 301e83b3 2022-10-16 stsp return got_privsep_send_object_idlist_done(ibuf);
94 301e83b3 2022-10-16 stsp static const struct got_error *
95 301e83b3 2022-10-16 stsp recv_reused_delta(struct got_imsg_reused_delta *delta,
96 301e83b3 2022-10-16 stsp struct got_object_idset *idset, struct got_pack_metavec *v)
98 301e83b3 2022-10-16 stsp struct got_pack_meta *m, *base;
100 301e83b3 2022-10-16 stsp if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
101 301e83b3 2022-10-16 stsp delta->delta_offset +
102 301e83b3 2022-10-16 stsp delta->delta_compressed_size < delta->delta_offset)
103 301e83b3 2022-10-16 stsp return got_error(GOT_ERR_BAD_PACKFILE);
105 301e83b3 2022-10-16 stsp m = got_object_idset_get(idset, &delta->id);
106 301e83b3 2022-10-16 stsp if (m == NULL)
107 301e83b3 2022-10-16 stsp return got_error(GOT_ERR_NO_OBJ);
109 301e83b3 2022-10-16 stsp base = got_object_idset_get(idset, &delta->base_id);
110 301e83b3 2022-10-16 stsp if (base == NULL)
111 301e83b3 2022-10-16 stsp return got_error(GOT_ERR_NO_OBJ);
113 301e83b3 2022-10-16 stsp m->delta_len = delta->delta_size;
114 301e83b3 2022-10-16 stsp m->delta_compressed_len = delta->delta_compressed_size;
115 301e83b3 2022-10-16 stsp m->delta_offset = delta->delta_out_offset;
116 301e83b3 2022-10-16 stsp m->prev = base;
117 301e83b3 2022-10-16 stsp m->size = delta->result_size;
118 301e83b3 2022-10-16 stsp m->reused_delta_offset = delta->delta_offset;
119 301e83b3 2022-10-16 stsp m->base_obj_id = got_object_id_dup(&delta->base_id);
120 301e83b3 2022-10-16 stsp if (m->base_obj_id == NULL)
121 301e83b3 2022-10-16 stsp return got_error_from_errno("got_object_id_dup");
123 301e83b3 2022-10-16 stsp return got_pack_add_meta(m, v);
126 301e83b3 2022-10-16 stsp static const struct got_error *
127 301e83b3 2022-10-16 stsp prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx,
128 301e83b3 2022-10-16 stsp int delta_outfd, struct got_repository *repo)
130 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
132 301e83b3 2022-10-16 stsp if (!pack->child_has_delta_outfd) {
133 301e83b3 2022-10-16 stsp int outfd_child;
134 301e83b3 2022-10-16 stsp outfd_child = dup(delta_outfd);
135 301e83b3 2022-10-16 stsp if (outfd_child == -1) {
136 301e83b3 2022-10-16 stsp err = got_error_from_errno("dup");
139 301e83b3 2022-10-16 stsp err = got_privsep_send_raw_delta_outfd(
140 301e83b3 2022-10-16 stsp pack->privsep_child->ibuf, outfd_child);
143 301e83b3 2022-10-16 stsp pack->child_has_delta_outfd = 1;
146 301e83b3 2022-10-16 stsp err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf);
148 301e83b3 2022-10-16 stsp return err;
151 301e83b3 2022-10-16 stsp const struct got_error *
152 301e83b3 2022-10-16 stsp got_pack_search_deltas(struct got_pack_metavec *v,
153 301e83b3 2022-10-16 stsp struct got_object_idset *idset, int delta_cache_fd,
154 301e83b3 2022-10-16 stsp int ncolored, int nfound, int ntrees, int ncommits,
155 301e83b3 2022-10-16 stsp struct got_repository *repo,
156 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb, void *progress_arg,
157 301e83b3 2022-10-16 stsp struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
159 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
160 301e83b3 2022-10-16 stsp struct got_packidx *packidx;
161 301e83b3 2022-10-16 stsp struct got_pack *pack;
162 301e83b3 2022-10-16 stsp struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
163 301e83b3 2022-10-16 stsp size_t ndeltas, i;
165 301e83b3 2022-10-16 stsp err = got_pack_find_pack_for_reuse(&packidx, repo);
167 301e83b3 2022-10-16 stsp return err;
169 301e83b3 2022-10-16 stsp if (packidx == NULL)
170 301e83b3 2022-10-16 stsp return NULL;
172 301e83b3 2022-10-16 stsp err = got_pack_cache_pack_for_packidx(&pack, packidx, repo);
174 301e83b3 2022-10-16 stsp return err;
176 301e83b3 2022-10-16 stsp if (pack->privsep_child == NULL) {
177 301e83b3 2022-10-16 stsp err = got_pack_start_privsep_child(pack, packidx);
179 301e83b3 2022-10-16 stsp return err;
182 301e83b3 2022-10-16 stsp err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo);
184 301e83b3 2022-10-16 stsp return err;
186 301e83b3 2022-10-16 stsp err = send_idset(pack->privsep_child->ibuf, idset);
188 301e83b3 2022-10-16 stsp return err;
191 301e83b3 2022-10-16 stsp int done = 0;
193 301e83b3 2022-10-16 stsp if (cancel_cb) {
194 301e83b3 2022-10-16 stsp err = (*cancel_cb)(cancel_arg);
199 301e83b3 2022-10-16 stsp err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
200 301e83b3 2022-10-16 stsp pack->privsep_child->ibuf);
201 301e83b3 2022-10-16 stsp if (err || done)
204 301e83b3 2022-10-16 stsp for (i = 0; i < ndeltas; i++) {
205 301e83b3 2022-10-16 stsp struct got_imsg_reused_delta *delta = &deltas[i];
206 301e83b3 2022-10-16 stsp err = recv_reused_delta(delta, idset, v);
211 301e83b3 2022-10-16 stsp err = got_pack_report_progress(progress_cb, progress_arg, rl,
212 301e83b3 2022-10-16 stsp ncolored, nfound, ntrees, 0L, ncommits,
213 301e83b3 2022-10-16 stsp got_object_idset_num_elements(idset), v->nmeta, 0);
218 301e83b3 2022-10-16 stsp return err;
221 301e83b3 2022-10-16 stsp struct recv_painted_commit_arg {
222 301e83b3 2022-10-16 stsp int *ncolored;
223 301e83b3 2022-10-16 stsp int *nqueued;
224 301e83b3 2022-10-16 stsp int *nskip;
225 301e83b3 2022-10-16 stsp struct got_object_id_queue *ids;
226 301e83b3 2022-10-16 stsp struct got_object_idset *keep;
227 301e83b3 2022-10-16 stsp struct got_object_idset *drop;
228 301e83b3 2022-10-16 stsp struct got_object_idset *skip;
229 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb;
230 301e83b3 2022-10-16 stsp void *progress_arg;
231 301e83b3 2022-10-16 stsp struct got_ratelimit *rl;
232 301e83b3 2022-10-16 stsp got_cancel_cb cancel_cb;
233 301e83b3 2022-10-16 stsp void *cancel_arg;
236 301e83b3 2022-10-16 stsp static const struct got_error *
237 301e83b3 2022-10-16 stsp recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
239 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
240 301e83b3 2022-10-16 stsp struct recv_painted_commit_arg *a = arg;
241 301e83b3 2022-10-16 stsp struct got_object_qid *qid, *tmp;
243 301e83b3 2022-10-16 stsp if (a->cancel_cb) {
244 301e83b3 2022-10-16 stsp err = a->cancel_cb(a->cancel_arg);
246 301e83b3 2022-10-16 stsp return err;
249 301e83b3 2022-10-16 stsp switch (color) {
250 301e83b3 2022-10-16 stsp case COLOR_KEEP:
251 301e83b3 2022-10-16 stsp err = got_object_idset_add(a->keep, id, NULL);
253 301e83b3 2022-10-16 stsp return err;
254 301e83b3 2022-10-16 stsp (*a->ncolored)++;
256 301e83b3 2022-10-16 stsp case COLOR_DROP:
257 301e83b3 2022-10-16 stsp err = got_object_idset_add(a->drop, id, NULL);
259 301e83b3 2022-10-16 stsp return err;
260 301e83b3 2022-10-16 stsp (*a->ncolored)++;
262 301e83b3 2022-10-16 stsp case COLOR_SKIP:
263 301e83b3 2022-10-16 stsp err = got_object_idset_add(a->skip, id, NULL);
265 301e83b3 2022-10-16 stsp return err;
268 301e83b3 2022-10-16 stsp /* should not happen */
269 301e83b3 2022-10-16 stsp return got_error_fmt(GOT_ERR_NOT_IMPL,
270 301e83b3 2022-10-16 stsp "%s invalid commit color %"PRIdPTR, __func__, color);
273 301e83b3 2022-10-16 stsp STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
274 301e83b3 2022-10-16 stsp if (got_object_id_cmp(&qid->id, id) != 0)
276 301e83b3 2022-10-16 stsp STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
277 301e83b3 2022-10-16 stsp color = (intptr_t)qid->data;
278 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
279 301e83b3 2022-10-16 stsp (*a->nqueued)--;
280 301e83b3 2022-10-16 stsp if (color == COLOR_SKIP)
281 301e83b3 2022-10-16 stsp (*a->nskip)--;
285 301e83b3 2022-10-16 stsp return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
286 301e83b3 2022-10-16 stsp *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
289 301e83b3 2022-10-16 stsp static const struct got_error *
290 301e83b3 2022-10-16 stsp paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
291 301e83b3 2022-10-16 stsp int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
292 301e83b3 2022-10-16 stsp struct got_object_id_queue *ids,
293 301e83b3 2022-10-16 stsp struct got_object_idset *keep, struct got_object_idset *drop,
294 301e83b3 2022-10-16 stsp struct got_object_idset *skip, struct got_repository *repo,
295 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb, void *progress_arg,
296 301e83b3 2022-10-16 stsp struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
298 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
299 301e83b3 2022-10-16 stsp struct got_object_id_queue next_ids;
300 301e83b3 2022-10-16 stsp struct got_object_qid *qid, *tmp;
301 301e83b3 2022-10-16 stsp struct recv_painted_commit_arg arg;
303 301e83b3 2022-10-16 stsp STAILQ_INIT(&next_ids);
305 301e83b3 2022-10-16 stsp err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
306 301e83b3 2022-10-16 stsp idx, id, color);
308 301e83b3 2022-10-16 stsp return err;
310 301e83b3 2022-10-16 stsp arg.ncolored = ncolored;
311 301e83b3 2022-10-16 stsp arg.nqueued = nqueued;
312 301e83b3 2022-10-16 stsp arg.nskip = nskip;
313 301e83b3 2022-10-16 stsp arg.ids = ids;
314 301e83b3 2022-10-16 stsp arg.keep = keep;
315 301e83b3 2022-10-16 stsp arg.drop = drop;
316 301e83b3 2022-10-16 stsp arg.skip = skip;
317 301e83b3 2022-10-16 stsp arg.progress_cb = progress_cb;
318 301e83b3 2022-10-16 stsp arg.progress_arg = progress_arg;
319 301e83b3 2022-10-16 stsp arg.rl = rl;
320 301e83b3 2022-10-16 stsp arg.cancel_cb = cancel_cb;
321 301e83b3 2022-10-16 stsp arg.cancel_arg = cancel_arg;
322 301e83b3 2022-10-16 stsp err = got_privsep_recv_painted_commits(&next_ids,
323 301e83b3 2022-10-16 stsp recv_painted_commit, &arg, pack->privsep_child->ibuf);
325 301e83b3 2022-10-16 stsp return err;
327 301e83b3 2022-10-16 stsp STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
328 301e83b3 2022-10-16 stsp struct got_object_qid *old_id;
329 301e83b3 2022-10-16 stsp intptr_t qcolor, ocolor;
330 301e83b3 2022-10-16 stsp STAILQ_FOREACH(old_id, ids, entry) {
331 301e83b3 2022-10-16 stsp if (got_object_id_cmp(&qid->id, &old_id->id))
333 301e83b3 2022-10-16 stsp qcolor = (intptr_t)qid->data;
334 301e83b3 2022-10-16 stsp ocolor = (intptr_t)old_id->data;
335 301e83b3 2022-10-16 stsp STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
336 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
337 301e83b3 2022-10-16 stsp qid = NULL;
338 301e83b3 2022-10-16 stsp if (qcolor != ocolor) {
339 301e83b3 2022-10-16 stsp got_pack_paint_commit(old_id, qcolor);
340 301e83b3 2022-10-16 stsp if (ocolor == COLOR_SKIP)
341 301e83b3 2022-10-16 stsp (*nskip)--;
342 301e83b3 2022-10-16 stsp else if (qcolor == COLOR_SKIP)
343 301e83b3 2022-10-16 stsp (*nskip)++;
348 301e83b3 2022-10-16 stsp while (!STAILQ_EMPTY(&next_ids)) {
349 301e83b3 2022-10-16 stsp qid = STAILQ_FIRST(&next_ids);
350 301e83b3 2022-10-16 stsp STAILQ_REMOVE_HEAD(&next_ids, entry);
351 301e83b3 2022-10-16 stsp got_pack_paint_commit(qid, color);
352 301e83b3 2022-10-16 stsp STAILQ_INSERT_TAIL(ids, qid, entry);
353 301e83b3 2022-10-16 stsp (*nqueued)++;
354 301e83b3 2022-10-16 stsp if (color == COLOR_SKIP)
355 301e83b3 2022-10-16 stsp (*nskip)++;
358 301e83b3 2022-10-16 stsp return err;
361 301e83b3 2022-10-16 stsp const struct got_error *
362 301e83b3 2022-10-16 stsp got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
363 301e83b3 2022-10-16 stsp struct got_object_idset *keep, struct got_object_idset *drop,
364 301e83b3 2022-10-16 stsp struct got_object_idset *skip, struct got_repository *repo,
365 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb, void *progress_arg,
366 301e83b3 2022-10-16 stsp struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
368 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
369 301e83b3 2022-10-16 stsp struct got_commit_object *commit = NULL;
370 301e83b3 2022-10-16 stsp struct got_packidx *packidx = NULL;
371 301e83b3 2022-10-16 stsp struct got_pack *pack = NULL;
372 301e83b3 2022-10-16 stsp const struct got_object_id_queue *parents;
373 301e83b3 2022-10-16 stsp struct got_object_qid *qid = NULL;
374 301e83b3 2022-10-16 stsp int nqueued = nids, nskip = 0;
377 301e83b3 2022-10-16 stsp while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
378 301e83b3 2022-10-16 stsp intptr_t color;
380 301e83b3 2022-10-16 stsp if (cancel_cb) {
381 301e83b3 2022-10-16 stsp err = cancel_cb(cancel_arg);
386 301e83b3 2022-10-16 stsp qid = STAILQ_FIRST(ids);
387 301e83b3 2022-10-16 stsp STAILQ_REMOVE_HEAD(ids, entry);
389 301e83b3 2022-10-16 stsp color = (intptr_t)qid->data;
390 301e83b3 2022-10-16 stsp if (color == COLOR_SKIP)
393 301e83b3 2022-10-16 stsp if (got_object_idset_contains(skip, &qid->id)) {
394 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
395 301e83b3 2022-10-16 stsp qid = NULL;
398 301e83b3 2022-10-16 stsp if (color == COLOR_KEEP &&
399 301e83b3 2022-10-16 stsp got_object_idset_contains(keep, &qid->id)) {
400 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
401 301e83b3 2022-10-16 stsp qid = NULL;
404 301e83b3 2022-10-16 stsp if (color == COLOR_DROP &&
405 301e83b3 2022-10-16 stsp got_object_idset_contains(drop, &qid->id)) {
406 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
407 301e83b3 2022-10-16 stsp qid = NULL;
411 301e83b3 2022-10-16 stsp /* Pinned pack may have moved to different cache slot. */
412 301e83b3 2022-10-16 stsp pack = got_repo_get_pinned_pack(repo);
414 301e83b3 2022-10-16 stsp if (packidx && pack) {
415 301e83b3 2022-10-16 stsp idx = got_packidx_get_object_idx(packidx, &qid->id);
416 301e83b3 2022-10-16 stsp if (idx != -1) {
417 301e83b3 2022-10-16 stsp err = paint_packed_commits(pack, &qid->id,
418 301e83b3 2022-10-16 stsp idx, color, ncolored, &nqueued, &nskip,
419 301e83b3 2022-10-16 stsp ids, keep, drop, skip, repo,
420 301e83b3 2022-10-16 stsp progress_cb, progress_arg, rl,
421 301e83b3 2022-10-16 stsp cancel_cb, cancel_arg);
424 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
425 301e83b3 2022-10-16 stsp qid = NULL;
430 301e83b3 2022-10-16 stsp switch (color) {
431 301e83b3 2022-10-16 stsp case COLOR_KEEP:
432 301e83b3 2022-10-16 stsp if (got_object_idset_contains(drop, &qid->id)) {
433 301e83b3 2022-10-16 stsp err = got_pack_paint_commit(qid, COLOR_SKIP);
437 301e83b3 2022-10-16 stsp (*ncolored)++;
438 301e83b3 2022-10-16 stsp err = got_object_idset_add(keep, &qid->id, NULL);
442 301e83b3 2022-10-16 stsp case COLOR_DROP:
443 301e83b3 2022-10-16 stsp if (got_object_idset_contains(keep, &qid->id)) {
444 301e83b3 2022-10-16 stsp err = got_pack_paint_commit(qid, COLOR_SKIP);
448 301e83b3 2022-10-16 stsp (*ncolored)++;
449 301e83b3 2022-10-16 stsp err = got_object_idset_add(drop, &qid->id, NULL);
453 301e83b3 2022-10-16 stsp case COLOR_SKIP:
454 301e83b3 2022-10-16 stsp if (!got_object_idset_contains(skip, &qid->id)) {
455 301e83b3 2022-10-16 stsp err = got_object_idset_add(skip, &qid->id,
462 301e83b3 2022-10-16 stsp /* should not happen */
463 301e83b3 2022-10-16 stsp err = got_error_fmt(GOT_ERR_NOT_IMPL,
464 301e83b3 2022-10-16 stsp "%s invalid commit color %"PRIdPTR, __func__,
469 301e83b3 2022-10-16 stsp err = got_pack_report_progress(progress_cb, progress_arg, rl,
470 301e83b3 2022-10-16 stsp *ncolored, 0, 0, 0L, 0, 0, 0, 0);
474 301e83b3 2022-10-16 stsp err = got_object_open_as_commit(&commit, repo, &qid->id);
478 301e83b3 2022-10-16 stsp parents = got_object_commit_get_parent_ids(commit);
479 301e83b3 2022-10-16 stsp if (parents) {
480 301e83b3 2022-10-16 stsp struct got_object_qid *pid;
481 301e83b3 2022-10-16 stsp color = (intptr_t)qid->data;
482 301e83b3 2022-10-16 stsp STAILQ_FOREACH(pid, parents, entry) {
483 301e83b3 2022-10-16 stsp err = got_pack_queue_commit_id(ids, &pid->id,
484 301e83b3 2022-10-16 stsp color, repo);
488 301e83b3 2022-10-16 stsp if (color == COLOR_SKIP)
493 301e83b3 2022-10-16 stsp if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
494 301e83b3 2022-10-16 stsp if (packidx == NULL) {
495 301e83b3 2022-10-16 stsp err = got_pack_find_pack_for_commit_painting(
496 301e83b3 2022-10-16 stsp &packidx, ids, nqueued, repo);
500 301e83b3 2022-10-16 stsp if (packidx != NULL) {
501 301e83b3 2022-10-16 stsp err = got_pack_cache_pack_for_packidx(&pack,
502 301e83b3 2022-10-16 stsp packidx, repo);
505 301e83b3 2022-10-16 stsp if (pack->privsep_child == NULL) {
506 301e83b3 2022-10-16 stsp err = got_pack_start_privsep_child(
507 301e83b3 2022-10-16 stsp pack, packidx);
511 301e83b3 2022-10-16 stsp err = got_privsep_init_commit_painting(
512 301e83b3 2022-10-16 stsp pack->privsep_child->ibuf);
515 301e83b3 2022-10-16 stsp err = send_idset(pack->privsep_child->ibuf,
519 301e83b3 2022-10-16 stsp err = send_idset(pack->privsep_child->ibuf, drop);
522 301e83b3 2022-10-16 stsp err = send_idset(pack->privsep_child->ibuf, skip);
525 301e83b3 2022-10-16 stsp err = got_repo_pin_pack(repo, packidx, pack);
531 301e83b3 2022-10-16 stsp got_object_commit_close(commit);
532 301e83b3 2022-10-16 stsp commit = NULL;
534 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
535 301e83b3 2022-10-16 stsp qid = NULL;
538 301e83b3 2022-10-16 stsp if (pack) {
539 301e83b3 2022-10-16 stsp const struct got_error *pack_err;
540 301e83b3 2022-10-16 stsp pack_err = got_privsep_send_painting_commits_done(
541 301e83b3 2022-10-16 stsp pack->privsep_child->ibuf);
542 301e83b3 2022-10-16 stsp if (err == NULL)
543 301e83b3 2022-10-16 stsp err = pack_err;
545 301e83b3 2022-10-16 stsp if (commit)
546 301e83b3 2022-10-16 stsp got_object_commit_close(commit);
547 301e83b3 2022-10-16 stsp got_object_qid_free(qid);
548 301e83b3 2022-10-16 stsp got_repo_unpin_pack(repo);
549 301e83b3 2022-10-16 stsp return err;
552 301e83b3 2022-10-16 stsp struct load_packed_obj_arg {
553 301e83b3 2022-10-16 stsp /* output parameters: */
554 301e83b3 2022-10-16 stsp struct got_object_id *id;
555 301e83b3 2022-10-16 stsp char *dpath;
556 301e83b3 2022-10-16 stsp time_t mtime;
558 301e83b3 2022-10-16 stsp /* input parameters: */
559 301e83b3 2022-10-16 stsp uint32_t seed;
560 301e83b3 2022-10-16 stsp int want_meta;
561 301e83b3 2022-10-16 stsp struct got_object_idset *idset;
562 301e83b3 2022-10-16 stsp struct got_object_idset *idset_exclude;
563 301e83b3 2022-10-16 stsp int loose_obj_only;
564 301e83b3 2022-10-16 stsp int *ncolored;
565 301e83b3 2022-10-16 stsp int *nfound;
566 301e83b3 2022-10-16 stsp int *ntrees;
567 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb;
568 301e83b3 2022-10-16 stsp void *progress_arg;
569 301e83b3 2022-10-16 stsp struct got_ratelimit *rl;
570 301e83b3 2022-10-16 stsp got_cancel_cb cancel_cb;
571 301e83b3 2022-10-16 stsp void *cancel_arg;
574 301e83b3 2022-10-16 stsp static const struct got_error *
575 301e83b3 2022-10-16 stsp load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
576 301e83b3 2022-10-16 stsp struct got_repository *repo)
578 301e83b3 2022-10-16 stsp struct load_packed_obj_arg *a = arg;
580 301e83b3 2022-10-16 stsp if (got_object_idset_contains(a->idset, id) ||
581 301e83b3 2022-10-16 stsp got_object_idset_contains(a->idset_exclude, id))
582 301e83b3 2022-10-16 stsp return NULL;
584 301e83b3 2022-10-16 stsp return got_pack_add_object(a->want_meta,
585 301e83b3 2022-10-16 stsp a->want_meta ? a->idset : a->idset_exclude,
586 301e83b3 2022-10-16 stsp id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
587 301e83b3 2022-10-16 stsp repo, a->ncolored, a->nfound, a->ntrees,
588 301e83b3 2022-10-16 stsp a->progress_cb, a->progress_arg, a->rl);
591 301e83b3 2022-10-16 stsp static const struct got_error *
592 301e83b3 2022-10-16 stsp load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
593 301e83b3 2022-10-16 stsp struct got_object_id *id, const char *dpath, struct got_repository *repo)
595 301e83b3 2022-10-16 stsp const struct got_error *err;
596 301e83b3 2022-10-16 stsp struct load_packed_obj_arg *a = arg;
597 301e83b3 2022-10-16 stsp const char *relpath;
600 301e83b3 2022-10-16 stsp * When we receive a tree's ID and path but not the tree itself,
601 301e83b3 2022-10-16 stsp * this tree object was not found in the pack file. This is the
602 301e83b3 2022-10-16 stsp * last time we are being called for this optimized traversal.
603 301e83b3 2022-10-16 stsp * Return from here and switch to loading objects the slow way.
605 301e83b3 2022-10-16 stsp if (tree == NULL) {
606 301e83b3 2022-10-16 stsp free(a->id);
607 301e83b3 2022-10-16 stsp a->id = got_object_id_dup(id);
608 301e83b3 2022-10-16 stsp if (a->id == NULL) {
609 301e83b3 2022-10-16 stsp err = got_error_from_errno("got_object_id_dup");
610 301e83b3 2022-10-16 stsp free(a->dpath);
611 301e83b3 2022-10-16 stsp a->dpath = NULL;
612 301e83b3 2022-10-16 stsp return err;
615 301e83b3 2022-10-16 stsp free(a->dpath);
616 301e83b3 2022-10-16 stsp a->dpath = strdup(dpath);
617 301e83b3 2022-10-16 stsp if (a->dpath == NULL) {
618 301e83b3 2022-10-16 stsp err = got_error_from_errno("strdup");
619 301e83b3 2022-10-16 stsp free(a->id);
620 301e83b3 2022-10-16 stsp a->id = NULL;
621 301e83b3 2022-10-16 stsp return err;
624 301e83b3 2022-10-16 stsp a->mtime = mtime;
625 301e83b3 2022-10-16 stsp return NULL;
628 301e83b3 2022-10-16 stsp if (got_object_idset_contains(a->idset, id) ||
629 301e83b3 2022-10-16 stsp got_object_idset_contains(a->idset_exclude, id))
630 301e83b3 2022-10-16 stsp return NULL;
632 301e83b3 2022-10-16 stsp relpath = dpath;
633 301e83b3 2022-10-16 stsp while (relpath[0] == '/')
636 301e83b3 2022-10-16 stsp err = got_pack_add_object(a->want_meta,
637 301e83b3 2022-10-16 stsp a->want_meta ? a->idset : a->idset_exclude,
638 301e83b3 2022-10-16 stsp id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
639 301e83b3 2022-10-16 stsp a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
640 301e83b3 2022-10-16 stsp a->progress_cb, a->progress_arg, a->rl);
642 301e83b3 2022-10-16 stsp return err;
644 301e83b3 2022-10-16 stsp return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
645 301e83b3 2022-10-16 stsp a->idset_exclude, tree, dpath, mtime, a->seed, repo,
646 301e83b3 2022-10-16 stsp a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
647 301e83b3 2022-10-16 stsp a->progress_cb, a->progress_arg, a->rl,
648 301e83b3 2022-10-16 stsp a->cancel_cb, a->cancel_arg);
651 301e83b3 2022-10-16 stsp const struct got_error *
652 301e83b3 2022-10-16 stsp got_pack_load_packed_object_ids(int *found_all_objects,
653 301e83b3 2022-10-16 stsp struct got_object_id **ours, int nours,
654 301e83b3 2022-10-16 stsp struct got_object_id **theirs, int ntheirs,
655 301e83b3 2022-10-16 stsp int want_meta, uint32_t seed, struct got_object_idset *idset,
656 301e83b3 2022-10-16 stsp struct got_object_idset *idset_exclude, int loose_obj_only,
657 301e83b3 2022-10-16 stsp struct got_repository *repo, struct got_packidx *packidx,
658 301e83b3 2022-10-16 stsp int *ncolored, int *nfound, int *ntrees,
659 301e83b3 2022-10-16 stsp got_pack_progress_cb progress_cb, void *progress_arg,
660 301e83b3 2022-10-16 stsp struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
662 301e83b3 2022-10-16 stsp const struct got_error *err = NULL;
663 301e83b3 2022-10-16 stsp struct load_packed_obj_arg lpa;
665 301e83b3 2022-10-16 stsp memset(&lpa, 0, sizeof(lpa));
666 301e83b3 2022-10-16 stsp lpa.seed = seed;
667 301e83b3 2022-10-16 stsp lpa.want_meta = want_meta;
668 301e83b3 2022-10-16 stsp lpa.idset = idset;
669 301e83b3 2022-10-16 stsp lpa.idset_exclude = idset_exclude;
670 301e83b3 2022-10-16 stsp lpa.loose_obj_only = loose_obj_only;
671 301e83b3 2022-10-16 stsp lpa.ncolored = ncolored;
672 301e83b3 2022-10-16 stsp lpa.nfound = nfound;
673 301e83b3 2022-10-16 stsp lpa.ntrees = ntrees;
674 301e83b3 2022-10-16 stsp lpa.progress_cb = progress_cb;
675 301e83b3 2022-10-16 stsp lpa.progress_arg = progress_arg;
676 301e83b3 2022-10-16 stsp lpa.rl = rl;
677 301e83b3 2022-10-16 stsp lpa.cancel_cb = cancel_cb;
678 301e83b3 2022-10-16 stsp lpa.cancel_arg = cancel_arg;
680 301e83b3 2022-10-16 stsp /* Attempt to load objects via got-read-pack, as far as possible. */
681 301e83b3 2022-10-16 stsp err = got_object_enumerate(found_all_objects, load_packed_commit_id,
682 301e83b3 2022-10-16 stsp load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
683 301e83b3 2022-10-16 stsp packidx, repo);
685 301e83b3 2022-10-16 stsp return err;
687 301e83b3 2022-10-16 stsp if (lpa.id == NULL)
688 301e83b3 2022-10-16 stsp return NULL;
691 301e83b3 2022-10-16 stsp * An incomplete tree hierarchy was present in the pack file
692 301e83b3 2022-10-16 stsp * and caused loading to be aborted.
693 301e83b3 2022-10-16 stsp * Continue loading trees the slow way.
695 301e83b3 2022-10-16 stsp err = got_pack_load_tree(want_meta, idset, idset_exclude,
696 301e83b3 2022-10-16 stsp lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
697 301e83b3 2022-10-16 stsp ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
698 301e83b3 2022-10-16 stsp cancel_cb, cancel_arg);
699 301e83b3 2022-10-16 stsp free(lpa.id);
700 301e83b3 2022-10-16 stsp free(lpa.dpath);
701 301e83b3 2022-10-16 stsp return err;