2 097b408a 2022-10-17 thomas * Copyright (c) 2020 Ori Bernstein
3 097b408a 2022-10-17 thomas * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
5 097b408a 2022-10-17 thomas * Permission to use, copy, modify, and distribute this software for any
6 097b408a 2022-10-17 thomas * purpose with or without fee is hereby granted, provided that the above
7 097b408a 2022-10-17 thomas * copyright notice and this permission notice appear in all copies.
9 097b408a 2022-10-17 thomas * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 097b408a 2022-10-17 thomas * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 097b408a 2022-10-17 thomas * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 097b408a 2022-10-17 thomas * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 097b408a 2022-10-17 thomas * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 097b408a 2022-10-17 thomas * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 097b408a 2022-10-17 thomas * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 097b408a 2022-10-17 thomas #include <sys/types.h>
19 097b408a 2022-10-17 thomas #include <sys/queue.h>
20 097b408a 2022-10-17 thomas #include <sys/uio.h>
22 588a8092 2023-02-23 thomas #include <sha1.h>
23 588a8092 2023-02-23 thomas #include <sha2.h>
24 097b408a 2022-10-17 thomas #include <limits.h>
25 097b408a 2022-10-17 thomas #include <stdio.h>
26 097b408a 2022-10-17 thomas #include <stdint.h>
27 097b408a 2022-10-17 thomas #include <stdlib.h>
28 097b408a 2022-10-17 thomas #include <string.h>
29 abd46894 2022-10-18 thomas #include <time.h>
30 097b408a 2022-10-17 thomas #include <imsg.h>
31 097b408a 2022-10-17 thomas #include <inttypes.h>
32 097b408a 2022-10-17 thomas #include <unistd.h>
34 097b408a 2022-10-17 thomas #include "got_error.h"
35 097b408a 2022-10-17 thomas #include "got_cancel.h"
36 097b408a 2022-10-17 thomas #include "got_object.h"
37 097b408a 2022-10-17 thomas #include "got_reference.h"
38 097b408a 2022-10-17 thomas #include "got_repository_admin.h"
39 097b408a 2022-10-17 thomas #include "got_path.h"
41 097b408a 2022-10-17 thomas #include "got_lib_delta.h"
42 097b408a 2022-10-17 thomas #include "got_lib_object.h"
43 097b408a 2022-10-17 thomas #include "got_lib_object_cache.h"
44 097b408a 2022-10-17 thomas #include "got_lib_object_idset.h"
45 097b408a 2022-10-17 thomas #include "got_lib_privsep.h"
46 abd46894 2022-10-18 thomas #include "got_lib_ratelimit.h"
47 097b408a 2022-10-17 thomas #include "got_lib_pack.h"
48 097b408a 2022-10-17 thomas #include "got_lib_pack_create.h"
49 097b408a 2022-10-17 thomas #include "got_lib_repository.h"
51 097b408a 2022-10-17 thomas struct send_id_arg {
52 097b408a 2022-10-17 thomas struct imsgbuf *ibuf;
53 097b408a 2022-10-17 thomas struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
54 097b408a 2022-10-17 thomas size_t nids;
57 097b408a 2022-10-17 thomas static const struct got_error *
58 097b408a 2022-10-17 thomas send_id(struct got_object_id *id, void *data, void *arg)
60 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
61 097b408a 2022-10-17 thomas struct send_id_arg *a = arg;
63 097b408a 2022-10-17 thomas a->ids[a->nids++] = id;
65 097b408a 2022-10-17 thomas if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
66 097b408a 2022-10-17 thomas err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
68 097b408a 2022-10-17 thomas return err;
69 097b408a 2022-10-17 thomas a->nids = 0;
72 097b408a 2022-10-17 thomas return NULL;
75 097b408a 2022-10-17 thomas static const struct got_error *
76 097b408a 2022-10-17 thomas send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
78 097b408a 2022-10-17 thomas const struct got_error *err;
79 097b408a 2022-10-17 thomas struct send_id_arg sia;
81 097b408a 2022-10-17 thomas memset(&sia, 0, sizeof(sia));
82 097b408a 2022-10-17 thomas sia.ibuf = ibuf;
83 097b408a 2022-10-17 thomas err = got_object_idset_for_each(idset, send_id, &sia);
85 097b408a 2022-10-17 thomas return err;
87 097b408a 2022-10-17 thomas if (sia.nids > 0) {
88 097b408a 2022-10-17 thomas err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
90 097b408a 2022-10-17 thomas return err;
93 097b408a 2022-10-17 thomas return got_privsep_send_object_idlist_done(ibuf);
96 097b408a 2022-10-17 thomas static const struct got_error *
97 097b408a 2022-10-17 thomas recv_reused_delta(struct got_imsg_reused_delta *delta,
98 097b408a 2022-10-17 thomas struct got_object_idset *idset, struct got_pack_metavec *v)
100 097b408a 2022-10-17 thomas struct got_pack_meta *m, *base;
102 097b408a 2022-10-17 thomas if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
103 097b408a 2022-10-17 thomas delta->delta_offset +
104 097b408a 2022-10-17 thomas delta->delta_compressed_size < delta->delta_offset)
105 097b408a 2022-10-17 thomas return got_error(GOT_ERR_BAD_PACKFILE);
107 097b408a 2022-10-17 thomas m = got_object_idset_get(idset, &delta->id);
108 097b408a 2022-10-17 thomas if (m == NULL)
109 097b408a 2022-10-17 thomas return got_error(GOT_ERR_NO_OBJ);
111 097b408a 2022-10-17 thomas base = got_object_idset_get(idset, &delta->base_id);
112 097b408a 2022-10-17 thomas if (base == NULL)
113 097b408a 2022-10-17 thomas return got_error(GOT_ERR_NO_OBJ);
115 097b408a 2022-10-17 thomas m->delta_len = delta->delta_size;
116 097b408a 2022-10-17 thomas m->delta_compressed_len = delta->delta_compressed_size;
117 c44c7d6e 2022-12-04 thomas m->delta_offset = 0;
118 097b408a 2022-10-17 thomas m->prev = base;
119 097b408a 2022-10-17 thomas m->size = delta->result_size;
120 097b408a 2022-10-17 thomas m->reused_delta_offset = delta->delta_offset;
121 097b408a 2022-10-17 thomas m->base_obj_id = got_object_id_dup(&delta->base_id);
122 097b408a 2022-10-17 thomas if (m->base_obj_id == NULL)
123 097b408a 2022-10-17 thomas return got_error_from_errno("got_object_id_dup");
125 097b408a 2022-10-17 thomas return got_pack_add_meta(m, v);
128 097b408a 2022-10-17 thomas const struct got_error *
129 c44c7d6e 2022-12-04 thomas got_pack_search_deltas(struct got_packidx **packidx, struct got_pack **pack,
130 c44c7d6e 2022-12-04 thomas struct got_pack_metavec *v, struct got_object_idset *idset,
131 097b408a 2022-10-17 thomas int ncolored, int nfound, int ntrees, int ncommits,
132 097b408a 2022-10-17 thomas struct got_repository *repo,
133 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb, void *progress_arg,
134 097b408a 2022-10-17 thomas struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
136 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
137 097b408a 2022-10-17 thomas struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
138 097b408a 2022-10-17 thomas size_t ndeltas, i;
140 c44c7d6e 2022-12-04 thomas *packidx = NULL;
141 c44c7d6e 2022-12-04 thomas *pack = NULL;
143 c44c7d6e 2022-12-04 thomas err = got_pack_find_pack_for_reuse(packidx, repo);
145 c44c7d6e 2022-12-04 thomas return err;
147 c44c7d6e 2022-12-04 thomas if (*packidx == NULL)
148 097b408a 2022-10-17 thomas return NULL;
150 c44c7d6e 2022-12-04 thomas err = got_pack_cache_pack_for_packidx(pack, *packidx, repo);
152 c44c7d6e 2022-12-04 thomas goto done;
154 c44c7d6e 2022-12-04 thomas if ((*pack)->privsep_child == NULL) {
155 c44c7d6e 2022-12-04 thomas err = got_pack_start_privsep_child(*pack, *packidx);
157 c44c7d6e 2022-12-04 thomas goto done;
160 c44c7d6e 2022-12-04 thomas err = got_privsep_send_delta_reuse_req((*pack)->privsep_child->ibuf);
162 c44c7d6e 2022-12-04 thomas goto done;
164 c44c7d6e 2022-12-04 thomas err = send_idset((*pack)->privsep_child->ibuf, idset);
166 c44c7d6e 2022-12-04 thomas goto done;
168 097b408a 2022-10-17 thomas for (;;) {
169 097b408a 2022-10-17 thomas int done = 0;
171 097b408a 2022-10-17 thomas if (cancel_cb) {
172 097b408a 2022-10-17 thomas err = (*cancel_cb)(cancel_arg);
177 097b408a 2022-10-17 thomas err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
178 c44c7d6e 2022-12-04 thomas (*pack)->privsep_child->ibuf);
179 097b408a 2022-10-17 thomas if (err || done)
182 097b408a 2022-10-17 thomas for (i = 0; i < ndeltas; i++) {
183 097b408a 2022-10-17 thomas struct got_imsg_reused_delta *delta = &deltas[i];
184 097b408a 2022-10-17 thomas err = recv_reused_delta(delta, idset, v);
186 097b408a 2022-10-17 thomas goto done;
189 097b408a 2022-10-17 thomas err = got_pack_report_progress(progress_cb, progress_arg, rl,
190 097b408a 2022-10-17 thomas ncolored, nfound, ntrees, 0L, ncommits,
191 097b408a 2022-10-17 thomas got_object_idset_num_elements(idset), v->nmeta, 0);
196 097b408a 2022-10-17 thomas return err;
199 097b408a 2022-10-17 thomas struct recv_painted_commit_arg {
200 097b408a 2022-10-17 thomas int *ncolored;
201 097b408a 2022-10-17 thomas int *nqueued;
202 097b408a 2022-10-17 thomas int *nskip;
203 097b408a 2022-10-17 thomas struct got_object_id_queue *ids;
204 097b408a 2022-10-17 thomas struct got_object_idset *keep;
205 097b408a 2022-10-17 thomas struct got_object_idset *drop;
206 097b408a 2022-10-17 thomas struct got_object_idset *skip;
207 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb;
208 097b408a 2022-10-17 thomas void *progress_arg;
209 097b408a 2022-10-17 thomas struct got_ratelimit *rl;
210 097b408a 2022-10-17 thomas got_cancel_cb cancel_cb;
211 097b408a 2022-10-17 thomas void *cancel_arg;
214 097b408a 2022-10-17 thomas static const struct got_error *
215 097b408a 2022-10-17 thomas recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
217 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
218 097b408a 2022-10-17 thomas struct recv_painted_commit_arg *a = arg;
219 097b408a 2022-10-17 thomas struct got_object_qid *qid, *tmp;
221 097b408a 2022-10-17 thomas if (a->cancel_cb) {
222 097b408a 2022-10-17 thomas err = a->cancel_cb(a->cancel_arg);
224 097b408a 2022-10-17 thomas return err;
227 097b408a 2022-10-17 thomas switch (color) {
228 097b408a 2022-10-17 thomas case COLOR_KEEP:
229 097b408a 2022-10-17 thomas err = got_object_idset_add(a->keep, id, NULL);
231 097b408a 2022-10-17 thomas return err;
232 097b408a 2022-10-17 thomas (*a->ncolored)++;
234 097b408a 2022-10-17 thomas case COLOR_DROP:
235 097b408a 2022-10-17 thomas err = got_object_idset_add(a->drop, id, NULL);
237 097b408a 2022-10-17 thomas return err;
238 097b408a 2022-10-17 thomas (*a->ncolored)++;
240 097b408a 2022-10-17 thomas case COLOR_SKIP:
241 097b408a 2022-10-17 thomas err = got_object_idset_add(a->skip, id, NULL);
243 097b408a 2022-10-17 thomas return err;
246 097b408a 2022-10-17 thomas /* should not happen */
247 097b408a 2022-10-17 thomas return got_error_fmt(GOT_ERR_NOT_IMPL,
248 097b408a 2022-10-17 thomas "%s invalid commit color %"PRIdPTR, __func__, color);
251 097b408a 2022-10-17 thomas STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
252 097b408a 2022-10-17 thomas if (got_object_id_cmp(&qid->id, id) != 0)
253 097b408a 2022-10-17 thomas continue;
254 097b408a 2022-10-17 thomas STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
255 097b408a 2022-10-17 thomas color = (intptr_t)qid->data;
256 097b408a 2022-10-17 thomas got_object_qid_free(qid);
257 097b408a 2022-10-17 thomas (*a->nqueued)--;
258 097b408a 2022-10-17 thomas if (color == COLOR_SKIP)
259 097b408a 2022-10-17 thomas (*a->nskip)--;
263 097b408a 2022-10-17 thomas return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
264 097b408a 2022-10-17 thomas *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
267 097b408a 2022-10-17 thomas static const struct got_error *
268 097b408a 2022-10-17 thomas paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
269 097b408a 2022-10-17 thomas int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
270 097b408a 2022-10-17 thomas struct got_object_id_queue *ids,
271 097b408a 2022-10-17 thomas struct got_object_idset *keep, struct got_object_idset *drop,
272 097b408a 2022-10-17 thomas struct got_object_idset *skip, struct got_repository *repo,
273 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb, void *progress_arg,
274 097b408a 2022-10-17 thomas struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
276 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
277 097b408a 2022-10-17 thomas struct got_object_id_queue next_ids;
278 097b408a 2022-10-17 thomas struct got_object_qid *qid, *tmp;
279 097b408a 2022-10-17 thomas struct recv_painted_commit_arg arg;
281 097b408a 2022-10-17 thomas STAILQ_INIT(&next_ids);
283 097b408a 2022-10-17 thomas err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
284 097b408a 2022-10-17 thomas idx, id, color);
286 097b408a 2022-10-17 thomas return err;
288 097b408a 2022-10-17 thomas arg.ncolored = ncolored;
289 097b408a 2022-10-17 thomas arg.nqueued = nqueued;
290 097b408a 2022-10-17 thomas arg.nskip = nskip;
291 097b408a 2022-10-17 thomas arg.ids = ids;
292 097b408a 2022-10-17 thomas arg.keep = keep;
293 097b408a 2022-10-17 thomas arg.drop = drop;
294 097b408a 2022-10-17 thomas arg.skip = skip;
295 097b408a 2022-10-17 thomas arg.progress_cb = progress_cb;
296 097b408a 2022-10-17 thomas arg.progress_arg = progress_arg;
297 097b408a 2022-10-17 thomas arg.rl = rl;
298 097b408a 2022-10-17 thomas arg.cancel_cb = cancel_cb;
299 097b408a 2022-10-17 thomas arg.cancel_arg = cancel_arg;
300 097b408a 2022-10-17 thomas err = got_privsep_recv_painted_commits(&next_ids,
301 097b408a 2022-10-17 thomas recv_painted_commit, &arg, pack->privsep_child->ibuf);
303 097b408a 2022-10-17 thomas return err;
305 097b408a 2022-10-17 thomas STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
306 097b408a 2022-10-17 thomas struct got_object_qid *old_id;
307 097b408a 2022-10-17 thomas intptr_t qcolor, ocolor;
308 097b408a 2022-10-17 thomas STAILQ_FOREACH(old_id, ids, entry) {
309 097b408a 2022-10-17 thomas if (got_object_id_cmp(&qid->id, &old_id->id))
310 097b408a 2022-10-17 thomas continue;
311 097b408a 2022-10-17 thomas qcolor = (intptr_t)qid->data;
312 097b408a 2022-10-17 thomas ocolor = (intptr_t)old_id->data;
313 097b408a 2022-10-17 thomas STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
314 097b408a 2022-10-17 thomas got_object_qid_free(qid);
315 097b408a 2022-10-17 thomas qid = NULL;
316 097b408a 2022-10-17 thomas if (qcolor != ocolor) {
317 097b408a 2022-10-17 thomas got_pack_paint_commit(old_id, qcolor);
318 097b408a 2022-10-17 thomas if (ocolor == COLOR_SKIP)
319 097b408a 2022-10-17 thomas (*nskip)--;
320 097b408a 2022-10-17 thomas else if (qcolor == COLOR_SKIP)
321 097b408a 2022-10-17 thomas (*nskip)++;
326 097b408a 2022-10-17 thomas while (!STAILQ_EMPTY(&next_ids)) {
327 097b408a 2022-10-17 thomas qid = STAILQ_FIRST(&next_ids);
328 097b408a 2022-10-17 thomas STAILQ_REMOVE_HEAD(&next_ids, entry);
329 097b408a 2022-10-17 thomas got_pack_paint_commit(qid, color);
330 097b408a 2022-10-17 thomas STAILQ_INSERT_TAIL(ids, qid, entry);
331 097b408a 2022-10-17 thomas (*nqueued)++;
332 097b408a 2022-10-17 thomas if (color == COLOR_SKIP)
333 097b408a 2022-10-17 thomas (*nskip)++;
336 097b408a 2022-10-17 thomas return err;
339 097b408a 2022-10-17 thomas const struct got_error *
340 097b408a 2022-10-17 thomas got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
341 097b408a 2022-10-17 thomas struct got_object_idset *keep, struct got_object_idset *drop,
342 097b408a 2022-10-17 thomas struct got_object_idset *skip, struct got_repository *repo,
343 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb, void *progress_arg,
344 097b408a 2022-10-17 thomas struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
346 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
347 097b408a 2022-10-17 thomas struct got_commit_object *commit = NULL;
348 097b408a 2022-10-17 thomas struct got_packidx *packidx = NULL;
349 097b408a 2022-10-17 thomas struct got_pack *pack = NULL;
350 097b408a 2022-10-17 thomas const struct got_object_id_queue *parents;
351 097b408a 2022-10-17 thomas struct got_object_qid *qid = NULL;
352 097b408a 2022-10-17 thomas int nqueued = nids, nskip = 0;
355 097b408a 2022-10-17 thomas while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
356 097b408a 2022-10-17 thomas intptr_t color;
358 097b408a 2022-10-17 thomas if (cancel_cb) {
359 097b408a 2022-10-17 thomas err = cancel_cb(cancel_arg);
364 097b408a 2022-10-17 thomas qid = STAILQ_FIRST(ids);
365 097b408a 2022-10-17 thomas STAILQ_REMOVE_HEAD(ids, entry);
366 097b408a 2022-10-17 thomas nqueued--;
367 097b408a 2022-10-17 thomas color = (intptr_t)qid->data;
368 097b408a 2022-10-17 thomas if (color == COLOR_SKIP)
371 097b408a 2022-10-17 thomas if (got_object_idset_contains(skip, &qid->id)) {
372 097b408a 2022-10-17 thomas got_object_qid_free(qid);
373 097b408a 2022-10-17 thomas qid = NULL;
374 097b408a 2022-10-17 thomas continue;
376 097b408a 2022-10-17 thomas if (color == COLOR_KEEP &&
377 097b408a 2022-10-17 thomas got_object_idset_contains(keep, &qid->id)) {
378 097b408a 2022-10-17 thomas got_object_qid_free(qid);
379 097b408a 2022-10-17 thomas qid = NULL;
380 097b408a 2022-10-17 thomas continue;
382 097b408a 2022-10-17 thomas if (color == COLOR_DROP &&
383 097b408a 2022-10-17 thomas got_object_idset_contains(drop, &qid->id)) {
384 097b408a 2022-10-17 thomas got_object_qid_free(qid);
385 097b408a 2022-10-17 thomas qid = NULL;
386 097b408a 2022-10-17 thomas continue;
389 097b408a 2022-10-17 thomas /* Pinned pack may have moved to different cache slot. */
390 097b408a 2022-10-17 thomas pack = got_repo_get_pinned_pack(repo);
392 097b408a 2022-10-17 thomas if (packidx && pack) {
393 097b408a 2022-10-17 thomas idx = got_packidx_get_object_idx(packidx, &qid->id);
394 097b408a 2022-10-17 thomas if (idx != -1) {
395 097b408a 2022-10-17 thomas err = paint_packed_commits(pack, &qid->id,
396 097b408a 2022-10-17 thomas idx, color, ncolored, &nqueued, &nskip,
397 097b408a 2022-10-17 thomas ids, keep, drop, skip, repo,
398 097b408a 2022-10-17 thomas progress_cb, progress_arg, rl,
399 097b408a 2022-10-17 thomas cancel_cb, cancel_arg);
402 097b408a 2022-10-17 thomas got_object_qid_free(qid);
403 097b408a 2022-10-17 thomas qid = NULL;
404 097b408a 2022-10-17 thomas continue;
408 097b408a 2022-10-17 thomas switch (color) {
409 097b408a 2022-10-17 thomas case COLOR_KEEP:
410 097b408a 2022-10-17 thomas if (got_object_idset_contains(drop, &qid->id)) {
411 097b408a 2022-10-17 thomas err = got_pack_paint_commit(qid, COLOR_SKIP);
413 097b408a 2022-10-17 thomas goto done;
415 097b408a 2022-10-17 thomas (*ncolored)++;
416 097b408a 2022-10-17 thomas err = got_object_idset_add(keep, &qid->id, NULL);
418 097b408a 2022-10-17 thomas goto done;
420 097b408a 2022-10-17 thomas case COLOR_DROP:
421 097b408a 2022-10-17 thomas if (got_object_idset_contains(keep, &qid->id)) {
422 097b408a 2022-10-17 thomas err = got_pack_paint_commit(qid, COLOR_SKIP);
424 097b408a 2022-10-17 thomas goto done;
426 097b408a 2022-10-17 thomas (*ncolored)++;
427 097b408a 2022-10-17 thomas err = got_object_idset_add(drop, &qid->id, NULL);
429 097b408a 2022-10-17 thomas goto done;
431 097b408a 2022-10-17 thomas case COLOR_SKIP:
432 097b408a 2022-10-17 thomas if (!got_object_idset_contains(skip, &qid->id)) {
433 097b408a 2022-10-17 thomas err = got_object_idset_add(skip, &qid->id,
436 097b408a 2022-10-17 thomas goto done;
440 097b408a 2022-10-17 thomas /* should not happen */
441 097b408a 2022-10-17 thomas err = got_error_fmt(GOT_ERR_NOT_IMPL,
442 097b408a 2022-10-17 thomas "%s invalid commit color %"PRIdPTR, __func__,
444 097b408a 2022-10-17 thomas goto done;
447 097b408a 2022-10-17 thomas err = got_pack_report_progress(progress_cb, progress_arg, rl,
448 097b408a 2022-10-17 thomas *ncolored, 0, 0, 0L, 0, 0, 0, 0);
452 097b408a 2022-10-17 thomas err = got_object_open_as_commit(&commit, repo, &qid->id);
456 097b408a 2022-10-17 thomas parents = got_object_commit_get_parent_ids(commit);
457 097b408a 2022-10-17 thomas if (parents) {
458 097b408a 2022-10-17 thomas struct got_object_qid *pid;
459 097b408a 2022-10-17 thomas color = (intptr_t)qid->data;
460 097b408a 2022-10-17 thomas STAILQ_FOREACH(pid, parents, entry) {
461 097b408a 2022-10-17 thomas err = got_pack_queue_commit_id(ids, &pid->id,
462 097b408a 2022-10-17 thomas color, repo);
465 097b408a 2022-10-17 thomas nqueued++;
466 097b408a 2022-10-17 thomas if (color == COLOR_SKIP)
471 097b408a 2022-10-17 thomas if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
472 097b408a 2022-10-17 thomas if (packidx == NULL) {
473 097b408a 2022-10-17 thomas err = got_pack_find_pack_for_commit_painting(
474 097b408a 2022-10-17 thomas &packidx, ids, nqueued, repo);
476 097b408a 2022-10-17 thomas goto done;
478 097b408a 2022-10-17 thomas if (packidx != NULL) {
479 097b408a 2022-10-17 thomas err = got_pack_cache_pack_for_packidx(&pack,
480 097b408a 2022-10-17 thomas packidx, repo);
482 097b408a 2022-10-17 thomas goto done;
483 097b408a 2022-10-17 thomas if (pack->privsep_child == NULL) {
484 097b408a 2022-10-17 thomas err = got_pack_start_privsep_child(
485 097b408a 2022-10-17 thomas pack, packidx);
487 097b408a 2022-10-17 thomas goto done;
489 097b408a 2022-10-17 thomas err = got_privsep_init_commit_painting(
490 097b408a 2022-10-17 thomas pack->privsep_child->ibuf);
492 097b408a 2022-10-17 thomas goto done;
493 097b408a 2022-10-17 thomas err = send_idset(pack->privsep_child->ibuf,
496 097b408a 2022-10-17 thomas goto done;
497 097b408a 2022-10-17 thomas err = send_idset(pack->privsep_child->ibuf, drop);
499 097b408a 2022-10-17 thomas goto done;
500 097b408a 2022-10-17 thomas err = send_idset(pack->privsep_child->ibuf, skip);
502 097b408a 2022-10-17 thomas goto done;
503 097b408a 2022-10-17 thomas err = got_repo_pin_pack(repo, packidx, pack);
505 097b408a 2022-10-17 thomas goto done;
509 097b408a 2022-10-17 thomas got_object_commit_close(commit);
510 097b408a 2022-10-17 thomas commit = NULL;
512 097b408a 2022-10-17 thomas got_object_qid_free(qid);
513 097b408a 2022-10-17 thomas qid = NULL;
516 097b408a 2022-10-17 thomas if (pack) {
517 097b408a 2022-10-17 thomas const struct got_error *pack_err;
518 097b408a 2022-10-17 thomas pack_err = got_privsep_send_painting_commits_done(
519 097b408a 2022-10-17 thomas pack->privsep_child->ibuf);
520 097b408a 2022-10-17 thomas if (err == NULL)
521 097b408a 2022-10-17 thomas err = pack_err;
523 097b408a 2022-10-17 thomas if (commit)
524 097b408a 2022-10-17 thomas got_object_commit_close(commit);
525 097b408a 2022-10-17 thomas got_object_qid_free(qid);
526 097b408a 2022-10-17 thomas got_repo_unpin_pack(repo);
527 097b408a 2022-10-17 thomas return err;
530 097b408a 2022-10-17 thomas struct load_packed_obj_arg {
531 097b408a 2022-10-17 thomas /* output parameters: */
532 097b408a 2022-10-17 thomas struct got_object_id *id;
533 097b408a 2022-10-17 thomas char *dpath;
534 097b408a 2022-10-17 thomas time_t mtime;
536 097b408a 2022-10-17 thomas /* input parameters: */
537 097b408a 2022-10-17 thomas uint32_t seed;
538 097b408a 2022-10-17 thomas int want_meta;
539 097b408a 2022-10-17 thomas struct got_object_idset *idset;
540 097b408a 2022-10-17 thomas struct got_object_idset *idset_exclude;
541 097b408a 2022-10-17 thomas int loose_obj_only;
542 097b408a 2022-10-17 thomas int *ncolored;
543 097b408a 2022-10-17 thomas int *nfound;
544 097b408a 2022-10-17 thomas int *ntrees;
545 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb;
546 097b408a 2022-10-17 thomas void *progress_arg;
547 097b408a 2022-10-17 thomas struct got_ratelimit *rl;
548 097b408a 2022-10-17 thomas got_cancel_cb cancel_cb;
549 097b408a 2022-10-17 thomas void *cancel_arg;
552 097b408a 2022-10-17 thomas static const struct got_error *
553 097b408a 2022-10-17 thomas load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
554 097b408a 2022-10-17 thomas struct got_repository *repo)
556 097b408a 2022-10-17 thomas struct load_packed_obj_arg *a = arg;
558 097b408a 2022-10-17 thomas if (got_object_idset_contains(a->idset, id) ||
559 097b408a 2022-10-17 thomas got_object_idset_contains(a->idset_exclude, id))
560 097b408a 2022-10-17 thomas return NULL;
562 097b408a 2022-10-17 thomas return got_pack_add_object(a->want_meta,
563 097b408a 2022-10-17 thomas a->want_meta ? a->idset : a->idset_exclude,
564 097b408a 2022-10-17 thomas id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
565 097b408a 2022-10-17 thomas repo, a->ncolored, a->nfound, a->ntrees,
566 097b408a 2022-10-17 thomas a->progress_cb, a->progress_arg, a->rl);
569 097b408a 2022-10-17 thomas static const struct got_error *
570 097b408a 2022-10-17 thomas load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
571 097b408a 2022-10-17 thomas struct got_object_id *id, const char *dpath, struct got_repository *repo)
573 097b408a 2022-10-17 thomas const struct got_error *err;
574 097b408a 2022-10-17 thomas struct load_packed_obj_arg *a = arg;
575 097b408a 2022-10-17 thomas const char *relpath;
578 097b408a 2022-10-17 thomas * When we receive a tree's ID and path but not the tree itself,
579 097b408a 2022-10-17 thomas * this tree object was not found in the pack file. This is the
580 097b408a 2022-10-17 thomas * last time we are being called for this optimized traversal.
581 097b408a 2022-10-17 thomas * Return from here and switch to loading objects the slow way.
583 097b408a 2022-10-17 thomas if (tree == NULL) {
584 097b408a 2022-10-17 thomas free(a->id);
585 097b408a 2022-10-17 thomas a->id = got_object_id_dup(id);
586 097b408a 2022-10-17 thomas if (a->id == NULL) {
587 097b408a 2022-10-17 thomas err = got_error_from_errno("got_object_id_dup");
588 097b408a 2022-10-17 thomas free(a->dpath);
589 097b408a 2022-10-17 thomas a->dpath = NULL;
590 097b408a 2022-10-17 thomas return err;
593 097b408a 2022-10-17 thomas free(a->dpath);
594 097b408a 2022-10-17 thomas a->dpath = strdup(dpath);
595 097b408a 2022-10-17 thomas if (a->dpath == NULL) {
596 097b408a 2022-10-17 thomas err = got_error_from_errno("strdup");
597 097b408a 2022-10-17 thomas free(a->id);
598 097b408a 2022-10-17 thomas a->id = NULL;
599 097b408a 2022-10-17 thomas return err;
602 097b408a 2022-10-17 thomas a->mtime = mtime;
603 097b408a 2022-10-17 thomas return NULL;
606 097b408a 2022-10-17 thomas if (got_object_idset_contains(a->idset, id) ||
607 097b408a 2022-10-17 thomas got_object_idset_contains(a->idset_exclude, id))
608 097b408a 2022-10-17 thomas return NULL;
610 097b408a 2022-10-17 thomas relpath = dpath;
611 097b408a 2022-10-17 thomas while (relpath[0] == '/')
612 097b408a 2022-10-17 thomas relpath++;
614 097b408a 2022-10-17 thomas err = got_pack_add_object(a->want_meta,
615 097b408a 2022-10-17 thomas a->want_meta ? a->idset : a->idset_exclude,
616 097b408a 2022-10-17 thomas id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
617 097b408a 2022-10-17 thomas a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
618 097b408a 2022-10-17 thomas a->progress_cb, a->progress_arg, a->rl);
620 097b408a 2022-10-17 thomas return err;
622 097b408a 2022-10-17 thomas return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
623 097b408a 2022-10-17 thomas a->idset_exclude, tree, dpath, mtime, a->seed, repo,
624 097b408a 2022-10-17 thomas a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
625 097b408a 2022-10-17 thomas a->progress_cb, a->progress_arg, a->rl,
626 097b408a 2022-10-17 thomas a->cancel_cb, a->cancel_arg);
629 097b408a 2022-10-17 thomas const struct got_error *
630 097b408a 2022-10-17 thomas got_pack_load_packed_object_ids(int *found_all_objects,
631 097b408a 2022-10-17 thomas struct got_object_id **ours, int nours,
632 097b408a 2022-10-17 thomas struct got_object_id **theirs, int ntheirs,
633 097b408a 2022-10-17 thomas int want_meta, uint32_t seed, struct got_object_idset *idset,
634 097b408a 2022-10-17 thomas struct got_object_idset *idset_exclude, int loose_obj_only,
635 097b408a 2022-10-17 thomas struct got_repository *repo, struct got_packidx *packidx,
636 097b408a 2022-10-17 thomas int *ncolored, int *nfound, int *ntrees,
637 097b408a 2022-10-17 thomas got_pack_progress_cb progress_cb, void *progress_arg,
638 097b408a 2022-10-17 thomas struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
640 097b408a 2022-10-17 thomas const struct got_error *err = NULL;
641 097b408a 2022-10-17 thomas struct load_packed_obj_arg lpa;
643 097b408a 2022-10-17 thomas memset(&lpa, 0, sizeof(lpa));
644 097b408a 2022-10-17 thomas lpa.seed = seed;
645 097b408a 2022-10-17 thomas lpa.want_meta = want_meta;
646 097b408a 2022-10-17 thomas lpa.idset = idset;
647 097b408a 2022-10-17 thomas lpa.idset_exclude = idset_exclude;
648 097b408a 2022-10-17 thomas lpa.loose_obj_only = loose_obj_only;
649 097b408a 2022-10-17 thomas lpa.ncolored = ncolored;
650 097b408a 2022-10-17 thomas lpa.nfound = nfound;
651 097b408a 2022-10-17 thomas lpa.ntrees = ntrees;
652 097b408a 2022-10-17 thomas lpa.progress_cb = progress_cb;
653 097b408a 2022-10-17 thomas lpa.progress_arg = progress_arg;
654 097b408a 2022-10-17 thomas lpa.rl = rl;
655 097b408a 2022-10-17 thomas lpa.cancel_cb = cancel_cb;
656 097b408a 2022-10-17 thomas lpa.cancel_arg = cancel_arg;
658 097b408a 2022-10-17 thomas /* Attempt to load objects via got-read-pack, as far as possible. */
659 097b408a 2022-10-17 thomas err = got_object_enumerate(found_all_objects, load_packed_commit_id,
660 097b408a 2022-10-17 thomas load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
661 097b408a 2022-10-17 thomas packidx, repo);
663 097b408a 2022-10-17 thomas return err;
665 097b408a 2022-10-17 thomas if (lpa.id == NULL)
666 097b408a 2022-10-17 thomas return NULL;
669 097b408a 2022-10-17 thomas * An incomplete tree hierarchy was present in the pack file
670 097b408a 2022-10-17 thomas * and caused loading to be aborted.
671 097b408a 2022-10-17 thomas * Continue loading trees the slow way.
673 097b408a 2022-10-17 thomas err = got_pack_load_tree(want_meta, idset, idset_exclude,
674 097b408a 2022-10-17 thomas lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
675 097b408a 2022-10-17 thomas ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
676 097b408a 2022-10-17 thomas cancel_cb, cancel_arg);
677 097b408a 2022-10-17 thomas free(lpa.id);
678 097b408a 2022-10-17 thomas free(lpa.dpath);
679 097b408a 2022-10-17 thomas return err;