Gotweb

Blame

Date:: Sat May 7 11:50:56 2022 UTC
Message:: run the search for deltas to reuse in got-read-pack This significantly speeds up the deltification step of packing by avoiding imsg traffic. gotadmin no longer requests individual raw deltas from got-read-pack to check whether it can reuse them. Instead, got-read-pack obtains a list of objects we want to pack, and hands back the list of all deltas in its pack file which can be reused. Messages are now batched such that imsg buffers are filled as much as possible. Another advantage is that deltas we are not going to reuse will no longer be written to the delta cache file, saving disk space. Before this patch, any raw delta candidate was written to the delta cache file by got-read-pack, and the decision whether to reuse the delta happened afterwards in the gotadmin process. Code for reading individual raw deltas is now unused and could be removed at some point. ok op@
Actions:: History | Blob | Raw File
   1 e6bcace5 2021-06-22 stsp /*
   2 e6bcace5 2021-06-22 stsp  * Copyright (c) 2020 Ori Bernstein
   3 e6bcace5 2021-06-22 stsp  * Copyright (c) 2021 Stefan Sperling <stsp@openbsd.org>
   4 e6bcace5 2021-06-22 stsp  *
   5 e6bcace5 2021-06-22 stsp  * Permission to use, copy, modify, and distribute this software for any
   6 e6bcace5 2021-06-22 stsp  * purpose with or without fee is hereby granted, provided that the above
   7 e6bcace5 2021-06-22 stsp  * copyright notice and this permission notice appear in all copies.
   8 e6bcace5 2021-06-22 stsp  *
   9 e6bcace5 2021-06-22 stsp  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10 e6bcace5 2021-06-22 stsp  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11 e6bcace5 2021-06-22 stsp  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12 e6bcace5 2021-06-22 stsp  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13 e6bcace5 2021-06-22 stsp  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14 e6bcace5 2021-06-22 stsp  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15 e6bcace5 2021-06-22 stsp  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16 e6bcace5 2021-06-22 stsp  */
  17 e6bcace5 2021-06-22 stsp 
  18 08736cf9 2021-06-23 stsp #include <sys/types.h>
  19 e6bcace5 2021-06-22 stsp #include <sys/queue.h>
  20 f8b19efd 2021-10-13 stsp #include <sys/tree.h>
  21 08736cf9 2021-06-23 stsp #include <sys/uio.h>
  22 e6bcace5 2021-06-22 stsp #include <sys/stat.h>
  23 211cfef0 2022-01-05 stsp #include <sys/time.h>
  24 e6bcace5 2021-06-22 stsp 
  25 e3f86256 2022-02-18 naddy #include <endian.h>
  26 08736cf9 2021-06-23 stsp #include <stdint.h>
  27 05118f5a 2021-06-22 stsp #include <imsg.h>
  28 e6bcace5 2021-06-22 stsp #include <stdio.h>
  29 e6bcace5 2021-06-22 stsp #include <stdlib.h>
  30 e6bcace5 2021-06-22 stsp #include <string.h>
  31 e6bcace5 2021-06-22 stsp #include <sha1.h>
  32 211cfef0 2022-01-05 stsp #include <time.h>
  33 bfc73a47 2022-03-19 naddy #include <unistd.h>
  34 e6bcace5 2021-06-22 stsp #include <limits.h>
  35 e6bcace5 2021-06-22 stsp #include <zlib.h>
  36 e6bcace5 2021-06-22 stsp 
  37 e6bcace5 2021-06-22 stsp #include "got_error.h"
  38 e6bcace5 2021-06-22 stsp #include "got_cancel.h"
  39 e6bcace5 2021-06-22 stsp #include "got_object.h"
  40 f8b19efd 2021-10-13 stsp #include "got_path.h"
  41 05118f5a 2021-06-22 stsp #include "got_reference.h"
  42 05118f5a 2021-06-22 stsp #include "got_repository_admin.h"
  43 08347b73 2021-10-14 stsp #include "got_opentemp.h"
  44 e6bcace5 2021-06-22 stsp 
  45 e6bcace5 2021-06-22 stsp #include "got_lib_deltify.h"
  46 e6bcace5 2021-06-22 stsp #include "got_lib_delta.h"
  47 e6bcace5 2021-06-22 stsp #include "got_lib_object.h"
  48 e6bcace5 2021-06-22 stsp #include "got_lib_object_idset.h"
  49 05118f5a 2021-06-22 stsp #include "got_lib_object_cache.h"
  50 e6bcace5 2021-06-22 stsp #include "got_lib_deflate.h"
  51 e6bcace5 2021-06-22 stsp #include "got_lib_pack.h"
  52 05118f5a 2021-06-22 stsp #include "got_lib_privsep.h"
  53 05118f5a 2021-06-22 stsp #include "got_lib_repository.h"
  54 211cfef0 2022-01-05 stsp #include "got_lib_ratelimit.h"
  55 2d9e6abf 2022-05-04 stsp #include "got_lib_inflate.h"
  56 e6bcace5 2021-06-22 stsp 
  57 74881701 2021-10-15 stsp #ifndef MIN
  58 74881701 2021-10-15 stsp #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  59 74881701 2021-10-15 stsp #endif
  60 74881701 2021-10-15 stsp 
  61 e6bcace5 2021-06-22 stsp #ifndef MAX
  62 e6bcace5 2021-06-22 stsp #define	MAX(_a,_b) ((_a) > (_b) ? (_a) : (_b))
  63 e6bcace5 2021-06-22 stsp #endif
  64 e6bcace5 2021-06-22 stsp 
  65 70f8f24d 2022-04-14 stsp #ifndef nitems
  66 70f8f24d 2022-04-14 stsp #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
  67 70f8f24d 2022-04-14 stsp #endif
  68 70f8f24d 2022-04-14 stsp 
  69 e6bcace5 2021-06-22 stsp struct got_pack_meta {
  70 e6bcace5 2021-06-22 stsp 	struct got_object_id id;
  71 e6bcace5 2021-06-22 stsp 	char	*path;
  72 e6bcace5 2021-06-22 stsp 	int	obj_type;
  73 600b755e 2021-10-14 stsp 	off_t	size;
  74 e6bcace5 2021-06-22 stsp 	time_t	mtime;
  75 e6bcace5 2021-06-22 stsp 
  76 e6bcace5 2021-06-22 stsp 	/* The best delta we picked */
  77 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *head;
  78 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *prev;
  79 2d9e6abf 2022-05-04 stsp 	unsigned char *delta_buf; /* if encoded in memory (compressed) */
  80 2d9e6abf 2022-05-04 stsp 	off_t	delta_offset;	/* offset in delta cache file (compressed) */
  81 5060d5a1 2022-01-10 stsp 	off_t	delta_len;	/* encoded delta length */
  82 2d9e6abf 2022-05-04 stsp 	off_t	delta_compressed_len; /* encoded+compressed delta length */
  83 e6bcace5 2021-06-22 stsp 	int	nchain;
  84 e6bcace5 2021-06-22 stsp 
  85 67fd6849 2022-02-13 stsp 	int	have_reused_delta;
  86 67fd6849 2022-02-13 stsp 	off_t   reused_delta_offset; /* offset of delta in reused pack file */
  87 67fd6849 2022-02-13 stsp 	struct got_object_id *base_obj_id;
  88 67fd6849 2022-02-13 stsp 
  89 e6bcace5 2021-06-22 stsp 	/* Only used for delta window */
  90 e6bcace5 2021-06-22 stsp 	struct got_delta_table *dtab;
  91 e6bcace5 2021-06-22 stsp 
  92 e6bcace5 2021-06-22 stsp 	/* Only used for writing offset deltas */
  93 e6bcace5 2021-06-22 stsp 	off_t	off;
  94 e6bcace5 2021-06-22 stsp };
  95 e6bcace5 2021-06-22 stsp 
  96 e6bcace5 2021-06-22 stsp struct got_pack_metavec {
  97 e6bcace5 2021-06-22 stsp 	struct got_pack_meta **meta;
  98 e6bcace5 2021-06-22 stsp 	int nmeta;
  99 e6bcace5 2021-06-22 stsp 	int metasz;
 100 e6bcace5 2021-06-22 stsp };
 101 e6bcace5 2021-06-22 stsp 
 102 e6bcace5 2021-06-22 stsp static const struct got_error *
 103 e6bcace5 2021-06-22 stsp alloc_meta(struct got_pack_meta **new, struct got_object_id *id,
 104 e6bcace5 2021-06-22 stsp     const char *path, int obj_type, time_t mtime)
 105 e6bcace5 2021-06-22 stsp {
 106 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
 107 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *m;
 108 e6bcace5 2021-06-22 stsp 
 109 e6bcace5 2021-06-22 stsp 	*new = NULL;
 110 e6bcace5 2021-06-22 stsp 
 111 e6bcace5 2021-06-22 stsp 	m = calloc(1, sizeof(*m));
 112 e6bcace5 2021-06-22 stsp 	if (m == NULL)
 113 1d19226a 2021-10-13 stsp 		return got_error_from_errno("calloc");
 114 e6bcace5 2021-06-22 stsp 
 115 e6bcace5 2021-06-22 stsp 	memcpy(&m->id, id, sizeof(m->id));
 116 e6bcace5 2021-06-22 stsp 
 117 e6bcace5 2021-06-22 stsp 	m->path = strdup(path);
 118 e6bcace5 2021-06-22 stsp 	if (m->path == NULL) {
 119 e6bcace5 2021-06-22 stsp 		err = got_error_from_errno("strdup");
 120 e6bcace5 2021-06-22 stsp 		free(m);
 121 e6bcace5 2021-06-22 stsp 		return err;
 122 e6bcace5 2021-06-22 stsp 	}
 123 e6bcace5 2021-06-22 stsp 
 124 e6bcace5 2021-06-22 stsp 	m->obj_type = obj_type;
 125 e6bcace5 2021-06-22 stsp 	m->mtime = mtime;
 126 e6bcace5 2021-06-22 stsp 	*new = m;
 127 e6bcace5 2021-06-22 stsp 	return NULL;
 128 e6bcace5 2021-06-22 stsp }
 129 e6bcace5 2021-06-22 stsp 
 130 e6bcace5 2021-06-22 stsp static void
 131 e6bcace5 2021-06-22 stsp clear_meta(struct got_pack_meta *meta)
 132 e6bcace5 2021-06-22 stsp {
 133 e6bcace5 2021-06-22 stsp 	if (meta == NULL)
 134 e6bcace5 2021-06-22 stsp 		return;
 135 e6bcace5 2021-06-22 stsp 	free(meta->path);
 136 e6bcace5 2021-06-22 stsp 	meta->path = NULL;
 137 5060d5a1 2022-01-10 stsp 	free(meta->delta_buf);
 138 5060d5a1 2022-01-10 stsp 	meta->delta_buf = NULL;
 139 67fd6849 2022-02-13 stsp 	free(meta->base_obj_id);
 140 67fd6849 2022-02-13 stsp 	meta->base_obj_id = NULL;
 141 e6bcace5 2021-06-22 stsp }
 142 e6bcace5 2021-06-22 stsp 
 143 e6bcace5 2021-06-22 stsp static void
 144 e6bcace5 2021-06-22 stsp free_nmeta(struct got_pack_meta **meta, int nmeta)
 145 e6bcace5 2021-06-22 stsp {
 146 e6bcace5 2021-06-22 stsp 	int i;
 147 e6bcace5 2021-06-22 stsp 
 148 e6bcace5 2021-06-22 stsp 	for (i = 0; i < nmeta; i++)
 149 e6bcace5 2021-06-22 stsp 		clear_meta(meta[i]);
 150 e6bcace5 2021-06-22 stsp 	free(meta);
 151 e6bcace5 2021-06-22 stsp }
 152 e6bcace5 2021-06-22 stsp 
 153 e6bcace5 2021-06-22 stsp static int
 154 e6bcace5 2021-06-22 stsp delta_order_cmp(const void *pa, const void *pb)
 155 e6bcace5 2021-06-22 stsp {
 156 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *a, *b;
 157 e6bcace5 2021-06-22 stsp 	int cmp;
 158 e6bcace5 2021-06-22 stsp 
 159 e6bcace5 2021-06-22 stsp 	a = *(struct got_pack_meta **)pa;
 160 e6bcace5 2021-06-22 stsp 	b = *(struct got_pack_meta **)pb;
 161 e6bcace5 2021-06-22 stsp 
 162 e6bcace5 2021-06-22 stsp 	if (a->obj_type != b->obj_type)
 163 e6bcace5 2021-06-22 stsp 		return a->obj_type - b->obj_type;
 164 e6bcace5 2021-06-22 stsp 	cmp = strcmp(a->path, b->path);
 165 e6bcace5 2021-06-22 stsp 	if (cmp != 0)
 166 e6bcace5 2021-06-22 stsp 		return cmp;
 167 611e8e31 2022-05-01 stsp 	if (a->mtime < b->mtime)
 168 611e8e31 2022-05-01 stsp 		return -1;
 169 611e8e31 2022-05-01 stsp 	if (a->mtime > b->mtime)
 170 611e8e31 2022-05-01 stsp 		return 1;
 171 e6bcace5 2021-06-22 stsp 	return got_object_id_cmp(&a->id, &b->id);
 172 e6bcace5 2021-06-22 stsp }
 173 e6bcace5 2021-06-22 stsp 
 174 5060d5a1 2022-01-10 stsp static off_t
 175 e6bcace5 2021-06-22 stsp delta_size(struct got_delta_instruction *deltas, int ndeltas)
 176 e6bcace5 2021-06-22 stsp {
 177 5060d5a1 2022-01-10 stsp 	int i;
 178 5060d5a1 2022-01-10 stsp 	off_t size = 32;
 179 e6bcace5 2021-06-22 stsp 	for (i = 0; i < ndeltas; i++) {
 180 e6bcace5 2021-06-22 stsp 		if (deltas[i].copy)
 181 e6bcace5 2021-06-22 stsp 			size += GOT_DELTA_SIZE_SHIFT;
 182 e6bcace5 2021-06-22 stsp 		else
 183 e6bcace5 2021-06-22 stsp 			size += deltas[i].len + 1;
 184 e6bcace5 2021-06-22 stsp 	}
 185 e6bcace5 2021-06-22 stsp 	return size;
 186 5060d5a1 2022-01-10 stsp }
 187 5060d5a1 2022-01-10 stsp 
 188 5060d5a1 2022-01-10 stsp static const struct got_error *
 189 5060d5a1 2022-01-10 stsp append(unsigned char **p, size_t *len, off_t *sz, void *seg, int nseg)
 190 5060d5a1 2022-01-10 stsp {
 191 5060d5a1 2022-01-10 stsp 	char *n;
 192 5060d5a1 2022-01-10 stsp 
 193 5060d5a1 2022-01-10 stsp 	if (*len + nseg >= *sz) {
 194 5060d5a1 2022-01-10 stsp 		while (*len + nseg >= *sz)
 195 5060d5a1 2022-01-10 stsp 			*sz += *sz / 2;
 196 5060d5a1 2022-01-10 stsp 		n = realloc(*p, *sz);
 197 5060d5a1 2022-01-10 stsp 		if (n == NULL)
 198 5060d5a1 2022-01-10 stsp 			return got_error_from_errno("realloc");
 199 5060d5a1 2022-01-10 stsp 		*p = n;
 200 5060d5a1 2022-01-10 stsp 	}
 201 5060d5a1 2022-01-10 stsp 	memcpy(*p + *len, seg, nseg);
 202 5060d5a1 2022-01-10 stsp 	*len += nseg;
 203 5060d5a1 2022-01-10 stsp 	return NULL;
 204 5060d5a1 2022-01-10 stsp }
 205 5060d5a1 2022-01-10 stsp 
 206 5060d5a1 2022-01-10 stsp static const struct got_error *
 207 5060d5a1 2022-01-10 stsp encode_delta_in_mem(struct got_pack_meta *m, struct got_raw_object *o,
 208 5060d5a1 2022-01-10 stsp     struct got_delta_instruction *deltas, int ndeltas,
 209 5060d5a1 2022-01-10 stsp     off_t delta_size, off_t base_size)
 210 5060d5a1 2022-01-10 stsp {
 211 5060d5a1 2022-01-10 stsp 	const struct got_error *err;
 212 5060d5a1 2022-01-10 stsp 	unsigned char buf[16], *bp;
 213 5060d5a1 2022-01-10 stsp 	int i, j;
 214 2d9e6abf 2022-05-04 stsp 	size_t len = 0, compressed_len;
 215 2d9e6abf 2022-05-04 stsp 	off_t bufsize = delta_size;
 216 5060d5a1 2022-01-10 stsp 	off_t n;
 217 5060d5a1 2022-01-10 stsp 	struct got_delta_instruction *d;
 218 2d9e6abf 2022-05-04 stsp 	uint8_t *delta_buf;
 219 5060d5a1 2022-01-10 stsp 
 220 2d9e6abf 2022-05-04 stsp 	delta_buf = malloc(bufsize);
 221 2d9e6abf 2022-05-04 stsp 	if (delta_buf == NULL)
 222 2d9e6abf 2022-05-04 stsp 		return got_error_from_errno("malloc");
 223 5060d5a1 2022-01-10 stsp 
 224 5060d5a1 2022-01-10 stsp 	/* base object size */
 225 5060d5a1 2022-01-10 stsp 	buf[0] = base_size & GOT_DELTA_SIZE_VAL_MASK;
 226 5060d5a1 2022-01-10 stsp 	n = base_size >> GOT_DELTA_SIZE_SHIFT;
 227 5060d5a1 2022-01-10 stsp 	for (i = 1; n > 0; i++) {
 228 5060d5a1 2022-01-10 stsp 		buf[i - 1] |= GOT_DELTA_SIZE_MORE;
 229 5060d5a1 2022-01-10 stsp 		buf[i] = n & GOT_DELTA_SIZE_VAL_MASK;
 230 5060d5a1 2022-01-10 stsp 		n >>= GOT_DELTA_SIZE_SHIFT;
 231 5060d5a1 2022-01-10 stsp 	}
 232 2d9e6abf 2022-05-04 stsp 	err = append(&delta_buf, &len, &bufsize, buf, i);
 233 5060d5a1 2022-01-10 stsp 	if (err)
 234 2d9e6abf 2022-05-04 stsp 		goto done;
 235 5060d5a1 2022-01-10 stsp 
 236 5060d5a1 2022-01-10 stsp 	/* target object size */
 237 5060d5a1 2022-01-10 stsp 	buf[0] = o->size & GOT_DELTA_SIZE_VAL_MASK;
 238 5060d5a1 2022-01-10 stsp 	n = o->size >> GOT_DELTA_SIZE_SHIFT;
 239 5060d5a1 2022-01-10 stsp 	for (i = 1; n > 0; i++) {
 240 5060d5a1 2022-01-10 stsp 		buf[i - 1] |= GOT_DELTA_SIZE_MORE;
 241 5060d5a1 2022-01-10 stsp 		buf[i] = n & GOT_DELTA_SIZE_VAL_MASK;
 242 5060d5a1 2022-01-10 stsp 		n >>= GOT_DELTA_SIZE_SHIFT;
 243 5060d5a1 2022-01-10 stsp 	}
 244 2d9e6abf 2022-05-04 stsp 	err = append(&delta_buf, &len, &bufsize, buf, i);
 245 5060d5a1 2022-01-10 stsp 	if (err)
 246 2d9e6abf 2022-05-04 stsp 		goto done;
 247 5060d5a1 2022-01-10 stsp 
 248 5060d5a1 2022-01-10 stsp 	for (j = 0; j < ndeltas; j++) {
 249 5060d5a1 2022-01-10 stsp 		d = &deltas[j];
 250 5060d5a1 2022-01-10 stsp 		if (d->copy) {
 251 5060d5a1 2022-01-10 stsp 			n = d->offset;
 252 5060d5a1 2022-01-10 stsp 			bp = &buf[1];
 253 5060d5a1 2022-01-10 stsp 			buf[0] = GOT_DELTA_BASE_COPY;
 254 5060d5a1 2022-01-10 stsp 			for (i = 0; i < 4; i++) {
 255 5060d5a1 2022-01-10 stsp 				/* DELTA_COPY_OFF1 ... DELTA_COPY_OFF4 */
 256 5060d5a1 2022-01-10 stsp 				buf[0] |= 1 << i;
 257 5060d5a1 2022-01-10 stsp 				*bp++ = n & 0xff;
 258 5060d5a1 2022-01-10 stsp 				n >>= 8;
 259 5060d5a1 2022-01-10 stsp 				if (n == 0)
 260 5060d5a1 2022-01-10 stsp 					break;
 261 5060d5a1 2022-01-10 stsp 			}
 262 5060d5a1 2022-01-10 stsp 
 263 5060d5a1 2022-01-10 stsp 			n = d->len;
 264 5060d5a1 2022-01-10 stsp 			if (n != GOT_DELTA_COPY_DEFAULT_LEN) {
 265 5060d5a1 2022-01-10 stsp 				/* DELTA_COPY_LEN1 ... DELTA_COPY_LEN3 */
 266 5060d5a1 2022-01-10 stsp 				for (i = 0; i < 3 && n > 0; i++) {
 267 5060d5a1 2022-01-10 stsp 					buf[0] |= 1 << (i + 4);
 268 5060d5a1 2022-01-10 stsp 					*bp++ = n & 0xff;
 269 5060d5a1 2022-01-10 stsp 					n >>= 8;
 270 5060d5a1 2022-01-10 stsp 				}
 271 5060d5a1 2022-01-10 stsp 			}
 272 2d9e6abf 2022-05-04 stsp 			err = append(&delta_buf, &len, &bufsize,
 273 5060d5a1 2022-01-10 stsp 			    buf, bp - buf);
 274 5060d5a1 2022-01-10 stsp 			if (err)
 275 2d9e6abf 2022-05-04 stsp 				goto done;
 276 5060d5a1 2022-01-10 stsp 		} else if (o->f == NULL) {
 277 5060d5a1 2022-01-10 stsp 			n = 0;
 278 5060d5a1 2022-01-10 stsp 			while (n != d->len) {
 279 5060d5a1 2022-01-10 stsp 				buf[0] = (d->len - n < 127) ? d->len - n : 127;
 280 2d9e6abf 2022-05-04 stsp 				err = append(&delta_buf, &len, &bufsize,
 281 5060d5a1 2022-01-10 stsp 				    buf, 1);
 282 5060d5a1 2022-01-10 stsp 				if (err)
 283 2d9e6abf 2022-05-04 stsp 					goto done;
 284 2d9e6abf 2022-05-04 stsp 				err = append(&delta_buf, &len, &bufsize,
 285 5060d5a1 2022-01-10 stsp 				    o->data + o->hdrlen + d->offset + n,
 286 5060d5a1 2022-01-10 stsp 				    buf[0]);
 287 5060d5a1 2022-01-10 stsp 				if (err)
 288 2d9e6abf 2022-05-04 stsp 					goto done;
 289 5060d5a1 2022-01-10 stsp 				n += buf[0];
 290 5060d5a1 2022-01-10 stsp 			}
 291 5060d5a1 2022-01-10 stsp 		} else {
 292 5060d5a1 2022-01-10 stsp 			char content[128];
 293 5060d5a1 2022-01-10 stsp 			size_t r;
 294 2d9e6abf 2022-05-04 stsp 			if (fseeko(o->f, o->hdrlen + d->offset, SEEK_SET) == -1) {
 295 2d9e6abf 2022-05-04 stsp 				err = got_error_from_errno("fseeko");
 296 2d9e6abf 2022-05-04 stsp 				goto done;
 297 2d9e6abf 2022-05-04 stsp 			}
 298 5060d5a1 2022-01-10 stsp 			n = 0;
 299 5060d5a1 2022-01-10 stsp 			while (n != d->len) {
 300 5060d5a1 2022-01-10 stsp 				buf[0] = (d->len - n < 127) ? d->len - n : 127;
 301 2d9e6abf 2022-05-04 stsp 				err = append(&delta_buf, &len, &bufsize,
 302 5060d5a1 2022-01-10 stsp 				    buf, 1);
 303 5060d5a1 2022-01-10 stsp 				if (err)
 304 2d9e6abf 2022-05-04 stsp 					goto done;
 305 5060d5a1 2022-01-10 stsp 				r = fread(content, 1, buf[0], o->f);
 306 2d9e6abf 2022-05-04 stsp 				if (r != buf[0]) {
 307 2d9e6abf 2022-05-04 stsp 					err = got_ferror(o->f, GOT_ERR_IO);
 308 2d9e6abf 2022-05-04 stsp 					goto done;
 309 2d9e6abf 2022-05-04 stsp 				}
 310 2d9e6abf 2022-05-04 stsp 				err = append(&delta_buf, &len, &bufsize,
 311 5060d5a1 2022-01-10 stsp 				    content, buf[0]);
 312 5060d5a1 2022-01-10 stsp 				if (err)
 313 2d9e6abf 2022-05-04 stsp 					goto done;
 314 5060d5a1 2022-01-10 stsp 				n += buf[0];
 315 5060d5a1 2022-01-10 stsp 			}
 316 5060d5a1 2022-01-10 stsp 		}
 317 5060d5a1 2022-01-10 stsp 	}
 318 5060d5a1 2022-01-10 stsp 
 319 2d9e6abf 2022-05-04 stsp 	err = got_deflate_to_mem_mmap(&m->delta_buf, &compressed_len,
 320 2d9e6abf 2022-05-04 stsp 	    NULL, NULL, delta_buf, 0, len);
 321 2d9e6abf 2022-05-04 stsp 	if (err)
 322 2d9e6abf 2022-05-04 stsp 		goto done;
 323 2d9e6abf 2022-05-04 stsp 
 324 5060d5a1 2022-01-10 stsp 	m->delta_len = len;
 325 2d9e6abf 2022-05-04 stsp 	m->delta_compressed_len = compressed_len;
 326 2d9e6abf 2022-05-04 stsp done:
 327 2d9e6abf 2022-05-04 stsp 	free(delta_buf);
 328 2d9e6abf 2022-05-04 stsp 	return err;
 329 e6bcace5 2021-06-22 stsp }
 330 e6bcace5 2021-06-22 stsp 
 331 74881701 2021-10-15 stsp static const struct got_error *
 332 74881701 2021-10-15 stsp encode_delta(struct got_pack_meta *m, struct got_raw_object *o,
 333 74881701 2021-10-15 stsp     struct got_delta_instruction *deltas, int ndeltas,
 334 a319ca8c 2021-10-15 stsp     off_t base_size, FILE *f)
 335 a319ca8c 2021-10-15 stsp {
 336 2d9e6abf 2022-05-04 stsp 	const struct got_error *err;
 337 a319ca8c 2021-10-15 stsp 	unsigned char buf[16], *bp;
 338 a319ca8c 2021-10-15 stsp 	int i, j;
 339 a319ca8c 2021-10-15 stsp 	off_t n;
 340 2d9e6abf 2022-05-04 stsp 	struct got_deflate_buf zb;
 341 a319ca8c 2021-10-15 stsp 	struct got_delta_instruction *d;
 342 2d9e6abf 2022-05-04 stsp 	off_t delta_len = 0, compressed_len = 0;
 343 a319ca8c 2021-10-15 stsp 
 344 2d9e6abf 2022-05-04 stsp 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
 345 2d9e6abf 2022-05-04 stsp 	if (err)
 346 2d9e6abf 2022-05-04 stsp 		return err;
 347 2d9e6abf 2022-05-04 stsp 
 348 a319ca8c 2021-10-15 stsp 	/* base object size */
 349 a319ca8c 2021-10-15 stsp 	buf[0] = base_size & GOT_DELTA_SIZE_VAL_MASK;
 350 a319ca8c 2021-10-15 stsp 	n = base_size >> GOT_DELTA_SIZE_SHIFT;
 351 a319ca8c 2021-10-15 stsp 	for (i = 1; n > 0; i++) {
 352 a319ca8c 2021-10-15 stsp 		buf[i - 1] |= GOT_DELTA_SIZE_MORE;
 353 a319ca8c 2021-10-15 stsp 		buf[i] = n & GOT_DELTA_SIZE_VAL_MASK;
 354 a319ca8c 2021-10-15 stsp 		n >>= GOT_DELTA_SIZE_SHIFT;
 355 a319ca8c 2021-10-15 stsp 	}
 356 a319ca8c 2021-10-15 stsp 
 357 2d9e6abf 2022-05-04 stsp 	err = got_deflate_append_to_file_mmap(&zb, &compressed_len,
 358 2d9e6abf 2022-05-04 stsp 	    buf, 0, i, f, NULL);
 359 2d9e6abf 2022-05-04 stsp 	if (err)
 360 2d9e6abf 2022-05-04 stsp 		goto done;
 361 2d9e6abf 2022-05-04 stsp 	delta_len += i;
 362 2d9e6abf 2022-05-04 stsp 
 363 a319ca8c 2021-10-15 stsp 	/* target object size */
 364 a319ca8c 2021-10-15 stsp 	buf[0] = o->size & GOT_DELTA_SIZE_VAL_MASK;
 365 a319ca8c 2021-10-15 stsp 	n = o->size >> GOT_DELTA_SIZE_SHIFT;
 366 a319ca8c 2021-10-15 stsp 	for (i = 1; n > 0; i++) {
 367 a319ca8c 2021-10-15 stsp 		buf[i - 1] |= GOT_DELTA_SIZE_MORE;
 368 a319ca8c 2021-10-15 stsp 		buf[i] = n & GOT_DELTA_SIZE_VAL_MASK;
 369 a319ca8c 2021-10-15 stsp 		n >>= GOT_DELTA_SIZE_SHIFT;
 370 a319ca8c 2021-10-15 stsp 	}
 371 a319ca8c 2021-10-15 stsp 
 372 2d9e6abf 2022-05-04 stsp 	err = got_deflate_append_to_file_mmap(&zb, &compressed_len,
 373 2d9e6abf 2022-05-04 stsp 	    buf, 0, i, f, NULL);
 374 2d9e6abf 2022-05-04 stsp 	if (err)
 375 2d9e6abf 2022-05-04 stsp 		goto done;
 376 2d9e6abf 2022-05-04 stsp 	delta_len += i;
 377 2d9e6abf 2022-05-04 stsp 
 378 a319ca8c 2021-10-15 stsp 	for (j = 0; j < ndeltas; j++) {
 379 a319ca8c 2021-10-15 stsp 		d = &deltas[j];
 380 a319ca8c 2021-10-15 stsp 		if (d->copy) {
 381 a319ca8c 2021-10-15 stsp 			n = d->offset;
 382 a319ca8c 2021-10-15 stsp 			bp = &buf[1];
 383 a319ca8c 2021-10-15 stsp 			buf[0] = GOT_DELTA_BASE_COPY;
 384 a319ca8c 2021-10-15 stsp 			for (i = 0; i < 4; i++) {
 385 a319ca8c 2021-10-15 stsp 				/* DELTA_COPY_OFF1 ... DELTA_COPY_OFF4 */
 386 a319ca8c 2021-10-15 stsp 				buf[0] |= 1 << i;
 387 a319ca8c 2021-10-15 stsp 				*bp++ = n & 0xff;
 388 a319ca8c 2021-10-15 stsp 				n >>= 8;
 389 a319ca8c 2021-10-15 stsp 				if (n == 0)
 390 a319ca8c 2021-10-15 stsp 					break;
 391 a319ca8c 2021-10-15 stsp 			}
 392 a319ca8c 2021-10-15 stsp 			n = d->len;
 393 a319ca8c 2021-10-15 stsp 			if (n != GOT_DELTA_COPY_DEFAULT_LEN) {
 394 a319ca8c 2021-10-15 stsp 				/* DELTA_COPY_LEN1 ... DELTA_COPY_LEN3 */
 395 a319ca8c 2021-10-15 stsp 				for (i = 0; i < 3 && n > 0; i++) {
 396 a319ca8c 2021-10-15 stsp 					buf[0] |= 1 << (i + 4);
 397 a319ca8c 2021-10-15 stsp 					*bp++ = n & 0xff;
 398 a319ca8c 2021-10-15 stsp 					n >>= 8;
 399 a319ca8c 2021-10-15 stsp 				}
 400 a319ca8c 2021-10-15 stsp 			}
 401 2d9e6abf 2022-05-04 stsp 			err = got_deflate_append_to_file_mmap(&zb,
 402 2d9e6abf 2022-05-04 stsp 			    &compressed_len, buf, 0, bp - buf, f, NULL);
 403 2d9e6abf 2022-05-04 stsp 			if (err)
 404 2d9e6abf 2022-05-04 stsp 				goto done;
 405 2d9e6abf 2022-05-04 stsp 			delta_len += (bp - buf);
 406 64a8571e 2022-01-07 stsp 		} else if (o->f == NULL) {
 407 64a8571e 2022-01-07 stsp 			n = 0;
 408 64a8571e 2022-01-07 stsp 			while (n != d->len) {
 409 64a8571e 2022-01-07 stsp 				buf[0] = (d->len - n < 127) ? d->len - n : 127;
 410 2d9e6abf 2022-05-04 stsp 				err = got_deflate_append_to_file_mmap(&zb,
 411 2d9e6abf 2022-05-04 stsp 				    &compressed_len, buf, 0, 1, f, NULL);
 412 2d9e6abf 2022-05-04 stsp 				if (err)
 413 2d9e6abf 2022-05-04 stsp 					goto done;
 414 2d9e6abf 2022-05-04 stsp 				delta_len++;
 415 2d9e6abf 2022-05-04 stsp 				err = got_deflate_append_to_file_mmap(&zb,
 416 2d9e6abf 2022-05-04 stsp 				    &compressed_len,
 417 2d9e6abf 2022-05-04 stsp 				    o->data + o->hdrlen + d->offset + n, 0,
 418 2d9e6abf 2022-05-04 stsp 				    buf[0], f, NULL);
 419 2d9e6abf 2022-05-04 stsp 				if (err)
 420 2d9e6abf 2022-05-04 stsp 					goto done;
 421 2d9e6abf 2022-05-04 stsp 				delta_len += buf[0];
 422 64a8571e 2022-01-07 stsp 				n += buf[0];
 423 64a8571e 2022-01-07 stsp 			}
 424 a319ca8c 2021-10-15 stsp 		} else {
 425 a319ca8c 2021-10-15 stsp 			char content[128];
 426 a319ca8c 2021-10-15 stsp 			size_t r;
 427 2d9e6abf 2022-05-04 stsp 			if (fseeko(o->f, o->hdrlen + d->offset, SEEK_SET) == -1) {
 428 2d9e6abf 2022-05-04 stsp 				err = got_error_from_errno("fseeko");
 429 2d9e6abf 2022-05-04 stsp 				goto done;
 430 2d9e6abf 2022-05-04 stsp 			}
 431 a319ca8c 2021-10-15 stsp 			n = 0;
 432 a319ca8c 2021-10-15 stsp 			while (n != d->len) {
 433 a319ca8c 2021-10-15 stsp 				buf[0] = (d->len - n < 127) ? d->len - n : 127;
 434 2d9e6abf 2022-05-04 stsp 				err = got_deflate_append_to_file_mmap(&zb,
 435 2d9e6abf 2022-05-04 stsp 				    &compressed_len, buf, 0, 1, f, NULL);
 436 2d9e6abf 2022-05-04 stsp 				if (err)
 437 2d9e6abf 2022-05-04 stsp 					goto done;
 438 2d9e6abf 2022-05-04 stsp 				delta_len++;
 439 a319ca8c 2021-10-15 stsp 				r = fread(content, 1, buf[0], o->f);
 440 2d9e6abf 2022-05-04 stsp 				if (r != buf[0]) {
 441 2d9e6abf 2022-05-04 stsp 					err = got_ferror(o->f, GOT_ERR_IO);
 442 2d9e6abf 2022-05-04 stsp 					goto done;
 443 2d9e6abf 2022-05-04 stsp 				}
 444 2d9e6abf 2022-05-04 stsp 				err = got_deflate_append_to_file_mmap(&zb,
 445 2d9e6abf 2022-05-04 stsp 				    &compressed_len, content, 0, buf[0], f,
 446 2d9e6abf 2022-05-04 stsp 				    NULL);
 447 2d9e6abf 2022-05-04 stsp 				if (err)
 448 2d9e6abf 2022-05-04 stsp 					goto done;
 449 2d9e6abf 2022-05-04 stsp 				delta_len += buf[0];
 450 a319ca8c 2021-10-15 stsp 				n += buf[0];
 451 a319ca8c 2021-10-15 stsp 			}
 452 a319ca8c 2021-10-15 stsp 		}
 453 a319ca8c 2021-10-15 stsp 	}
 454 e6bcace5 2021-06-22 stsp 
 455 2d9e6abf 2022-05-04 stsp 	err = got_deflate_flush(&zb, f, NULL, &compressed_len);
 456 2d9e6abf 2022-05-04 stsp 	if (err)
 457 2d9e6abf 2022-05-04 stsp 		goto done;
 458 2d9e6abf 2022-05-04 stsp 
 459 2d9e6abf 2022-05-04 stsp 	/* sanity check */
 460 2d9e6abf 2022-05-04 stsp 	if (compressed_len != ftello(f) - m->delta_offset) {
 461 2d9e6abf 2022-05-04 stsp 		err = got_error(GOT_ERR_COMPRESSION);
 462 2d9e6abf 2022-05-04 stsp 		goto done;
 463 2d9e6abf 2022-05-04 stsp 	}
 464 2d9e6abf 2022-05-04 stsp 
 465 2d9e6abf 2022-05-04 stsp 	m->delta_len = delta_len;
 466 2d9e6abf 2022-05-04 stsp 	m->delta_compressed_len = compressed_len;
 467 2d9e6abf 2022-05-04 stsp done:
 468 2d9e6abf 2022-05-04 stsp 	got_deflate_end(&zb);
 469 2d9e6abf 2022-05-04 stsp 	return err;
 470 a319ca8c 2021-10-15 stsp }
 471 211cfef0 2022-01-05 stsp 
 472 211cfef0 2022-01-05 stsp static const struct got_error *
 473 211cfef0 2022-01-05 stsp report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
 474 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, int ncolored, int nfound, int ntrees,
 475 b8af7c06 2022-03-15 stsp     off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
 476 b8af7c06 2022-03-15 stsp     int nobj_written)
 477 211cfef0 2022-01-05 stsp {
 478 211cfef0 2022-01-05 stsp 	const struct got_error *err;
 479 211cfef0 2022-01-05 stsp 	int elapsed;
 480 211cfef0 2022-01-05 stsp 
 481 211cfef0 2022-01-05 stsp 	if (progress_cb == NULL)
 482 211cfef0 2022-01-05 stsp 		return NULL;
 483 211cfef0 2022-01-05 stsp 
 484 211cfef0 2022-01-05 stsp 	err = got_ratelimit_check(&elapsed, rl);
 485 211cfef0 2022-01-05 stsp 	if (err || !elapsed)
 486 211cfef0 2022-01-05 stsp 		return err;
 487 a319ca8c 2021-10-15 stsp 
 488 b8af7c06 2022-03-15 stsp 	return progress_cb(progress_arg, ncolored, nfound, ntrees,
 489 b8af7c06 2022-03-15 stsp 	    packfile_size, ncommits, nobj_total, obj_deltify, nobj_written);
 490 211cfef0 2022-01-05 stsp }
 491 a319ca8c 2021-10-15 stsp 
 492 e6bcace5 2021-06-22 stsp static const struct got_error *
 493 67fd6849 2022-02-13 stsp add_meta(struct got_pack_meta *m, struct got_pack_metavec *v)
 494 67fd6849 2022-02-13 stsp {
 495 67fd6849 2022-02-13 stsp 	if (v->nmeta == v->metasz){
 496 67fd6849 2022-02-13 stsp 		size_t newsize = 2 * v->metasz;
 497 67fd6849 2022-02-13 stsp 		struct got_pack_meta **new;
 498 67fd6849 2022-02-13 stsp 		new = reallocarray(v->meta, newsize, sizeof(*new));
 499 67fd6849 2022-02-13 stsp 		if (new == NULL)
 500 67fd6849 2022-02-13 stsp 			return got_error_from_errno("reallocarray");
 501 67fd6849 2022-02-13 stsp 		v->meta = new;
 502 67fd6849 2022-02-13 stsp 		v->metasz = newsize; 
 503 67fd6849 2022-02-13 stsp 	}
 504 67fd6849 2022-02-13 stsp 
 505 67fd6849 2022-02-13 stsp 	v->meta[v->nmeta++] = m;
 506 67fd6849 2022-02-13 stsp 	return NULL;
 507 67fd6849 2022-02-13 stsp }
 508 67fd6849 2022-02-13 stsp 
 509 67fd6849 2022-02-13 stsp static const struct got_error *
 510 67fd6849 2022-02-13 stsp find_pack_for_reuse(struct got_packidx **best_packidx,
 511 67fd6849 2022-02-13 stsp     struct got_repository *repo)
 512 67fd6849 2022-02-13 stsp {
 513 9b576444 2022-03-14 stsp 	const struct got_error *err = NULL;
 514 67fd6849 2022-02-13 stsp 	struct got_pathlist_entry *pe;
 515 67fd6849 2022-02-13 stsp 	const char *best_packidx_path = NULL;
 516 67fd6849 2022-02-13 stsp 	int nobj_max = 0;
 517 67fd6849 2022-02-13 stsp 
 518 67fd6849 2022-02-13 stsp 	*best_packidx = NULL;
 519 67fd6849 2022-02-13 stsp 
 520 9b576444 2022-03-14 stsp 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
 521 67fd6849 2022-02-13 stsp 		const char *path_packidx = pe->path;
 522 67fd6849 2022-02-13 stsp 		struct got_packidx *packidx;
 523 67fd6849 2022-02-13 stsp 		int nobj;
 524 67fd6849 2022-02-13 stsp 
 525 67fd6849 2022-02-13 stsp 		err = got_repo_get_packidx(&packidx, path_packidx, repo);
 526 67fd6849 2022-02-13 stsp 		if (err)
 527 67fd6849 2022-02-13 stsp 			break;
 528 67fd6849 2022-02-13 stsp 
 529 67fd6849 2022-02-13 stsp 		nobj = be32toh(packidx->hdr.fanout_table[0xff]);
 530 67fd6849 2022-02-13 stsp 		if (nobj > nobj_max) {
 531 67fd6849 2022-02-13 stsp 			best_packidx_path = path_packidx;
 532 67fd6849 2022-02-13 stsp 			nobj_max = nobj;
 533 67fd6849 2022-02-13 stsp 		}
 534 67fd6849 2022-02-13 stsp 	}
 535 67fd6849 2022-02-13 stsp 
 536 67fd6849 2022-02-13 stsp 	if (best_packidx_path) {
 537 67fd6849 2022-02-13 stsp 		err = got_repo_get_packidx(best_packidx, best_packidx_path,
 538 67fd6849 2022-02-13 stsp 		    repo);
 539 67fd6849 2022-02-13 stsp 	}
 540 67fd6849 2022-02-13 stsp 
 541 67fd6849 2022-02-13 stsp 	return err;
 542 67fd6849 2022-02-13 stsp }
 543 67fd6849 2022-02-13 stsp 
 544 fae7e038 2022-05-07 stsp struct send_id_arg {
 545 fae7e038 2022-05-07 stsp 	struct imsgbuf *ibuf;
 546 fae7e038 2022-05-07 stsp 	struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
 547 fae7e038 2022-05-07 stsp 	size_t nids;
 548 67fd6849 2022-02-13 stsp };
 549 67fd6849 2022-02-13 stsp 
 550 67fd6849 2022-02-13 stsp static const struct got_error *
 551 fae7e038 2022-05-07 stsp send_id(struct got_object_id *id, void *data, void *arg)
 552 67fd6849 2022-02-13 stsp {
 553 fae7e038 2022-05-07 stsp 	const struct got_error *err = NULL;
 554 fae7e038 2022-05-07 stsp 	struct send_id_arg *a = arg;
 555 67fd6849 2022-02-13 stsp 
 556 fae7e038 2022-05-07 stsp 	a->ids[a->nids++] = id;
 557 fae7e038 2022-05-07 stsp 
 558 fae7e038 2022-05-07 stsp 	if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
 559 fae7e038 2022-05-07 stsp 		err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
 560 67fd6849 2022-02-13 stsp 		if (err)
 561 67fd6849 2022-02-13 stsp 			return err;
 562 fae7e038 2022-05-07 stsp 		a->nids = 0;
 563 67fd6849 2022-02-13 stsp 	}
 564 67fd6849 2022-02-13 stsp 
 565 fae7e038 2022-05-07 stsp 	return NULL;
 566 fae7e038 2022-05-07 stsp }
 567 67fd6849 2022-02-13 stsp 
 568 fae7e038 2022-05-07 stsp static const struct got_error *
 569 fae7e038 2022-05-07 stsp recv_reused_delta(struct got_imsg_reused_delta *delta,
 570 fae7e038 2022-05-07 stsp     struct got_object_idset *idset, struct got_pack_metavec *v)
 571 fae7e038 2022-05-07 stsp {
 572 fae7e038 2022-05-07 stsp 	struct got_pack_meta *m, *base;
 573 67fd6849 2022-02-13 stsp 
 574 fae7e038 2022-05-07 stsp 	if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
 575 fae7e038 2022-05-07 stsp 	    delta->delta_offset +
 576 fae7e038 2022-05-07 stsp 	    delta->delta_compressed_size < delta->delta_offset)
 577 fae7e038 2022-05-07 stsp 		return got_error(GOT_ERR_BAD_PACKFILE);
 578 67fd6849 2022-02-13 stsp 
 579 fae7e038 2022-05-07 stsp 	m = got_object_idset_get(idset, &delta->id);
 580 fae7e038 2022-05-07 stsp 	if (m == NULL)
 581 fae7e038 2022-05-07 stsp 		return got_error(GOT_ERR_NO_OBJ);
 582 fae7e038 2022-05-07 stsp 
 583 fae7e038 2022-05-07 stsp 	base = got_object_idset_get(idset, &delta->base_id);
 584 fae7e038 2022-05-07 stsp 	if (base == NULL)
 585 fae7e038 2022-05-07 stsp 		return got_error(GOT_ERR_NO_OBJ);
 586 fae7e038 2022-05-07 stsp 
 587 fae7e038 2022-05-07 stsp 	m->delta_len = delta->delta_size;
 588 fae7e038 2022-05-07 stsp 	m->delta_compressed_len = delta->delta_compressed_size;
 589 fae7e038 2022-05-07 stsp 	m->delta_offset = delta->delta_out_offset;
 590 fae7e038 2022-05-07 stsp 	m->prev = base;
 591 fae7e038 2022-05-07 stsp 	m->size = delta->result_size;
 592 fae7e038 2022-05-07 stsp 	m->have_reused_delta = 1;
 593 fae7e038 2022-05-07 stsp 	m->reused_delta_offset = delta->delta_offset;
 594 fae7e038 2022-05-07 stsp 	m->base_obj_id = got_object_id_dup(&delta->base_id);
 595 fae7e038 2022-05-07 stsp 	if (m->base_obj_id == NULL)
 596 fae7e038 2022-05-07 stsp 		return got_error_from_errno("got_object_id_dup");
 597 fae7e038 2022-05-07 stsp 
 598 fae7e038 2022-05-07 stsp 	return add_meta(m, v);
 599 67fd6849 2022-02-13 stsp }
 600 67fd6849 2022-02-13 stsp 
 601 67fd6849 2022-02-13 stsp static const struct got_error *
 602 67fd6849 2022-02-13 stsp search_deltas(struct got_pack_metavec *v, struct got_object_idset *idset,
 603 b8af7c06 2022-03-15 stsp     int delta_cache_fd, int ncolored, int nfound, int ntrees, int ncommits,
 604 b8af7c06 2022-03-15 stsp     struct got_repository *repo,
 605 67fd6849 2022-02-13 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
 606 67fd6849 2022-02-13 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 607 67fd6849 2022-02-13 stsp {
 608 67fd6849 2022-02-13 stsp 	const struct got_error *err = NULL;
 609 67fd6849 2022-02-13 stsp 	struct got_packidx *packidx;
 610 67fd6849 2022-02-13 stsp 	struct got_pack *pack;
 611 fae7e038 2022-05-07 stsp 	struct send_id_arg sia;
 612 fae7e038 2022-05-07 stsp 	struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
 613 fae7e038 2022-05-07 stsp 	size_t ndeltas, i;
 614 67fd6849 2022-02-13 stsp 
 615 67fd6849 2022-02-13 stsp 	err = find_pack_for_reuse(&packidx, repo);
 616 67fd6849 2022-02-13 stsp 	if (err)
 617 67fd6849 2022-02-13 stsp 		return err;
 618 67fd6849 2022-02-13 stsp 
 619 67fd6849 2022-02-13 stsp 	if (packidx == NULL)
 620 67fd6849 2022-02-13 stsp 		return NULL;
 621 67fd6849 2022-02-13 stsp 
 622 fae7e038 2022-05-07 stsp 	err = got_object_prepare_delta_reuse(&pack, packidx,
 623 fae7e038 2022-05-07 stsp 	    delta_cache_fd, repo);
 624 67fd6849 2022-02-13 stsp 	if (err)
 625 67fd6849 2022-02-13 stsp 		return err;
 626 67fd6849 2022-02-13 stsp 
 627 fae7e038 2022-05-07 stsp 	memset(&sia, 0, sizeof(sia));
 628 fae7e038 2022-05-07 stsp 	sia.ibuf = pack->privsep_child->ibuf;
 629 fae7e038 2022-05-07 stsp 	err = got_object_idset_for_each(idset, send_id, &sia);
 630 fae7e038 2022-05-07 stsp 	if (err)
 631 fae7e038 2022-05-07 stsp 		return err;
 632 fae7e038 2022-05-07 stsp 	if (sia.nids > 0) {
 633 fae7e038 2022-05-07 stsp 		err = got_privsep_send_object_idlist(pack->privsep_child->ibuf,
 634 fae7e038 2022-05-07 stsp 		    sia.ids, sia.nids);
 635 fae7e038 2022-05-07 stsp 		if (err)
 636 fae7e038 2022-05-07 stsp 			return err;
 637 67fd6849 2022-02-13 stsp 	}
 638 fae7e038 2022-05-07 stsp 	err = got_privsep_send_object_idlist_done(pack->privsep_child->ibuf);
 639 fae7e038 2022-05-07 stsp 	if (err)
 640 fae7e038 2022-05-07 stsp 		return err;
 641 67fd6849 2022-02-13 stsp 
 642 fae7e038 2022-05-07 stsp 	for (;;) {
 643 fae7e038 2022-05-07 stsp 		int done = 0;
 644 fae7e038 2022-05-07 stsp 
 645 fae7e038 2022-05-07 stsp 		if (cancel_cb) {
 646 fae7e038 2022-05-07 stsp 			err = (*cancel_cb)(cancel_arg);
 647 fae7e038 2022-05-07 stsp 			if (err)
 648 fae7e038 2022-05-07 stsp 				break;
 649 fae7e038 2022-05-07 stsp 		}
 650 fae7e038 2022-05-07 stsp 
 651 fae7e038 2022-05-07 stsp 		err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
 652 fae7e038 2022-05-07 stsp 		    pack->privsep_child->ibuf);
 653 fae7e038 2022-05-07 stsp 		if (err || done)
 654 fae7e038 2022-05-07 stsp 			break;
 655 fae7e038 2022-05-07 stsp 
 656 fae7e038 2022-05-07 stsp 		for (i = 0; i < ndeltas; i++) {
 657 fae7e038 2022-05-07 stsp 			struct got_imsg_reused_delta *delta = &deltas[i];
 658 fae7e038 2022-05-07 stsp 			err = recv_reused_delta(delta, idset, v);
 659 fae7e038 2022-05-07 stsp 			if (err)
 660 fae7e038 2022-05-07 stsp 				goto done;
 661 fae7e038 2022-05-07 stsp 		}
 662 fae7e038 2022-05-07 stsp 
 663 fae7e038 2022-05-07 stsp 		err = report_progress(progress_cb, progress_arg, rl,
 664 fae7e038 2022-05-07 stsp 		    ncolored, nfound, ntrees, 0L, ncommits,
 665 fae7e038 2022-05-07 stsp 		    got_object_idset_num_elements(idset), v->nmeta, 0);
 666 fae7e038 2022-05-07 stsp 		if (err)
 667 fae7e038 2022-05-07 stsp 			break;
 668 fae7e038 2022-05-07 stsp 	}
 669 67fd6849 2022-02-13 stsp done:
 670 67fd6849 2022-02-13 stsp 	return err;
 671 67fd6849 2022-02-13 stsp }
 672 67fd6849 2022-02-13 stsp 
 673 67fd6849 2022-02-13 stsp static const struct got_error *
 674 b8af7c06 2022-03-15 stsp pick_deltas(struct got_pack_meta **meta, int nmeta, int ncolored,
 675 b8af7c06 2022-03-15 stsp     int nfound, int ntrees, int ncommits, int nreused, FILE *delta_cache,
 676 b8af7c06 2022-03-15 stsp     struct got_repository *repo,
 677 05118f5a 2021-06-22 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
 678 211cfef0 2022-01-05 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 679 e6bcace5 2021-06-22 stsp {
 680 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
 681 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *m = NULL, *base = NULL;
 682 e6bcace5 2021-06-22 stsp 	struct got_raw_object *raw = NULL, *base_raw = NULL;
 683 74881701 2021-10-15 stsp 	struct got_delta_instruction *deltas = NULL, *best_deltas = NULL;
 684 5060d5a1 2022-01-10 stsp 	int i, j, ndeltas, best_ndeltas;
 685 5060d5a1 2022-01-10 stsp 	off_t size, best_size;
 686 4f4d853e 2021-10-24 stsp 	const int max_base_candidates = 3;
 687 402a5ec1 2022-01-10 stsp 	size_t delta_memsize = 0;
 688 402a5ec1 2022-01-10 stsp 	const size_t max_delta_memsize = 25 * GOT_DELTA_RESULT_SIZE_CACHED_MAX;
 689 cc7a354a 2021-10-15 stsp 	int outfd = -1;
 690 e6bcace5 2021-06-22 stsp 
 691 e6bcace5 2021-06-22 stsp 	qsort(meta, nmeta, sizeof(struct got_pack_meta *), delta_order_cmp);
 692 e6bcace5 2021-06-22 stsp 	for (i = 0; i < nmeta; i++) {
 693 e6bcace5 2021-06-22 stsp 		if (cancel_cb) {
 694 e6bcace5 2021-06-22 stsp 			err = (*cancel_cb)(cancel_arg);
 695 e6bcace5 2021-06-22 stsp 			if (err)
 696 e6bcace5 2021-06-22 stsp 				break;
 697 e6bcace5 2021-06-22 stsp 		}
 698 211cfef0 2022-01-05 stsp 		err = report_progress(progress_cb, progress_arg, rl,
 699 b8af7c06 2022-03-15 stsp 		    ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta,
 700 b8af7c06 2022-03-15 stsp 		    nreused + i, 0);
 701 211cfef0 2022-01-05 stsp 		if (err)
 702 211cfef0 2022-01-05 stsp 			goto done;
 703 e6bcace5 2021-06-22 stsp 		m = meta[i];
 704 e6bcace5 2021-06-22 stsp 
 705 e6bcace5 2021-06-22 stsp 		if (m->obj_type == GOT_OBJ_TYPE_COMMIT ||
 706 e6bcace5 2021-06-22 stsp 		    m->obj_type == GOT_OBJ_TYPE_TAG)
 707 e6bcace5 2021-06-22 stsp 			continue;
 708 e6bcace5 2021-06-22 stsp 
 709 94dac27c 2021-10-15 stsp 		err = got_object_raw_open(&raw, &outfd, repo, &m->id);
 710 e6bcace5 2021-06-22 stsp 		if (err)
 711 e6bcace5 2021-06-22 stsp 			goto done;
 712 600b755e 2021-10-14 stsp 		m->size = raw->size;
 713 e6bcace5 2021-06-22 stsp 
 714 64a8571e 2022-01-07 stsp 		if (raw->f == NULL) {
 715 64a8571e 2022-01-07 stsp 			err = got_deltify_init_mem(&m->dtab, raw->data,
 716 64a8571e 2022-01-07 stsp 			    raw->hdrlen, raw->size + raw->hdrlen);
 717 64a8571e 2022-01-07 stsp 		} else {
 718 64a8571e 2022-01-07 stsp 			err = got_deltify_init(&m->dtab, raw->f, raw->hdrlen,
 719 64a8571e 2022-01-07 stsp 			    raw->size + raw->hdrlen);
 720 64a8571e 2022-01-07 stsp 		}
 721 e6bcace5 2021-06-22 stsp 		if (err)
 722 e6bcace5 2021-06-22 stsp 			goto done;
 723 e6bcace5 2021-06-22 stsp 
 724 e6bcace5 2021-06-22 stsp 		if (i > max_base_candidates) {
 725 e6bcace5 2021-06-22 stsp 			struct got_pack_meta *n = NULL;
 726 e6bcace5 2021-06-22 stsp 			n = meta[i - (max_base_candidates + 1)];
 727 e6bcace5 2021-06-22 stsp 			got_deltify_free(n->dtab);
 728 e6bcace5 2021-06-22 stsp 			n->dtab = NULL;
 729 e6bcace5 2021-06-22 stsp 		}
 730 e6bcace5 2021-06-22 stsp 
 731 74881701 2021-10-15 stsp 		best_size = raw->size;
 732 74881701 2021-10-15 stsp 		best_ndeltas = 0;
 733 e6bcace5 2021-06-22 stsp 		for (j = MAX(0, i - max_base_candidates); j < i; j++) {
 734 e6bcace5 2021-06-22 stsp 			if (cancel_cb) {
 735 e6bcace5 2021-06-22 stsp 				err = (*cancel_cb)(cancel_arg);
 736 e6bcace5 2021-06-22 stsp 				if (err)
 737 e6bcace5 2021-06-22 stsp 					goto done;
 738 e6bcace5 2021-06-22 stsp 			}
 739 e6bcace5 2021-06-22 stsp 			base = meta[j];
 740 e6bcace5 2021-06-22 stsp 			/* long chains make unpacking slow, avoid such bases */
 741 22edbce7 2021-10-24 stsp 			if (base->nchain >= 128 ||
 742 e6bcace5 2021-06-22 stsp 			    base->obj_type != m->obj_type)
 743 e6bcace5 2021-06-22 stsp 				continue;
 744 e6bcace5 2021-06-22 stsp 
 745 d3c116bf 2021-10-15 stsp 			err = got_object_raw_open(&base_raw, &outfd, repo,
 746 94dac27c 2021-10-15 stsp 			    &base->id);
 747 e6bcace5 2021-06-22 stsp 			if (err)
 748 e6bcace5 2021-06-22 stsp 				goto done;
 749 67fd6849 2022-02-13 stsp 
 750 64a8571e 2022-01-07 stsp 			if (raw->f == NULL && base_raw->f == NULL) {
 751 64a8571e 2022-01-07 stsp 				err = got_deltify_mem_mem(&deltas, &ndeltas,
 752 64a8571e 2022-01-07 stsp 				    raw->data, raw->hdrlen,
 753 64a8571e 2022-01-07 stsp 				    raw->size + raw->hdrlen,
 754 64a8571e 2022-01-07 stsp 				    base->dtab, base_raw->data,
 755 64a8571e 2022-01-07 stsp 				    base_raw->hdrlen,
 756 64a8571e 2022-01-07 stsp 				    base_raw->size + base_raw->hdrlen);
 757 64a8571e 2022-01-07 stsp 			} else if (raw->f == NULL) {
 758 64a8571e 2022-01-07 stsp 				err = got_deltify_mem_file(&deltas, &ndeltas,
 759 64a8571e 2022-01-07 stsp 				    raw->data, raw->hdrlen,
 760 64a8571e 2022-01-07 stsp 				    raw->size + raw->hdrlen,
 761 64a8571e 2022-01-07 stsp 				    base->dtab, base_raw->f,
 762 64a8571e 2022-01-07 stsp 				    base_raw->hdrlen,
 763 64a8571e 2022-01-07 stsp 				    base_raw->size + base_raw->hdrlen);
 764 64a8571e 2022-01-07 stsp 			} else if (base_raw->f == NULL) {
 765 64a8571e 2022-01-07 stsp 				err = got_deltify_file_mem(&deltas, &ndeltas,
 766 64a8571e 2022-01-07 stsp 				    raw->f, raw->hdrlen,
 767 64a8571e 2022-01-07 stsp 				    raw->size + raw->hdrlen,
 768 64a8571e 2022-01-07 stsp 				    base->dtab, base_raw->data,
 769 64a8571e 2022-01-07 stsp 				    base_raw->hdrlen,
 770 64a8571e 2022-01-07 stsp 				    base_raw->size + base_raw->hdrlen);
 771 64a8571e 2022-01-07 stsp 			} else {
 772 64a8571e 2022-01-07 stsp 				err = got_deltify(&deltas, &ndeltas,
 773 64a8571e 2022-01-07 stsp 				    raw->f, raw->hdrlen,
 774 64a8571e 2022-01-07 stsp 				    raw->size + raw->hdrlen,
 775 64a8571e 2022-01-07 stsp 				    base->dtab, base_raw->f, base_raw->hdrlen,
 776 64a8571e 2022-01-07 stsp 				    base_raw->size + base_raw->hdrlen);
 777 64a8571e 2022-01-07 stsp 			}
 778 e6bcace5 2021-06-22 stsp 			got_object_raw_close(base_raw);
 779 e6bcace5 2021-06-22 stsp 			base_raw = NULL;
 780 e6bcace5 2021-06-22 stsp 			if (err)
 781 e6bcace5 2021-06-22 stsp 				goto done;
 782 e6bcace5 2021-06-22 stsp 
 783 e6bcace5 2021-06-22 stsp 			size = delta_size(deltas, ndeltas);
 784 74881701 2021-10-15 stsp 			if (size + 32 < best_size){
 785 e6bcace5 2021-06-22 stsp 				/*
 786 e6bcace5 2021-06-22 stsp 				 * if we already picked a best delta,
 787 e6bcace5 2021-06-22 stsp 				 * replace it.
 788 e6bcace5 2021-06-22 stsp 				 */
 789 74881701 2021-10-15 stsp 				best_size = size;
 790 74881701 2021-10-15 stsp 				free(best_deltas);
 791 74881701 2021-10-15 stsp 				best_deltas = deltas;
 792 74881701 2021-10-15 stsp 				best_ndeltas = ndeltas;
 793 74881701 2021-10-15 stsp 				deltas = NULL;
 794 e6bcace5 2021-06-22 stsp 				m->nchain = base->nchain + 1;
 795 e6bcace5 2021-06-22 stsp 				m->prev = base;
 796 e6bcace5 2021-06-22 stsp 				m->head = base->head;
 797 e6bcace5 2021-06-22 stsp 				if (m->head == NULL)
 798 e6bcace5 2021-06-22 stsp 					m->head = base;
 799 e6bcace5 2021-06-22 stsp 			} else {
 800 e6bcace5 2021-06-22 stsp 				free(deltas);
 801 e6bcace5 2021-06-22 stsp 				deltas = NULL;
 802 e6bcace5 2021-06-22 stsp 				ndeltas = 0;
 803 e6bcace5 2021-06-22 stsp 			}
 804 e6bcace5 2021-06-22 stsp 		}
 805 e6bcace5 2021-06-22 stsp 
 806 74881701 2021-10-15 stsp 		if (best_ndeltas > 0) {
 807 402a5ec1 2022-01-10 stsp 			if (best_size <= GOT_DELTA_RESULT_SIZE_CACHED_MAX &&
 808 402a5ec1 2022-01-10 stsp 			    delta_memsize + best_size <= max_delta_memsize) {
 809 402a5ec1 2022-01-10 stsp 				delta_memsize += best_size;
 810 5060d5a1 2022-01-10 stsp 				err = encode_delta_in_mem(m, raw, best_deltas,
 811 5060d5a1 2022-01-10 stsp 				    best_ndeltas, best_size, m->prev->size);
 812 5060d5a1 2022-01-10 stsp 			} else {
 813 5060d5a1 2022-01-10 stsp 				m->delta_offset = ftello(delta_cache);
 814 5060d5a1 2022-01-10 stsp 				err = encode_delta(m, raw, best_deltas,
 815 5060d5a1 2022-01-10 stsp 				    best_ndeltas, m->prev->size, delta_cache);
 816 5060d5a1 2022-01-10 stsp 			}
 817 74881701 2021-10-15 stsp 			free(best_deltas);
 818 74881701 2021-10-15 stsp 			best_deltas = NULL;
 819 74881701 2021-10-15 stsp 			best_ndeltas = 0;
 820 74881701 2021-10-15 stsp 			if (err)
 821 74881701 2021-10-15 stsp 				goto done;
 822 74881701 2021-10-15 stsp 		}
 823 74881701 2021-10-15 stsp 
 824 e6bcace5 2021-06-22 stsp 		got_object_raw_close(raw);
 825 e6bcace5 2021-06-22 stsp 		raw = NULL;
 826 e6bcace5 2021-06-22 stsp 	}
 827 e6bcace5 2021-06-22 stsp done:
 828 e6bcace5 2021-06-22 stsp 	for (i = MAX(0, nmeta - max_base_candidates); i < nmeta; i++) {
 829 e6bcace5 2021-06-22 stsp 		got_deltify_free(meta[i]->dtab);
 830 e6bcace5 2021-06-22 stsp 		meta[i]->dtab = NULL;
 831 e6bcace5 2021-06-22 stsp 	}
 832 e6bcace5 2021-06-22 stsp 	if (raw)
 833 e6bcace5 2021-06-22 stsp 		got_object_raw_close(raw);
 834 e6bcace5 2021-06-22 stsp 	if (base_raw)
 835 e6bcace5 2021-06-22 stsp 		got_object_raw_close(base_raw);
 836 cc7a354a 2021-10-15 stsp 	if (outfd != -1 && close(outfd) == -1 && err == NULL)
 837 cc7a354a 2021-10-15 stsp 		err = got_error_from_errno("close");
 838 74881701 2021-10-15 stsp 	free(deltas);
 839 74881701 2021-10-15 stsp 	free(best_deltas);
 840 e6bcace5 2021-06-22 stsp 	return err;
 841 e6bcace5 2021-06-22 stsp }
 842 e6bcace5 2021-06-22 stsp 
 843 e6bcace5 2021-06-22 stsp static const struct got_error *
 844 05118f5a 2021-06-22 stsp search_packidx(int *found, struct got_object_id *id,
 845 05118f5a 2021-06-22 stsp     struct got_repository *repo)
 846 05118f5a 2021-06-22 stsp {
 847 05118f5a 2021-06-22 stsp 	const struct got_error *err = NULL;
 848 05118f5a 2021-06-22 stsp 	struct got_packidx *packidx = NULL;
 849 05118f5a 2021-06-22 stsp 	int idx;
 850 05118f5a 2021-06-22 stsp 
 851 05118f5a 2021-06-22 stsp 	*found = 0;
 852 05118f5a 2021-06-22 stsp 
 853 05118f5a 2021-06-22 stsp 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
 854 05118f5a 2021-06-22 stsp 	if (err == NULL)
 855 05118f5a 2021-06-22 stsp 		*found = 1; /* object is already packed */
 856 05118f5a 2021-06-22 stsp 	else if (err->code == GOT_ERR_NO_OBJ)
 857 05118f5a 2021-06-22 stsp 		err = NULL;
 858 05118f5a 2021-06-22 stsp 	return err;
 859 05118f5a 2021-06-22 stsp }
 860 05118f5a 2021-06-22 stsp 
 861 05118f5a 2021-06-22 stsp static const struct got_error *
 862 67fd6849 2022-02-13 stsp add_object(int want_meta, struct got_object_idset *idset,
 863 e6bcace5 2021-06-22 stsp     struct got_object_id *id, const char *path, int obj_type,
 864 6863cbf9 2022-03-21 stsp     time_t mtime, int loose_obj_only, struct got_repository *repo,
 865 6863cbf9 2022-03-21 stsp     int *ncolored, int *nfound, int *ntrees,
 866 6863cbf9 2022-03-21 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
 867 6863cbf9 2022-03-21 stsp     struct got_ratelimit *rl)
 868 e6bcace5 2021-06-22 stsp {
 869 e6bcace5 2021-06-22 stsp 	const struct got_error *err;
 870 67fd6849 2022-02-13 stsp 	struct got_pack_meta *m = NULL;
 871 e6bcace5 2021-06-22 stsp 
 872 05118f5a 2021-06-22 stsp 	if (loose_obj_only) {
 873 05118f5a 2021-06-22 stsp 		int is_packed;
 874 05118f5a 2021-06-22 stsp 		err = search_packidx(&is_packed, id, repo);
 875 05118f5a 2021-06-22 stsp 		if (err)
 876 05118f5a 2021-06-22 stsp 			return err;
 877 cdeb891a 2022-03-21 stsp 		if (is_packed && want_meta)
 878 05118f5a 2021-06-22 stsp 			return NULL;
 879 05118f5a 2021-06-22 stsp 	}
 880 05118f5a 2021-06-22 stsp 
 881 67fd6849 2022-02-13 stsp 	if (want_meta) {
 882 67fd6849 2022-02-13 stsp 		err = alloc_meta(&m, id, path, obj_type, mtime);
 883 6863cbf9 2022-03-21 stsp 		if (err)
 884 6863cbf9 2022-03-21 stsp 			return err;
 885 6863cbf9 2022-03-21 stsp 
 886 6863cbf9 2022-03-21 stsp 		(*nfound)++;
 887 6863cbf9 2022-03-21 stsp 		err = report_progress(progress_cb, progress_arg, rl,
 888 6863cbf9 2022-03-21 stsp 		    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
 889 bb6672b6 2022-04-14 tb 		if (err) {
 890 bb6672b6 2022-04-14 tb 			clear_meta(m);
 891 bb6672b6 2022-04-14 tb 			free(m);
 892 67fd6849 2022-02-13 stsp 			return err;
 893 bb6672b6 2022-04-14 tb 		}
 894 e6bcace5 2021-06-22 stsp 	}
 895 e6bcace5 2021-06-22 stsp 
 896 bb6672b6 2022-04-14 tb 	err = got_object_idset_add(idset, id, m);
 897 bb6672b6 2022-04-14 tb 	if (err) {
 898 bb6672b6 2022-04-14 tb 		clear_meta(m);
 899 bb6672b6 2022-04-14 tb 		free(m);
 900 bb6672b6 2022-04-14 tb 	}
 901 bb6672b6 2022-04-14 tb 	return err;
 902 e6bcace5 2021-06-22 stsp }
 903 e6bcace5 2021-06-22 stsp 
 904 e6bcace5 2021-06-22 stsp static const struct got_error *
 905 67fd6849 2022-02-13 stsp load_tree_entries(struct got_object_id_queue *ids, int want_meta,
 906 2f8438b0 2022-05-04 stsp     struct got_object_idset *idset, struct got_object_idset *idset_exclude,
 907 2f8438b0 2022-05-04 stsp     struct got_object_id *tree_id,
 908 e6bcace5 2021-06-22 stsp     const char *dpath, time_t mtime, struct got_repository *repo,
 909 b8af7c06 2022-03-15 stsp     int loose_obj_only, int *ncolored, int *nfound, int *ntrees,
 910 b8af7c06 2022-03-15 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
 911 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 912 e6bcace5 2021-06-22 stsp {
 913 e6bcace5 2021-06-22 stsp 	const struct got_error *err;
 914 e6bcace5 2021-06-22 stsp 	struct got_tree_object *tree;
 915 e6bcace5 2021-06-22 stsp 	char *p = NULL;
 916 e6bcace5 2021-06-22 stsp 	int i;
 917 e6bcace5 2021-06-22 stsp 
 918 e6bcace5 2021-06-22 stsp 	err = got_object_open_as_tree(&tree, repo, tree_id);
 919 b8af7c06 2022-03-15 stsp 	if (err)
 920 b8af7c06 2022-03-15 stsp 		return err;
 921 b8af7c06 2022-03-15 stsp 
 922 b8af7c06 2022-03-15 stsp 	(*ntrees)++;
 923 b8af7c06 2022-03-15 stsp 	err = report_progress(progress_cb, progress_arg, rl,
 924 b8af7c06 2022-03-15 stsp 	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
 925 e6bcace5 2021-06-22 stsp 	if (err)
 926 e6bcace5 2021-06-22 stsp 		return err;
 927 e6bcace5 2021-06-22 stsp 
 928 e6bcace5 2021-06-22 stsp 	for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
 929 e6bcace5 2021-06-22 stsp 		struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
 930 e6bcace5 2021-06-22 stsp 		struct got_object_id *id = got_tree_entry_get_id(e);
 931 e6bcace5 2021-06-22 stsp 		mode_t mode = got_tree_entry_get_mode(e);
 932 e6bcace5 2021-06-22 stsp 
 933 e6bcace5 2021-06-22 stsp 		if (cancel_cb) {
 934 e6bcace5 2021-06-22 stsp 			err = (*cancel_cb)(cancel_arg);
 935 e6bcace5 2021-06-22 stsp 			if (err)
 936 e6bcace5 2021-06-22 stsp 				break;
 937 e6bcace5 2021-06-22 stsp 		}
 938 e6bcace5 2021-06-22 stsp 
 939 3af9de88 2021-09-22 stsp 		if (got_object_tree_entry_is_submodule(e) ||
 940 2f8438b0 2022-05-04 stsp 		    got_object_idset_contains(idset, id) ||
 941 2f8438b0 2022-05-04 stsp 		    got_object_idset_contains(idset_exclude, id))
 942 e6bcace5 2021-06-22 stsp 			continue;
 943 e6bcace5 2021-06-22 stsp 		
 944 e6bcace5 2021-06-22 stsp 		if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "",
 945 e6bcace5 2021-06-22 stsp 		    got_tree_entry_get_name(e)) == -1) {
 946 e6bcace5 2021-06-22 stsp 			err = got_error_from_errno("asprintf");
 947 e6bcace5 2021-06-22 stsp 			break;
 948 e6bcace5 2021-06-22 stsp 		}
 949 e6bcace5 2021-06-22 stsp 
 950 e6bcace5 2021-06-22 stsp 		if (S_ISDIR(mode)) {
 951 e6bcace5 2021-06-22 stsp 			struct got_object_qid *qid;
 952 e6bcace5 2021-06-22 stsp 			err = got_object_qid_alloc(&qid, id);
 953 e6bcace5 2021-06-22 stsp 			if (err)
 954 e6bcace5 2021-06-22 stsp 				break;
 955 dbdddfee 2021-06-23 naddy 			STAILQ_INSERT_TAIL(ids, qid, entry);
 956 3af9de88 2021-09-22 stsp 		} else if (S_ISREG(mode) || S_ISLNK(mode)) {
 957 2f8438b0 2022-05-04 stsp 			err = add_object(want_meta,
 958 2f8438b0 2022-05-04 stsp 			    want_meta ? idset : idset_exclude, id, p,
 959 6863cbf9 2022-03-21 stsp 			    GOT_OBJ_TYPE_BLOB, mtime, loose_obj_only, repo,
 960 6863cbf9 2022-03-21 stsp 			    ncolored, nfound, ntrees,
 961 6863cbf9 2022-03-21 stsp 			    progress_cb, progress_arg, rl);
 962 b8af7c06 2022-03-15 stsp 			if (err)
 963 b8af7c06 2022-03-15 stsp 				break;
 964 e6bcace5 2021-06-22 stsp 		}
 965 e6bcace5 2021-06-22 stsp 		free(p);
 966 e6bcace5 2021-06-22 stsp 		p = NULL;
 967 e6bcace5 2021-06-22 stsp 	}
 968 e6bcace5 2021-06-22 stsp 
 969 e6bcace5 2021-06-22 stsp 	got_object_tree_close(tree);
 970 e6bcace5 2021-06-22 stsp 	free(p);
 971 e6bcace5 2021-06-22 stsp 	return err;
 972 e6bcace5 2021-06-22 stsp }
 973 e6bcace5 2021-06-22 stsp 
 974 e6bcace5 2021-06-22 stsp static const struct got_error *
 975 67fd6849 2022-02-13 stsp load_tree(int want_meta, struct got_object_idset *idset,
 976 2f8438b0 2022-05-04 stsp     struct got_object_idset *idset_exclude,
 977 e6bcace5 2021-06-22 stsp     struct got_object_id *tree_id, const char *dpath, time_t mtime,
 978 b8af7c06 2022-03-15 stsp     struct got_repository *repo, int loose_obj_only,
 979 b8af7c06 2022-03-15 stsp     int *ncolored, int *nfound, int *ntrees,
 980 b8af7c06 2022-03-15 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
 981 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 982 e6bcace5 2021-06-22 stsp {
 983 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
 984 e6bcace5 2021-06-22 stsp 	struct got_object_id_queue tree_ids;
 985 e6bcace5 2021-06-22 stsp 	struct got_object_qid *qid;
 986 e6bcace5 2021-06-22 stsp 
 987 2f8438b0 2022-05-04 stsp 	if (got_object_idset_contains(idset, tree_id) ||
 988 2f8438b0 2022-05-04 stsp 	    got_object_idset_contains(idset_exclude, tree_id))
 989 e6bcace5 2021-06-22 stsp 		return NULL;
 990 e6bcace5 2021-06-22 stsp 
 991 e6bcace5 2021-06-22 stsp 	err = got_object_qid_alloc(&qid, tree_id);
 992 e6bcace5 2021-06-22 stsp 	if (err)
 993 e6bcace5 2021-06-22 stsp 		return err;
 994 e6bcace5 2021-06-22 stsp 
 995 dbdddfee 2021-06-23 naddy 	STAILQ_INIT(&tree_ids);
 996 dbdddfee 2021-06-23 naddy 	STAILQ_INSERT_TAIL(&tree_ids, qid, entry);
 997 e6bcace5 2021-06-22 stsp 
 998 dbdddfee 2021-06-23 naddy 	while (!STAILQ_EMPTY(&tree_ids)) {
 999 e6bcace5 2021-06-22 stsp 		if (cancel_cb) {
1000 e6bcace5 2021-06-22 stsp 			err = (*cancel_cb)(cancel_arg);
1001 e6bcace5 2021-06-22 stsp 			if (err)
1002 e6bcace5 2021-06-22 stsp 				break;
1003 e6bcace5 2021-06-22 stsp 		}
1004 e6bcace5 2021-06-22 stsp 
1005 dbdddfee 2021-06-23 naddy 		qid = STAILQ_FIRST(&tree_ids);
1006 dbdddfee 2021-06-23 naddy 		STAILQ_REMOVE_HEAD(&tree_ids, entry);
1007 e6bcace5 2021-06-22 stsp 
1008 2f8438b0 2022-05-04 stsp 		if (got_object_idset_contains(idset, &qid->id) ||
1009 2f8438b0 2022-05-04 stsp 		    got_object_idset_contains(idset_exclude, &qid->id)) {
1010 e6bcace5 2021-06-22 stsp 			got_object_qid_free(qid);
1011 e6bcace5 2021-06-22 stsp 			continue;
1012 e6bcace5 2021-06-22 stsp 		}
1013 e6bcace5 2021-06-22 stsp 
1014 2f8438b0 2022-05-04 stsp 		err = add_object(want_meta, want_meta ? idset : idset_exclude,
1015 2f8438b0 2022-05-04 stsp 		    &qid->id, dpath, GOT_OBJ_TYPE_TREE,
1016 2f8438b0 2022-05-04 stsp 		    mtime, loose_obj_only, repo,
1017 6863cbf9 2022-03-21 stsp 		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
1018 e6bcace5 2021-06-22 stsp 		if (err) {
1019 e6bcace5 2021-06-22 stsp 			got_object_qid_free(qid);
1020 e6bcace5 2021-06-22 stsp 			break;
1021 e6bcace5 2021-06-22 stsp 		}
1022 e6bcace5 2021-06-22 stsp 
1023 2f8438b0 2022-05-04 stsp 		err = load_tree_entries(&tree_ids, want_meta, idset,
1024 2f8438b0 2022-05-04 stsp 		    idset_exclude, &qid->id,
1025 b8af7c06 2022-03-15 stsp 		    dpath, mtime, repo, loose_obj_only, ncolored, nfound,
1026 b8af7c06 2022-03-15 stsp 		    ntrees, progress_cb, progress_arg, rl,
1027 b8af7c06 2022-03-15 stsp 		    cancel_cb, cancel_arg);
1028 e6bcace5 2021-06-22 stsp 		got_object_qid_free(qid);
1029 e6bcace5 2021-06-22 stsp 		if (err)
1030 e6bcace5 2021-06-22 stsp 			break;
1031 e6bcace5 2021-06-22 stsp 	}
1032 e6bcace5 2021-06-22 stsp 
1033 e6bcace5 2021-06-22 stsp 	got_object_id_queue_free(&tree_ids);
1034 e6bcace5 2021-06-22 stsp 	return err;
1035 e6bcace5 2021-06-22 stsp }
1036 e6bcace5 2021-06-22 stsp 
1037 e6bcace5 2021-06-22 stsp static const struct got_error *
1038 67fd6849 2022-02-13 stsp load_commit(int want_meta, struct got_object_idset *idset,
1039 2f8438b0 2022-05-04 stsp     struct got_object_idset *idset_exclude,
1040 05118f5a 2021-06-22 stsp     struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
1041 b8af7c06 2022-03-15 stsp     int *ncolored, int *nfound, int *ntrees,
1042 b8af7c06 2022-03-15 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1043 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
1044 e6bcace5 2021-06-22 stsp {
1045 e6bcace5 2021-06-22 stsp 	const struct got_error *err;
1046 e6bcace5 2021-06-22 stsp 	struct got_commit_object *commit;
1047 e6bcace5 2021-06-22 stsp 
1048 2f8438b0 2022-05-04 stsp 	if (got_object_idset_contains(idset, id) ||
1049 2f8438b0 2022-05-04 stsp 	    got_object_idset_contains(idset_exclude, id))
1050 e6bcace5 2021-06-22 stsp 		return NULL;
1051 05118f5a 2021-06-22 stsp 
1052 05118f5a 2021-06-22 stsp 	if (loose_obj_only) {
1053 05118f5a 2021-06-22 stsp 		int is_packed;
1054 05118f5a 2021-06-22 stsp 		err = search_packidx(&is_packed, id, repo);
1055 05118f5a 2021-06-22 stsp 		if (err)
1056 05118f5a 2021-06-22 stsp 			return err;
1057 cdeb891a 2022-03-21 stsp 		if (is_packed && want_meta)
1058 05118f5a 2021-06-22 stsp 			return NULL;
1059 05118f5a 2021-06-22 stsp 	}
1060 e6bcace5 2021-06-22 stsp 
1061 e6bcace5 2021-06-22 stsp 	err = got_object_open_as_commit(&commit, repo, id);
1062 e6bcace5 2021-06-22 stsp 	if (err)
1063 e6bcace5 2021-06-22 stsp 		return err;
1064 e6bcace5 2021-06-22 stsp 
1065 2f8438b0 2022-05-04 stsp 	err = add_object(want_meta, want_meta ? idset : idset_exclude,
1066 2f8438b0 2022-05-04 stsp 	    id, "", GOT_OBJ_TYPE_COMMIT,
1067 05118f5a 2021-06-22 stsp 	    got_object_commit_get_committer_time(commit),
1068 6863cbf9 2022-03-21 stsp 	    loose_obj_only, repo,
1069 6863cbf9 2022-03-21 stsp 	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
1070 e6bcace5 2021-06-22 stsp 	if (err)
1071 e6bcace5 2021-06-22 stsp 		goto done;
1072 b8af7c06 2022-03-15 stsp 
1073 2f8438b0 2022-05-04 stsp 	err = load_tree(want_meta, idset, idset_exclude,
1074 2f8438b0 2022-05-04 stsp 	    got_object_commit_get_tree_id(commit),
1075 05118f5a 2021-06-22 stsp 	    "", got_object_commit_get_committer_time(commit),
1076 b8af7c06 2022-03-15 stsp 	    repo, loose_obj_only, ncolored, nfound, ntrees,
1077 b8af7c06 2022-03-15 stsp 	    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
1078 e6bcace5 2021-06-22 stsp done:
1079 e6bcace5 2021-06-22 stsp 	got_object_commit_close(commit);
1080 e6bcace5 2021-06-22 stsp 	return err;
1081 e6bcace5 2021-06-22 stsp }
1082 e6bcace5 2021-06-22 stsp 
1083 05118f5a 2021-06-22 stsp static const struct got_error *
1084 67fd6849 2022-02-13 stsp load_tag(int want_meta, struct got_object_idset *idset,
1085 2f8438b0 2022-05-04 stsp     struct got_object_idset *idset_exclude,
1086 05118f5a 2021-06-22 stsp     struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
1087 b8af7c06 2022-03-15 stsp     int *ncolored, int *nfound, int *ntrees,
1088 b8af7c06 2022-03-15 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1089 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
1090 05118f5a 2021-06-22 stsp {
1091 05118f5a 2021-06-22 stsp 	const struct got_error *err;
1092 05118f5a 2021-06-22 stsp 	struct got_tag_object *tag = NULL;
1093 05118f5a 2021-06-22 stsp 
1094 2f8438b0 2022-05-04 stsp 	if (got_object_idset_contains(idset, id) ||
1095 2f8438b0 2022-05-04 stsp 	    got_object_idset_contains(idset_exclude, id))
1096 05118f5a 2021-06-22 stsp 		return NULL;
1097 05118f5a 2021-06-22 stsp 
1098 05118f5a 2021-06-22 stsp 	if (loose_obj_only) {
1099 05118f5a 2021-06-22 stsp 		int is_packed;
1100 05118f5a 2021-06-22 stsp 		err = search_packidx(&is_packed, id, repo);
1101 05118f5a 2021-06-22 stsp 		if (err)
1102 05118f5a 2021-06-22 stsp 			return err;
1103 cdeb891a 2022-03-21 stsp 		if (is_packed && want_meta)
1104 05118f5a 2021-06-22 stsp 			return NULL;
1105 05118f5a 2021-06-22 stsp 	}
1106 05118f5a 2021-06-22 stsp 
1107 05118f5a 2021-06-22 stsp 	err = got_object_open_as_tag(&tag, repo, id);
1108 05118f5a 2021-06-22 stsp 	if (err)
1109 05118f5a 2021-06-22 stsp 		return err;
1110 05118f5a 2021-06-22 stsp 
1111 2f8438b0 2022-05-04 stsp 	err = add_object(want_meta, want_meta ? idset : idset_exclude,
1112 2f8438b0 2022-05-04 stsp 	    id, "", GOT_OBJ_TYPE_TAG,
1113 6863cbf9 2022-03-21 stsp 	    got_object_tag_get_tagger_time(tag), loose_obj_only, repo,
1114 6863cbf9 2022-03-21 stsp 	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
1115 05118f5a 2021-06-22 stsp 	if (err)
1116 05118f5a 2021-06-22 stsp 		goto done;
1117 05118f5a 2021-06-22 stsp 
1118 05118f5a 2021-06-22 stsp 	switch (got_object_tag_get_object_type(tag)) {
1119 05118f5a 2021-06-22 stsp 	case GOT_OBJ_TYPE_COMMIT:
1120 2f8438b0 2022-05-04 stsp 		err = load_commit(want_meta, idset, idset_exclude,
1121 b8af7c06 2022-03-15 stsp 		    got_object_tag_get_object_id(tag), repo, loose_obj_only,
1122 b8af7c06 2022-03-15 stsp 		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
1123 b8af7c06 2022-03-15 stsp 		    cancel_cb, cancel_arg);
1124 05118f5a 2021-06-22 stsp 		break;
1125 05118f5a 2021-06-22 stsp 	case GOT_OBJ_TYPE_TREE:
1126 2f8438b0 2022-05-04 stsp 		err = load_tree(want_meta, idset, idset_exclude,
1127 67fd6849 2022-02-13 stsp 		    got_object_tag_get_object_id(tag), "",
1128 b8af7c06 2022-03-15 stsp 		    got_object_tag_get_tagger_time(tag), repo, loose_obj_only,
1129 b8af7c06 2022-03-15 stsp 		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
1130 b8af7c06 2022-03-15 stsp 		    cancel_cb, cancel_arg);
1131 05118f5a 2021-06-22 stsp 		break;
1132 05118f5a 2021-06-22 stsp 	default:
1133 05118f5a 2021-06-22 stsp 		break;
1134 05118f5a 2021-06-22 stsp 	}
1135 05118f5a 2021-06-22 stsp 
1136 05118f5a 2021-06-22 stsp done:
1137 05118f5a 2021-06-22 stsp 	got_object_tag_close(tag);
1138 05118f5a 2021-06-22 stsp 	return err;
1139 05118f5a 2021-06-22 stsp }
1140 05118f5a 2021-06-22 stsp 
1141 e6bcace5 2021-06-22 stsp enum findtwixt_color {
1142 e6bcace5 2021-06-22 stsp 	COLOR_KEEP = 0,
1143 e6bcace5 2021-06-22 stsp 	COLOR_DROP,
1144 e6bcace5 2021-06-22 stsp 	COLOR_BLANK,
1145 70f8f24d 2022-04-14 stsp 	COLOR_SKIP,
1146 e6bcace5 2021-06-22 stsp };
1147 70f8f24d 2022-04-14 stsp 
1148 e6bcace5 2021-06-22 stsp static const int findtwixt_colors[] = {
1149 e6bcace5 2021-06-22 stsp 	COLOR_KEEP,
1150 e6bcace5 2021-06-22 stsp 	COLOR_DROP,
1151 70f8f24d 2022-04-14 stsp 	COLOR_BLANK,
1152 70f8f24d 2022-04-14 stsp 	COLOR_SKIP,
1153 e6bcace5 2021-06-22 stsp };
1154 e6bcace5 2021-06-22 stsp 
1155 e6bcace5 2021-06-22 stsp static const struct got_error *
1156 70f8f24d 2022-04-14 stsp paint_commit(struct got_object_qid *qid, int color)
1157 e6bcace5 2021-06-22 stsp {
1158 70f8f24d 2022-04-14 stsp 	if (color < 0 || color >= nitems(findtwixt_colors))
1159 70f8f24d 2022-04-14 stsp 		return got_error(GOT_ERR_RANGE);
1160 e6bcace5 2021-06-22 stsp 
1161 e6bcace5 2021-06-22 stsp 	qid->data = (void *)&findtwixt_colors[color];
1162 e6bcace5 2021-06-22 stsp 	return NULL;
1163 e6bcace5 2021-06-22 stsp }
1164 e6bcace5 2021-06-22 stsp 
1165 e6bcace5 2021-06-22 stsp static const struct got_error *
1166 70f8f24d 2022-04-14 stsp queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1167 70f8f24d 2022-04-14 stsp     int color, struct got_repository *repo)
1168 e6bcace5 2021-06-22 stsp {
1169 70f8f24d 2022-04-14 stsp 	const struct got_error *err;
1170 e6bcace5 2021-06-22 stsp 	struct got_object_qid *qid;
1171 e6bcace5 2021-06-22 stsp 
1172 e6bcace5 2021-06-22 stsp 	err = got_object_qid_alloc(&qid, id);
1173 e6bcace5 2021-06-22 stsp 	if (err)
1174 e6bcace5 2021-06-22 stsp 		return err;
1175 e6bcace5 2021-06-22 stsp 
1176 70f8f24d 2022-04-14 stsp 	STAILQ_INSERT_TAIL(ids, qid, entry);
1177 70f8f24d 2022-04-14 stsp 	return paint_commit(qid, color);
1178 e6bcace5 2021-06-22 stsp }
1179 e6bcace5 2021-06-22 stsp 
1180 e6bcace5 2021-06-22 stsp struct append_id_arg {
1181 e6bcace5 2021-06-22 stsp 	struct got_object_id **array;
1182 e6bcace5 2021-06-22 stsp 	int idx;
1183 70f8f24d 2022-04-14 stsp 	struct got_object_idset *drop;
1184 70f8f24d 2022-04-14 stsp 	struct got_object_idset *skip;
1185 e6bcace5 2021-06-22 stsp };
1186 e6bcace5 2021-06-22 stsp 
1187 e6bcace5 2021-06-22 stsp static const struct got_error *
1188 e6bcace5 2021-06-22 stsp append_id(struct got_object_id *id, void *data, void *arg)
1189 e6bcace5 2021-06-22 stsp {
1190 e6bcace5 2021-06-22 stsp 	struct append_id_arg *a = arg;
1191 e6bcace5 2021-06-22 stsp 
1192 70f8f24d 2022-04-14 stsp 	if (got_object_idset_contains(a->skip, id) ||
1193 70f8f24d 2022-04-14 stsp 	    got_object_idset_contains(a->drop, id))
1194 70f8f24d 2022-04-14 stsp 		return NULL;
1195 70f8f24d 2022-04-14 stsp 
1196 70f8f24d 2022-04-14 stsp 	a->array[++a->idx] = got_object_id_dup(id);
1197 e6bcace5 2021-06-22 stsp 	if (a->array[a->idx] == NULL)
1198 e6bcace5 2021-06-22 stsp 		return got_error_from_errno("got_object_id_dup");
1199 e6bcace5 2021-06-22 stsp 
1200 e6bcace5 2021-06-22 stsp 	return NULL;
1201 29e0594f 2022-04-09 stsp }
1202 29e0594f 2022-04-09 stsp 
1203 29e0594f 2022-04-09 stsp static const struct got_error *
1204 29e0594f 2022-04-09 stsp queue_commit_or_tag_id(struct got_object_id *id, int color,
1205 29e0594f 2022-04-09 stsp     struct got_object_id_queue *ids, struct got_repository *repo)
1206 29e0594f 2022-04-09 stsp {
1207 29e0594f 2022-04-09 stsp 	const struct got_error *err;
1208 29e0594f 2022-04-09 stsp 	struct got_tag_object *tag = NULL;
1209 29e0594f 2022-04-09 stsp 	int obj_type;
1210 29e0594f 2022-04-09 stsp 
1211 29e0594f 2022-04-09 stsp 	err = got_object_get_type(&obj_type, repo, id);
1212 29e0594f 2022-04-09 stsp 	if (err)
1213 29e0594f 2022-04-09 stsp 		return err;
1214 29e0594f 2022-04-09 stsp 
1215 29e0594f 2022-04-09 stsp 	if (obj_type == GOT_OBJ_TYPE_TAG) {
1216 29e0594f 2022-04-09 stsp 		err = got_object_open_as_tag(&tag, repo, id);
1217 29e0594f 2022-04-09 stsp 		if (err)
1218 29e0594f 2022-04-09 stsp 			return err;
1219 29e0594f 2022-04-09 stsp 		obj_type = got_object_tag_get_object_type(tag);
1220 29e0594f 2022-04-09 stsp 		id = got_object_tag_get_object_id(tag);
1221 29e0594f 2022-04-09 stsp 	}
1222 29e0594f 2022-04-09 stsp 
1223 29e0594f 2022-04-09 stsp 	if (obj_type == GOT_OBJ_TYPE_COMMIT) {
1224 29e0594f 2022-04-09 stsp 		err = queue_commit_id(ids, id, color, repo);
1225 29e0594f 2022-04-09 stsp 		if (err)
1226 29e0594f 2022-04-09 stsp 			goto done;
1227 29e0594f 2022-04-09 stsp 	}
1228 29e0594f 2022-04-09 stsp done:
1229 29e0594f 2022-04-09 stsp 	if (tag)
1230 29e0594f 2022-04-09 stsp 		got_object_tag_close(tag);
1231 29e0594f 2022-04-09 stsp 	return err;
1232 e6bcace5 2021-06-22 stsp }
1233 e6bcace5 2021-06-22 stsp 
1234 e6bcace5 2021-06-22 stsp static const struct got_error *
1235 70f8f24d 2022-04-14 stsp paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
1236 14dbbf48 2022-04-10 stsp     struct got_object_idset *keep, struct got_object_idset *drop,
1237 70f8f24d 2022-04-14 stsp     struct got_object_idset *skip, struct got_repository *repo,
1238 b8af7c06 2022-03-15 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1239 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
1240 e6bcace5 2021-06-22 stsp {
1241 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
1242 14dbbf48 2022-04-10 stsp 	struct got_commit_object *commit = NULL;
1243 70f8f24d 2022-04-14 stsp 	const struct got_object_id_queue *parents;
1244 e6bcace5 2021-06-22 stsp 	struct got_object_qid *qid;
1245 70f8f24d 2022-04-14 stsp 	int nqueued = nids, nskip = 0;
1246 e6bcace5 2021-06-22 stsp 
1247 70f8f24d 2022-04-14 stsp 	while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1248 70f8f24d 2022-04-14 stsp 		int color;
1249 03c03172 2022-04-10 stsp 
1250 57bc7b6d 2022-04-10 stsp 		if (cancel_cb) {
1251 57bc7b6d 2022-04-10 stsp 			err = cancel_cb(cancel_arg);
1252 57bc7b6d 2022-04-10 stsp 			if (err)
1253 14dbbf48 2022-04-10 stsp 				break;
1254 57bc7b6d 2022-04-10 stsp 		}
1255 57bc7b6d 2022-04-10 stsp 
1256 14dbbf48 2022-04-10 stsp 		qid = STAILQ_FIRST(ids);
1257 70f8f24d 2022-04-14 stsp 		STAILQ_REMOVE_HEAD(ids, entry);
1258 70f8f24d 2022-04-14 stsp 		nqueued--;
1259 70f8f24d 2022-04-14 stsp 		color = *((int *)qid->data);
1260 70f8f24d 2022-04-14 stsp 		if (color == COLOR_SKIP)
1261 70f8f24d 2022-04-14 stsp 			nskip--;
1262 e6bcace5 2021-06-22 stsp 
1263 d7b5a0e8 2022-04-20 stsp 		if (got_object_idset_contains(skip, &qid->id)) {
1264 70f8f24d 2022-04-14 stsp 			got_object_qid_free(qid);
1265 70f8f24d 2022-04-14 stsp 			continue;
1266 70f8f24d 2022-04-14 stsp 		}
1267 b8af7c06 2022-03-15 stsp 
1268 70f8f24d 2022-04-14 stsp 		switch (color) {
1269 70f8f24d 2022-04-14 stsp 		case COLOR_KEEP:
1270 d7b5a0e8 2022-04-20 stsp 			if (got_object_idset_contains(keep, &qid->id)) {
1271 70f8f24d 2022-04-14 stsp 				got_object_qid_free(qid);
1272 70f8f24d 2022-04-14 stsp 				continue;
1273 70f8f24d 2022-04-14 stsp 			}
1274 d7b5a0e8 2022-04-20 stsp 			if (got_object_idset_contains(drop, &qid->id)) {
1275 70f8f24d 2022-04-14 stsp 				err = paint_commit(qid, COLOR_SKIP);
1276 70f8f24d 2022-04-14 stsp 				if (err)
1277 70f8f24d 2022-04-14 stsp 					goto done;
1278 70f8f24d 2022-04-14 stsp 				nskip++;
1279 70f8f24d 2022-04-14 stsp 			} else
1280 70f8f24d 2022-04-14 stsp 				(*ncolored)++;
1281 d7b5a0e8 2022-04-20 stsp 			err = got_object_idset_add(keep, &qid->id, NULL);
1282 70f8f24d 2022-04-14 stsp 			if (err)
1283 70f8f24d 2022-04-14 stsp 				goto done;
1284 70f8f24d 2022-04-14 stsp 			break;
1285 70f8f24d 2022-04-14 stsp 		case COLOR_DROP:
1286 d7b5a0e8 2022-04-20 stsp 			if (got_object_idset_contains(drop, &qid->id)) {
1287 70f8f24d 2022-04-14 stsp 				got_object_qid_free(qid);
1288 70f8f24d 2022-04-14 stsp 				continue;
1289 70f8f24d 2022-04-14 stsp 			}
1290 d7b5a0e8 2022-04-20 stsp 			if (got_object_idset_contains(keep, &qid->id)) {
1291 70f8f24d 2022-04-14 stsp 				err = paint_commit(qid, COLOR_SKIP);
1292 70f8f24d 2022-04-14 stsp 				if (err)
1293 70f8f24d 2022-04-14 stsp 					goto done;
1294 70f8f24d 2022-04-14 stsp 				nskip++;
1295 70f8f24d 2022-04-14 stsp 			} else
1296 70f8f24d 2022-04-14 stsp 				(*ncolored)++;
1297 d7b5a0e8 2022-04-20 stsp 			err = got_object_idset_add(drop, &qid->id, NULL);
1298 70f8f24d 2022-04-14 stsp 			if (err)
1299 70f8f24d 2022-04-14 stsp 				goto done;
1300 70f8f24d 2022-04-14 stsp 			break;
1301 70f8f24d 2022-04-14 stsp 		case COLOR_SKIP:
1302 d7b5a0e8 2022-04-20 stsp 			if (!got_object_idset_contains(skip, &qid->id)) {
1303 d7b5a0e8 2022-04-20 stsp 				err = got_object_idset_add(skip, &qid->id,
1304 d7b5a0e8 2022-04-20 stsp 				    NULL);
1305 70f8f24d 2022-04-14 stsp 				if (err)
1306 70f8f24d 2022-04-14 stsp 					goto done;
1307 70f8f24d 2022-04-14 stsp 			}
1308 70f8f24d 2022-04-14 stsp 			break;
1309 70f8f24d 2022-04-14 stsp 		default:
1310 70f8f24d 2022-04-14 stsp 			/* should not happen */
1311 70f8f24d 2022-04-14 stsp 			err = got_error_fmt(GOT_ERR_NOT_IMPL,
1312 70f8f24d 2022-04-14 stsp 			    "%s invalid commit color %d", __func__, color);
1313 70f8f24d 2022-04-14 stsp 			goto done;
1314 70f8f24d 2022-04-14 stsp 		}
1315 70f8f24d 2022-04-14 stsp 
1316 b8af7c06 2022-03-15 stsp 		err = report_progress(progress_cb, progress_arg, rl,
1317 b8af7c06 2022-03-15 stsp 		    *ncolored, 0, 0, 0L, 0, 0, 0, 0);
1318 b8af7c06 2022-03-15 stsp 		if (err)
1319 14dbbf48 2022-04-10 stsp 			break;
1320 e6bcace5 2021-06-22 stsp 
1321 e6bcace5 2021-06-22 stsp 
1322 d7b5a0e8 2022-04-20 stsp 		err = got_object_open_as_commit(&commit, repo, &qid->id);
1323 70f8f24d 2022-04-14 stsp 		if (err)
1324 70f8f24d 2022-04-14 stsp 			break;
1325 e6bcace5 2021-06-22 stsp 
1326 70f8f24d 2022-04-14 stsp 		parents = got_object_commit_get_parent_ids(commit);
1327 70f8f24d 2022-04-14 stsp 		if (parents) {
1328 70f8f24d 2022-04-14 stsp 			struct got_object_qid *pid;
1329 70f8f24d 2022-04-14 stsp 			color = *((int *)qid->data);
1330 70f8f24d 2022-04-14 stsp 			STAILQ_FOREACH(pid, parents, entry) {
1331 d7b5a0e8 2022-04-20 stsp 				err = queue_commit_id(ids, &pid->id, color,
1332 70f8f24d 2022-04-14 stsp 				    repo);
1333 70f8f24d 2022-04-14 stsp 				if (err)
1334 70f8f24d 2022-04-14 stsp 					break;
1335 70f8f24d 2022-04-14 stsp 				nqueued++;
1336 70f8f24d 2022-04-14 stsp 				if (color == COLOR_SKIP)
1337 70f8f24d 2022-04-14 stsp 					nskip++;
1338 e6bcace5 2021-06-22 stsp 			}
1339 e6bcace5 2021-06-22 stsp 		}
1340 e6bcace5 2021-06-22 stsp 
1341 70f8f24d 2022-04-14 stsp 		got_object_commit_close(commit);
1342 70f8f24d 2022-04-14 stsp 		commit = NULL;
1343 e6bcace5 2021-06-22 stsp 		got_object_qid_free(qid);
1344 e6bcace5 2021-06-22 stsp 	}
1345 70f8f24d 2022-04-14 stsp done:
1346 14dbbf48 2022-04-10 stsp 	if (commit)
1347 14dbbf48 2022-04-10 stsp 		got_object_commit_close(commit);
1348 14dbbf48 2022-04-10 stsp 	return err;
1349 14dbbf48 2022-04-10 stsp }
1350 14dbbf48 2022-04-10 stsp 
1351 14dbbf48 2022-04-10 stsp static const struct got_error *
1352 14dbbf48 2022-04-10 stsp findtwixt(struct got_object_id ***res, int *nres, int *ncolored,
1353 14dbbf48 2022-04-10 stsp     struct got_object_id **head, int nhead,
1354 14dbbf48 2022-04-10 stsp     struct got_object_id **tail, int ntail,
1355 14dbbf48 2022-04-10 stsp     struct got_repository *repo,
1356 14dbbf48 2022-04-10 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1357 14dbbf48 2022-04-10 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
1358 14dbbf48 2022-04-10 stsp {
1359 14dbbf48 2022-04-10 stsp 	const struct got_error *err = NULL;
1360 14dbbf48 2022-04-10 stsp 	struct got_object_id_queue ids;
1361 70f8f24d 2022-04-14 stsp 	struct got_object_idset *keep, *drop, *skip = NULL;
1362 14dbbf48 2022-04-10 stsp 	int i, nkeep;
1363 14dbbf48 2022-04-10 stsp 
1364 14dbbf48 2022-04-10 stsp 	STAILQ_INIT(&ids);
1365 14dbbf48 2022-04-10 stsp 	*res = NULL;
1366 14dbbf48 2022-04-10 stsp 	*nres = 0;
1367 14dbbf48 2022-04-10 stsp 	*ncolored = 0;
1368 14dbbf48 2022-04-10 stsp 
1369 14dbbf48 2022-04-10 stsp 	keep = got_object_idset_alloc();
1370 14dbbf48 2022-04-10 stsp 	if (keep == NULL)
1371 14dbbf48 2022-04-10 stsp 		return got_error_from_errno("got_object_idset_alloc");
1372 14dbbf48 2022-04-10 stsp 
1373 14dbbf48 2022-04-10 stsp 	drop = got_object_idset_alloc();
1374 14dbbf48 2022-04-10 stsp 	if (drop == NULL) {
1375 14dbbf48 2022-04-10 stsp 		err = got_error_from_errno("got_object_idset_alloc");
1376 14dbbf48 2022-04-10 stsp 		goto done;
1377 14dbbf48 2022-04-10 stsp 	}
1378 14dbbf48 2022-04-10 stsp 
1379 70f8f24d 2022-04-14 stsp 	skip = got_object_idset_alloc();
1380 70f8f24d 2022-04-14 stsp 	if (skip == NULL) {
1381 70f8f24d 2022-04-14 stsp 		err = got_error_from_errno("got_object_idset_alloc");
1382 70f8f24d 2022-04-14 stsp 		goto done;
1383 70f8f24d 2022-04-14 stsp 	}
1384 70f8f24d 2022-04-14 stsp 
1385 14dbbf48 2022-04-10 stsp 	for (i = 0; i < nhead; i++) {
1386 14dbbf48 2022-04-10 stsp 		struct got_object_id *id = head[i];
1387 14dbbf48 2022-04-10 stsp 		if (id == NULL)
1388 14dbbf48 2022-04-10 stsp 			continue;
1389 fbafdecf 2022-04-10 stsp 		err = queue_commit_or_tag_id(id, COLOR_KEEP, &ids, repo);
1390 14dbbf48 2022-04-10 stsp 		if (err)
1391 14dbbf48 2022-04-10 stsp 			goto done;
1392 14dbbf48 2022-04-10 stsp 	}		
1393 14dbbf48 2022-04-10 stsp 
1394 14dbbf48 2022-04-10 stsp 	for (i = 0; i < ntail; i++) {
1395 14dbbf48 2022-04-10 stsp 		struct got_object_id *id = tail[i];
1396 14dbbf48 2022-04-10 stsp 		if (id == NULL)
1397 14dbbf48 2022-04-10 stsp 			continue;
1398 14dbbf48 2022-04-10 stsp 		err = queue_commit_or_tag_id(id, COLOR_DROP, &ids, repo);
1399 14dbbf48 2022-04-10 stsp 		if (err)
1400 14dbbf48 2022-04-10 stsp 			goto done;
1401 14dbbf48 2022-04-10 stsp 	}
1402 14dbbf48 2022-04-10 stsp 
1403 70f8f24d 2022-04-14 stsp 	err = paint_commits(ncolored, &ids, nhead + ntail,
1404 70f8f24d 2022-04-14 stsp 	    keep, drop, skip, repo, progress_cb, progress_arg, rl,
1405 70f8f24d 2022-04-14 stsp 	    cancel_cb, cancel_arg);
1406 14dbbf48 2022-04-10 stsp 	if (err)
1407 14dbbf48 2022-04-10 stsp 		goto done;
1408 14dbbf48 2022-04-10 stsp 
1409 e6bcace5 2021-06-22 stsp 	nkeep = got_object_idset_num_elements(keep);
1410 e6bcace5 2021-06-22 stsp 	if (nkeep > 0) {
1411 e6bcace5 2021-06-22 stsp 		struct append_id_arg arg;
1412 e6bcace5 2021-06-22 stsp 		arg.array = calloc(nkeep, sizeof(struct got_object_id *));
1413 e6bcace5 2021-06-22 stsp 		if (arg.array == NULL) {
1414 e6bcace5 2021-06-22 stsp 			err = got_error_from_errno("calloc");
1415 e6bcace5 2021-06-22 stsp 			goto done;
1416 e6bcace5 2021-06-22 stsp 		}
1417 70f8f24d 2022-04-14 stsp 		arg.idx = -1;
1418 70f8f24d 2022-04-14 stsp 		arg.skip = skip;
1419 70f8f24d 2022-04-14 stsp 		arg.drop = drop;
1420 e6bcace5 2021-06-22 stsp 		err = got_object_idset_for_each(keep, append_id, &arg);
1421 e6bcace5 2021-06-22 stsp 		if (err) {
1422 e6bcace5 2021-06-22 stsp 			free(arg.array);
1423 e6bcace5 2021-06-22 stsp 			goto done;
1424 e6bcace5 2021-06-22 stsp 		}
1425 e6bcace5 2021-06-22 stsp 		*res = arg.array;
1426 70f8f24d 2022-04-14 stsp 		*nres = arg.idx + 1;
1427 e6bcace5 2021-06-22 stsp 	}
1428 e6bcace5 2021-06-22 stsp done:
1429 e6bcace5 2021-06-22 stsp 	got_object_idset_free(keep);
1430 e6bcace5 2021-06-22 stsp 	got_object_idset_free(drop);
1431 70f8f24d 2022-04-14 stsp 	if (skip)
1432 70f8f24d 2022-04-14 stsp 		got_object_idset_free(skip);
1433 e6bcace5 2021-06-22 stsp 	got_object_id_queue_free(&ids);
1434 e6bcace5 2021-06-22 stsp 	return err;
1435 e6bcace5 2021-06-22 stsp }
1436 e6bcace5 2021-06-22 stsp 
1437 e6bcace5 2021-06-22 stsp static const struct got_error *
1438 b8af7c06 2022-03-15 stsp load_object_ids(int *ncolored, int *nfound, int *ntrees,
1439 b8af7c06 2022-03-15 stsp     struct got_object_idset *idset, struct got_object_id **theirs, int ntheirs,
1440 e6bcace5 2021-06-22 stsp     struct got_object_id **ours, int nours, struct got_repository *repo,
1441 b8af7c06 2022-03-15 stsp     int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
1442 b8af7c06 2022-03-15 stsp     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
1443 e6bcace5 2021-06-22 stsp {
1444 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
1445 e6bcace5 2021-06-22 stsp 	struct got_object_id **ids = NULL;
1446 05118f5a 2021-06-22 stsp 	int i, nobj = 0, obj_type;
1447 2f8438b0 2022-05-04 stsp 	struct got_object_idset *idset_exclude;
1448 2f8438b0 2022-05-04 stsp 
1449 2f8438b0 2022-05-04 stsp 	idset_exclude = got_object_idset_alloc();
1450 2f8438b0 2022-05-04 stsp 	if (idset_exclude == NULL)
1451 2f8438b0 2022-05-04 stsp 		return got_error_from_errno("got_object_idset_alloc");
1452 e6bcace5 2021-06-22 stsp 
1453 b8af7c06 2022-03-15 stsp 	*ncolored = 0;
1454 b8af7c06 2022-03-15 stsp 	*nfound = 0;
1455 b8af7c06 2022-03-15 stsp 	*ntrees = 0;
1456 b8af7c06 2022-03-15 stsp 
1457 b8af7c06 2022-03-15 stsp 	err = findtwixt(&ids, &nobj, ncolored, ours, nours, theirs, ntheirs,
1458 b8af7c06 2022-03-15 stsp 	    repo, progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
1459 e6bcace5 2021-06-22 stsp 	if (err || nobj == 0)
1460 e6bcace5 2021-06-22 stsp 		goto done;
1461 e6bcace5 2021-06-22 stsp 
1462 e6bcace5 2021-06-22 stsp 	for (i = 0; i < ntheirs; i++) {
1463 05118f5a 2021-06-22 stsp 		struct got_object_id *id = theirs[i];
1464 05118f5a 2021-06-22 stsp 		if (id == NULL)
1465 05118f5a 2021-06-22 stsp 			continue;
1466 05118f5a 2021-06-22 stsp 		err = got_object_get_type(&obj_type, repo, id);
1467 05118f5a 2021-06-22 stsp 		if (err)
1468 05118f5a 2021-06-22 stsp 			return err;
1469 9d34261e 2022-04-07 stsp 		if (obj_type == GOT_OBJ_TYPE_COMMIT) {
1470 2f8438b0 2022-05-04 stsp 			err = load_commit(0, idset, idset_exclude, id, repo,
1471 9d34261e 2022-04-07 stsp 			    loose_obj_only, ncolored, nfound, ntrees,
1472 9d34261e 2022-04-07 stsp 			    progress_cb, progress_arg, rl,
1473 9d34261e 2022-04-07 stsp 			    cancel_cb, cancel_arg);
1474 07165b17 2021-07-01 stsp 			if (err)
1475 07165b17 2021-07-01 stsp 				goto done;
1476 9d34261e 2022-04-07 stsp 		} else if (obj_type == GOT_OBJ_TYPE_TAG) {
1477 2f8438b0 2022-05-04 stsp 			err = load_tag(0, idset, idset_exclude, id, repo,
1478 9d34261e 2022-04-07 stsp 			    loose_obj_only, ncolored, nfound, ntrees,
1479 9d34261e 2022-04-07 stsp 			    progress_cb, progress_arg, rl,
1480 9d34261e 2022-04-07 stsp 			    cancel_cb, cancel_arg);
1481 9d34261e 2022-04-07 stsp 			if (err)
1482 9d34261e 2022-04-07 stsp 				goto done;
1483 9d34261e 2022-04-07 stsp 		}
1484 05118f5a 2021-06-22 stsp 	}
1485 05118f5a 2021-06-22 stsp 
1486 eca70f98 2021-09-03 stsp 	for (i = 0; i < nobj; i++) {
1487 2f8438b0 2022-05-04 stsp 		err = load_commit(1, idset, idset_exclude,
1488 2f8438b0 2022-05-04 stsp 		    ids[i], repo, loose_obj_only, ncolored, nfound, ntrees,
1489 2f8438b0 2022-05-04 stsp 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
1490 eca70f98 2021-09-03 stsp 		if (err)
1491 eca70f98 2021-09-03 stsp 			goto done;
1492 eca70f98 2021-09-03 stsp 	}
1493 eca70f98 2021-09-03 stsp 
1494 05118f5a 2021-06-22 stsp 	for (i = 0; i < nours; i++) {
1495 05118f5a 2021-06-22 stsp 		struct got_object_id *id = ours[i];
1496 67fd6849 2022-02-13 stsp 		struct got_pack_meta *m;
1497 05118f5a 2021-06-22 stsp 		if (id == NULL)
1498 05118f5a 2021-06-22 stsp 			continue;
1499 67fd6849 2022-02-13 stsp 		m = got_object_idset_get(idset, id);
1500 67fd6849 2022-02-13 stsp 		if (m == NULL) {
1501 07165b17 2021-07-01 stsp 			err = got_object_get_type(&obj_type, repo, id);
1502 07165b17 2021-07-01 stsp 			if (err)
1503 07165b17 2021-07-01 stsp 				goto done;
1504 07165b17 2021-07-01 stsp 		} else
1505 67fd6849 2022-02-13 stsp 			obj_type = m->obj_type;
1506 05118f5a 2021-06-22 stsp 		if (obj_type != GOT_OBJ_TYPE_TAG)
1507 05118f5a 2021-06-22 stsp 			continue;
1508 2f8438b0 2022-05-04 stsp 		err = load_tag(1, idset, idset_exclude, id, repo,
1509 2f8438b0 2022-05-04 stsp 		    loose_obj_only, ncolored, nfound, ntrees,
1510 2f8438b0 2022-05-04 stsp 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
1511 05118f5a 2021-06-22 stsp 		if (err)
1512 e6bcace5 2021-06-22 stsp 			goto done;
1513 e6bcace5 2021-06-22 stsp 	}
1514 e6bcace5 2021-06-22 stsp done:
1515 e6bcace5 2021-06-22 stsp 	for (i = 0; i < nobj; i++) {
1516 e6bcace5 2021-06-22 stsp 		free(ids[i]);
1517 e6bcace5 2021-06-22 stsp 	}
1518 e6bcace5 2021-06-22 stsp 	free(ids);
1519 2f8438b0 2022-05-04 stsp 	got_object_idset_free(idset_exclude);
1520 e6bcace5 2021-06-22 stsp 	return err;
1521 e6bcace5 2021-06-22 stsp }
1522 e6bcace5 2021-06-22 stsp 
1523 e6bcace5 2021-06-22 stsp const struct got_error *
1524 2d9e6abf 2022-05-04 stsp hwrite(FILE *f, void *buf, off_t len, SHA1_CTX *ctx)
1525 e6bcace5 2021-06-22 stsp {
1526 e6bcace5 2021-06-22 stsp 	size_t n;
1527 e6bcace5 2021-06-22 stsp 
1528 e6bcace5 2021-06-22 stsp 	SHA1Update(ctx, buf, len);
1529 e6bcace5 2021-06-22 stsp 	n = fwrite(buf, 1, len, f);
1530 e6bcace5 2021-06-22 stsp 	if (n != len)
1531 e6bcace5 2021-06-22 stsp 		return got_ferror(f, GOT_ERR_IO);
1532 2d9e6abf 2022-05-04 stsp 	return NULL;
1533 2d9e6abf 2022-05-04 stsp }
1534 2d9e6abf 2022-05-04 stsp 
1535 2d9e6abf 2022-05-04 stsp const struct got_error *
1536 2d9e6abf 2022-05-04 stsp hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx)
1537 2d9e6abf 2022-05-04 stsp {
1538 2d9e6abf 2022-05-04 stsp 	unsigned char buf[65536];
1539 2d9e6abf 2022-05-04 stsp 	off_t remain = len;
1540 2d9e6abf 2022-05-04 stsp 	size_t n;
1541 2d9e6abf 2022-05-04 stsp 
1542 2d9e6abf 2022-05-04 stsp 	while (remain > 0) {
1543 2d9e6abf 2022-05-04 stsp 		size_t copylen = MIN(sizeof(buf), remain);
1544 2d9e6abf 2022-05-04 stsp 		n = fread(buf, 1, copylen, fsrc);
1545 2d9e6abf 2022-05-04 stsp 		if (n != copylen)
1546 2d9e6abf 2022-05-04 stsp 			return got_ferror(fsrc, GOT_ERR_IO);
1547 2d9e6abf 2022-05-04 stsp 		SHA1Update(ctx, buf, copylen);
1548 2d9e6abf 2022-05-04 stsp 		n = fwrite(buf, 1, copylen, fdst);
1549 2d9e6abf 2022-05-04 stsp 		if (n != copylen)
1550 2d9e6abf 2022-05-04 stsp 			return got_ferror(fdst, GOT_ERR_IO);
1551 2d9e6abf 2022-05-04 stsp 		remain -= copylen;
1552 2d9e6abf 2022-05-04 stsp 	}
1553 2d9e6abf 2022-05-04 stsp 
1554 e6bcace5 2021-06-22 stsp 	return NULL;
1555 e6bcace5 2021-06-22 stsp }
1556 e6bcace5 2021-06-22 stsp 
1557 e6bcace5 2021-06-22 stsp static void
1558 e6bcace5 2021-06-22 stsp putbe32(char *b, uint32_t n)
1559 e6bcace5 2021-06-22 stsp {
1560 e6bcace5 2021-06-22 stsp 	b[0] = n >> 24;
1561 e6bcace5 2021-06-22 stsp 	b[1] = n >> 16;
1562 e6bcace5 2021-06-22 stsp 	b[2] = n >> 8;
1563 e6bcace5 2021-06-22 stsp 	b[3] = n >> 0;
1564 e6bcace5 2021-06-22 stsp }
1565 e6bcace5 2021-06-22 stsp 
1566 e6bcace5 2021-06-22 stsp static int
1567 e6bcace5 2021-06-22 stsp write_order_cmp(const void *pa, const void *pb)
1568 e6bcace5 2021-06-22 stsp {
1569 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *a, *b, *ahd, *bhd;
1570 e6bcace5 2021-06-22 stsp 
1571 e6bcace5 2021-06-22 stsp 	a = *(struct got_pack_meta **)pa;
1572 e6bcace5 2021-06-22 stsp 	b = *(struct got_pack_meta **)pb;
1573 e6bcace5 2021-06-22 stsp 	ahd = (a->head == NULL) ? a : a->head;
1574 e6bcace5 2021-06-22 stsp 	bhd = (b->head == NULL) ? b : b->head;
1575 611e8e31 2022-05-01 stsp 	if (bhd->mtime < ahd->mtime)
1576 611e8e31 2022-05-01 stsp 		return -1;
1577 611e8e31 2022-05-01 stsp 	if (bhd->mtime > ahd->mtime)
1578 611e8e31 2022-05-01 stsp 		return 1;
1579 611e8e31 2022-05-01 stsp 	if (bhd < ahd)
1580 611e8e31 2022-05-01 stsp 		return -1;
1581 611e8e31 2022-05-01 stsp 	if (bhd > ahd)
1582 611e8e31 2022-05-01 stsp 		return 1;
1583 e6bcace5 2021-06-22 stsp 	if (a->nchain != b->nchain)
1584 e6bcace5 2021-06-22 stsp 		return a->nchain - b->nchain;
1585 611e8e31 2022-05-01 stsp 	if (a->mtime < b->mtime)
1586 611e8e31 2022-05-01 stsp 		return -1;
1587 611e8e31 2022-05-01 stsp 	if (a->mtime > b->mtime)
1588 611e8e31 2022-05-01 stsp 		return 1;
1589 611e8e31 2022-05-01 stsp 	return got_object_id_cmp(&a->id, &b->id);
1590 e6bcace5 2021-06-22 stsp }
1591 e6bcace5 2021-06-22 stsp 
1592 67fd6849 2022-02-13 stsp static int
1593 67fd6849 2022-02-13 stsp reuse_write_order_cmp(const void *pa, const void *pb)
1594 67fd6849 2022-02-13 stsp {
1595 67fd6849 2022-02-13 stsp 	struct got_pack_meta *a, *b;
1596 67fd6849 2022-02-13 stsp 
1597 67fd6849 2022-02-13 stsp 	a = *(struct got_pack_meta **)pa;
1598 67fd6849 2022-02-13 stsp 	b = *(struct got_pack_meta **)pb;
1599 67fd6849 2022-02-13 stsp 
1600 67fd6849 2022-02-13 stsp 	if (a->reused_delta_offset < b->reused_delta_offset)
1601 67fd6849 2022-02-13 stsp 		return -1;
1602 67fd6849 2022-02-13 stsp 	if (a->reused_delta_offset > b->reused_delta_offset)
1603 67fd6849 2022-02-13 stsp 		return 1;
1604 67fd6849 2022-02-13 stsp 	return 0;
1605 67fd6849 2022-02-13 stsp }
1606 67fd6849 2022-02-13 stsp 
1607 e6bcace5 2021-06-22 stsp static const struct got_error *
1608 e6bcace5 2021-06-22 stsp packhdr(int *hdrlen, char *hdr, size_t bufsize, int obj_type, size_t len)
1609 e6bcace5 2021-06-22 stsp {
1610 e6bcace5 2021-06-22 stsp 	size_t i;
1611 e6bcace5 2021-06-22 stsp 
1612 e6bcace5 2021-06-22 stsp 	*hdrlen = 0;
1613 e6bcace5 2021-06-22 stsp 
1614 e6bcace5 2021-06-22 stsp 	hdr[0] = obj_type << 4;
1615 e6bcace5 2021-06-22 stsp 	hdr[0] |= len & 0xf;
1616 e6bcace5 2021-06-22 stsp 	len >>= 4;
1617 e6bcace5 2021-06-22 stsp 	for (i = 1; len != 0; i++){
1618 e6bcace5 2021-06-22 stsp 		if (i >= bufsize)
1619 e6bcace5 2021-06-22 stsp 			return got_error(GOT_ERR_NO_SPACE);
1620 e6bcace5 2021-06-22 stsp 		hdr[i - 1] |= GOT_DELTA_SIZE_MORE;
1621 e6bcace5 2021-06-22 stsp 		hdr[i] = len & GOT_DELTA_SIZE_VAL_MASK;
1622 e6bcace5 2021-06-22 stsp 		len >>= GOT_DELTA_SIZE_SHIFT;
1623 e6bcace5 2021-06-22 stsp 	}
1624 e6bcace5 2021-06-22 stsp 
1625 e6bcace5 2021-06-22 stsp 	*hdrlen = i;
1626 e6bcace5 2021-06-22 stsp 	return NULL;
1627 e6bcace5 2021-06-22 stsp }
1628 e6bcace5 2021-06-22 stsp 
1629 e6bcace5 2021-06-22 stsp static int
1630 e6bcace5 2021-06-22 stsp packoff(char *hdr, off_t off)
1631 e6bcace5 2021-06-22 stsp {
1632 e6bcace5 2021-06-22 stsp 	int i, j;
1633 e6bcace5 2021-06-22 stsp 	char rbuf[8];
1634 e6bcace5 2021-06-22 stsp 
1635 e6bcace5 2021-06-22 stsp 	rbuf[0] = off & GOT_DELTA_SIZE_VAL_MASK;
1636 e6bcace5 2021-06-22 stsp 	for (i = 1; (off >>= GOT_DELTA_SIZE_SHIFT) != 0; i++) {
1637 e6bcace5 2021-06-22 stsp 		rbuf[i] = (--off & GOT_DELTA_SIZE_VAL_MASK) |
1638 e6bcace5 2021-06-22 stsp 		    GOT_DELTA_SIZE_MORE;
1639 e6bcace5 2021-06-22 stsp 	}
1640 e6bcace5 2021-06-22 stsp 
1641 e6bcace5 2021-06-22 stsp 	j = 0;
1642 e6bcace5 2021-06-22 stsp 	while (i > 0)
1643 e6bcace5 2021-06-22 stsp 		hdr[j++] = rbuf[--i];
1644 e6bcace5 2021-06-22 stsp 	return j;
1645 e6bcace5 2021-06-22 stsp }
1646 e6bcace5 2021-06-22 stsp 
1647 e6bcace5 2021-06-22 stsp static const struct got_error *
1648 5060d5a1 2022-01-10 stsp deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *packfile,
1649 67fd6849 2022-02-13 stsp     struct got_pack_meta *m)
1650 5060d5a1 2022-01-10 stsp {
1651 5060d5a1 2022-01-10 stsp 	const struct got_error *err;
1652 5060d5a1 2022-01-10 stsp 	char buf[32];
1653 5060d5a1 2022-01-10 stsp 	int nh;
1654 5060d5a1 2022-01-10 stsp 
1655 67fd6849 2022-02-13 stsp 	if (m->prev->off != 0) {
1656 5060d5a1 2022-01-10 stsp 		err = packhdr(&nh, buf, sizeof(buf),
1657 5060d5a1 2022-01-10 stsp 		    GOT_OBJ_TYPE_OFFSET_DELTA, m->delta_len);
1658 5060d5a1 2022-01-10 stsp 		if (err)
1659 5060d5a1 2022-01-10 stsp 			return err;
1660 5060d5a1 2022-01-10 stsp 		nh += packoff(buf + nh, m->off - m->prev->off);
1661 5060d5a1 2022-01-10 stsp 		err = hwrite(packfile, buf, nh, ctx);
1662 5060d5a1 2022-01-10 stsp 		if (err)
1663 5060d5a1 2022-01-10 stsp 			return err;
1664 5060d5a1 2022-01-10 stsp 		*packfile_size += nh;
1665 5060d5a1 2022-01-10 stsp 	} else {
1666 5060d5a1 2022-01-10 stsp 		err = packhdr(&nh, buf, sizeof(buf),
1667 5060d5a1 2022-01-10 stsp 		    GOT_OBJ_TYPE_REF_DELTA, m->delta_len);
1668 5060d5a1 2022-01-10 stsp 		if (err)
1669 5060d5a1 2022-01-10 stsp 			return err;
1670 5060d5a1 2022-01-10 stsp 		err = hwrite(packfile, buf, nh, ctx);
1671 5060d5a1 2022-01-10 stsp 		if (err)
1672 5060d5a1 2022-01-10 stsp 			return err;
1673 5060d5a1 2022-01-10 stsp 		*packfile_size += nh;
1674 5060d5a1 2022-01-10 stsp 		err = hwrite(packfile, m->prev->id.sha1,
1675 5060d5a1 2022-01-10 stsp 		    sizeof(m->prev->id.sha1), ctx);
1676 5060d5a1 2022-01-10 stsp 		if (err)
1677 5060d5a1 2022-01-10 stsp 			return err;
1678 5060d5a1 2022-01-10 stsp 		*packfile_size += sizeof(m->prev->id.sha1);
1679 5060d5a1 2022-01-10 stsp 	}
1680 5060d5a1 2022-01-10 stsp 
1681 5060d5a1 2022-01-10 stsp 	return NULL;
1682 5060d5a1 2022-01-10 stsp }
1683 5060d5a1 2022-01-10 stsp 
1684 5060d5a1 2022-01-10 stsp static const struct got_error *
1685 67fd6849 2022-02-13 stsp write_packed_object(off_t *packfile_size, FILE *packfile,
1686 67fd6849 2022-02-13 stsp     FILE *delta_cache, struct got_pack_meta *m, int *outfd,
1687 67fd6849 2022-02-13 stsp     SHA1_CTX *ctx, struct got_repository *repo)
1688 67fd6849 2022-02-13 stsp {
1689 67fd6849 2022-02-13 stsp 	const struct got_error *err = NULL;
1690 67fd6849 2022-02-13 stsp 	struct got_deflate_checksum csum;
1691 67fd6849 2022-02-13 stsp 	char buf[32];
1692 67fd6849 2022-02-13 stsp 	int nh;
1693 67fd6849 2022-02-13 stsp 	struct got_raw_object *raw = NULL;
1694 67fd6849 2022-02-13 stsp 	off_t outlen;
1695 67fd6849 2022-02-13 stsp 
1696 67fd6849 2022-02-13 stsp 	csum.output_sha1 = ctx;
1697 67fd6849 2022-02-13 stsp 	csum.output_crc = NULL;
1698 67fd6849 2022-02-13 stsp 
1699 67fd6849 2022-02-13 stsp 	m->off = ftello(packfile);
1700 67fd6849 2022-02-13 stsp 	if (m->delta_len == 0) {
1701 67fd6849 2022-02-13 stsp 		err = got_object_raw_open(&raw, outfd, repo, &m->id);
1702 67fd6849 2022-02-13 stsp 		if (err)
1703 67fd6849 2022-02-13 stsp 			goto done;
1704 67fd6849 2022-02-13 stsp 		err = packhdr(&nh, buf, sizeof(buf),
1705 67fd6849 2022-02-13 stsp 		    m->obj_type, raw->size);
1706 67fd6849 2022-02-13 stsp 		if (err)
1707 67fd6849 2022-02-13 stsp 			goto done;
1708 67fd6849 2022-02-13 stsp 		err = hwrite(packfile, buf, nh, ctx);
1709 67fd6849 2022-02-13 stsp 		if (err)
1710 67fd6849 2022-02-13 stsp 			goto done;
1711 67fd6849 2022-02-13 stsp 		*packfile_size += nh;
1712 67fd6849 2022-02-13 stsp 		if (raw->f == NULL) {
1713 67fd6849 2022-02-13 stsp 			err = got_deflate_to_file_mmap(&outlen,
1714 67fd6849 2022-02-13 stsp 			    raw->data + raw->hdrlen, 0, raw->size,
1715 67fd6849 2022-02-13 stsp 			    packfile, &csum);
1716 67fd6849 2022-02-13 stsp 			if (err)
1717 67fd6849 2022-02-13 stsp 				goto done;
1718 67fd6849 2022-02-13 stsp 		} else {
1719 67fd6849 2022-02-13 stsp 			if (fseeko(raw->f, raw->hdrlen, SEEK_SET)
1720 67fd6849 2022-02-13 stsp 			    == -1) {
1721 67fd6849 2022-02-13 stsp 				err = got_error_from_errno("fseeko");
1722 67fd6849 2022-02-13 stsp 				goto done;
1723 67fd6849 2022-02-13 stsp 			}
1724 67fd6849 2022-02-13 stsp 			err = got_deflate_to_file(&outlen, raw->f,
1725 67fd6849 2022-02-13 stsp 			    raw->size, packfile, &csum);
1726 67fd6849 2022-02-13 stsp 			if (err)
1727 67fd6849 2022-02-13 stsp 				goto done;
1728 67fd6849 2022-02-13 stsp 		}
1729 67fd6849 2022-02-13 stsp 		*packfile_size += outlen;
1730 67fd6849 2022-02-13 stsp 		got_object_raw_close(raw);
1731 67fd6849 2022-02-13 stsp 		raw = NULL;
1732 67fd6849 2022-02-13 stsp 	} else if (m->delta_buf) {
1733 67fd6849 2022-02-13 stsp 		err = deltahdr(packfile_size, ctx, packfile, m);
1734 67fd6849 2022-02-13 stsp 		if (err)
1735 67fd6849 2022-02-13 stsp 			goto done;
1736 2d9e6abf 2022-05-04 stsp 		err = hwrite(packfile, m->delta_buf,
1737 2d9e6abf 2022-05-04 stsp 		    m->delta_compressed_len, ctx);
1738 67fd6849 2022-02-13 stsp 		if (err)
1739 67fd6849 2022-02-13 stsp 			goto done;
1740 2d9e6abf 2022-05-04 stsp 		*packfile_size += m->delta_compressed_len;
1741 67fd6849 2022-02-13 stsp 		free(m->delta_buf);
1742 67fd6849 2022-02-13 stsp 		m->delta_buf = NULL;
1743 67fd6849 2022-02-13 stsp 	} else {
1744 67fd6849 2022-02-13 stsp 		if (fseeko(delta_cache, m->delta_offset, SEEK_SET)
1745 67fd6849 2022-02-13 stsp 		    == -1) {
1746 67fd6849 2022-02-13 stsp 			err = got_error_from_errno("fseeko");
1747 67fd6849 2022-02-13 stsp 			goto done;
1748 67fd6849 2022-02-13 stsp 		}
1749 67fd6849 2022-02-13 stsp 		err = deltahdr(packfile_size, ctx, packfile, m);
1750 67fd6849 2022-02-13 stsp 		if (err)
1751 67fd6849 2022-02-13 stsp 			goto done;
1752 2d9e6abf 2022-05-04 stsp 		err = hcopy(delta_cache, packfile,
1753 2d9e6abf 2022-05-04 stsp 		    m->delta_compressed_len, ctx);
1754 67fd6849 2022-02-13 stsp 		if (err)
1755 67fd6849 2022-02-13 stsp 			goto done;
1756 2d9e6abf 2022-05-04 stsp 		*packfile_size += m->delta_compressed_len;
1757 67fd6849 2022-02-13 stsp 	}
1758 67fd6849 2022-02-13 stsp done:
1759 67fd6849 2022-02-13 stsp 	if (raw)
1760 67fd6849 2022-02-13 stsp 		got_object_raw_close(raw);
1761 67fd6849 2022-02-13 stsp 	return err;
1762 67fd6849 2022-02-13 stsp }
1763 67fd6849 2022-02-13 stsp 
1764 67fd6849 2022-02-13 stsp static const struct got_error *
1765 74881701 2021-10-15 stsp genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
1766 67fd6849 2022-02-13 stsp     struct got_pack_meta **deltify, int ndeltify,
1767 67fd6849 2022-02-13 stsp     struct got_pack_meta **reuse, int nreuse,
1768 b8af7c06 2022-03-15 stsp     int ncolored, int nfound, int ntrees, int nours,
1769 b8af7c06 2022-03-15 stsp     struct got_repository *repo,
1770 05118f5a 2021-06-22 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1771 211cfef0 2022-01-05 stsp     struct got_ratelimit *rl,
1772 05118f5a 2021-06-22 stsp     got_cancel_cb cancel_cb, void *cancel_arg)
1773 e6bcace5 2021-06-22 stsp {
1774 e6bcace5 2021-06-22 stsp 	const struct got_error *err = NULL;
1775 67fd6849 2022-02-13 stsp 	int i;
1776 e6bcace5 2021-06-22 stsp 	SHA1_CTX ctx;
1777 e6bcace5 2021-06-22 stsp 	struct got_pack_meta *m;
1778 08347b73 2021-10-14 stsp 	char buf[32];
1779 72840534 2022-01-19 stsp 	size_t n;
1780 67fd6849 2022-02-13 stsp 	off_t packfile_size = 0;
1781 cc7a354a 2021-10-15 stsp 	int outfd = -1;
1782 e6bcace5 2021-06-22 stsp 
1783 e6bcace5 2021-06-22 stsp 	SHA1Init(&ctx);
1784 e6bcace5 2021-06-22 stsp 
1785 e6bcace5 2021-06-22 stsp 	err = hwrite(packfile, "PACK", 4, &ctx);
1786 e6bcace5 2021-06-22 stsp 	if (err)
1787 e6bcace5 2021-06-22 stsp 		return err;
1788 e6bcace5 2021-06-22 stsp 	putbe32(buf, GOT_PACKFILE_VERSION);
1789 e6bcace5 2021-06-22 stsp 	err = hwrite(packfile, buf, 4, &ctx);
1790 e6bcace5 2021-06-22 stsp 	if (err)
1791 e6bcace5 2021-06-22 stsp 		goto done;
1792 67fd6849 2022-02-13 stsp 	putbe32(buf, ndeltify + nreuse);
1793 e6bcace5 2021-06-22 stsp 	err = hwrite(packfile, buf, 4, &ctx);
1794 e6bcace5 2021-06-22 stsp 	if (err)
1795 e6bcace5 2021-06-22 stsp 		goto done;
1796 67fd6849 2022-02-13 stsp 
1797 67fd6849 2022-02-13 stsp 	qsort(deltify, ndeltify, sizeof(struct got_pack_meta *),
1798 67fd6849 2022-02-13 stsp 	    write_order_cmp);
1799 67fd6849 2022-02-13 stsp 	for (i = 0; i < ndeltify; i++) {
1800 211cfef0 2022-01-05 stsp 		err = report_progress(progress_cb, progress_arg, rl,
1801 b8af7c06 2022-03-15 stsp 		    ncolored, nfound, ntrees, packfile_size, nours,
1802 b8af7c06 2022-03-15 stsp 		    ndeltify + nreuse, ndeltify + nreuse, i);
1803 211cfef0 2022-01-05 stsp 		if (err)
1804 211cfef0 2022-01-05 stsp 			goto done;
1805 67fd6849 2022-02-13 stsp 		m = deltify[i];
1806 67fd6849 2022-02-13 stsp 		err = write_packed_object(&packfile_size, packfile,
1807 67fd6849 2022-02-13 stsp 		    delta_cache, m, &outfd, &ctx, repo);
1808 67fd6849 2022-02-13 stsp 		if (err)
1809 67fd6849 2022-02-13 stsp 			goto done;
1810 e6bcace5 2021-06-22 stsp 	}
1811 67fd6849 2022-02-13 stsp 
1812 67fd6849 2022-02-13 stsp 	qsort(reuse, nreuse, sizeof(struct got_pack_meta *),
1813 67fd6849 2022-02-13 stsp 	    reuse_write_order_cmp);
1814 67fd6849 2022-02-13 stsp 	for (i = 0; i < nreuse; i++) {
1815 67fd6849 2022-02-13 stsp 		err = report_progress(progress_cb, progress_arg, rl,
1816 b8af7c06 2022-03-15 stsp 		    ncolored, nfound, ntrees, packfile_size, nours,
1817 b8af7c06 2022-03-15 stsp 		    ndeltify + nreuse, ndeltify + nreuse, ndeltify + i);
1818 67fd6849 2022-02-13 stsp 		if (err)
1819 67fd6849 2022-02-13 stsp 			goto done;
1820 67fd6849 2022-02-13 stsp 		m = reuse[i];
1821 67fd6849 2022-02-13 stsp 		err = write_packed_object(&packfile_size, packfile,
1822 67fd6849 2022-02-13 stsp 		    delta_cache, m, &outfd, &ctx, repo);
1823 67fd6849 2022-02-13 stsp 		if (err)
1824 67fd6849 2022-02-13 stsp 			goto done;
1825 67fd6849 2022-02-13 stsp 	}
1826 67fd6849 2022-02-13 stsp 
1827 e6bcace5 2021-06-22 stsp 	SHA1Final(pack_sha1, &ctx);
1828 e6bcace5 2021-06-22 stsp 	n = fwrite(pack_sha1, 1, SHA1_DIGEST_LENGTH, packfile);
1829 e6bcace5 2021-06-22 stsp 	if (n != SHA1_DIGEST_LENGTH)
1830 e6bcace5 2021-06-22 stsp 		err = got_ferror(packfile, GOT_ERR_IO);
1831 05118f5a 2021-06-22 stsp 	packfile_size += SHA1_DIGEST_LENGTH;
1832 dc7edd42 2021-08-22 stsp 	packfile_size += sizeof(struct got_packfile_hdr);
1833 211cfef0 2022-01-05 stsp 	if (progress_cb) {
1834 b8af7c06 2022-03-15 stsp 		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
1835 b8af7c06 2022-03-15 stsp 		    packfile_size, nours, ndeltify + nreuse,
1836 b8af7c06 2022-03-15 stsp 		    ndeltify + nreuse, ndeltify + nreuse);
1837 211cfef0 2022-01-05 stsp 		if (err)
1838 211cfef0 2022-01-05 stsp 			goto done;
1839 211cfef0 2022-01-05 stsp 	}
1840 e6bcace5 2021-06-22 stsp done:
1841 cc7a354a 2021-10-15 stsp 	if (outfd != -1 && close(outfd) == -1 && err == NULL)
1842 cc7a354a 2021-10-15 stsp 		err = got_error_from_errno("close");
1843 e6bcace5 2021-06-22 stsp 	return err;
1844 67fd6849 2022-02-13 stsp }
1845 67fd6849 2022-02-13 stsp 
1846 67fd6849 2022-02-13 stsp static const struct got_error *
1847 67fd6849 2022-02-13 stsp add_meta_idset_cb(struct got_object_id *id, void *data, void *arg)
1848 67fd6849 2022-02-13 stsp {
1849 67fd6849 2022-02-13 stsp 	struct got_pack_meta *m = data;
1850 67fd6849 2022-02-13 stsp 	struct got_pack_metavec *v = arg;
1851 f5e78e05 2022-05-04 stsp 
1852 f5e78e05 2022-05-04 stsp 	if (m->have_reused_delta)
1853 f5e78e05 2022-05-04 stsp 		return NULL;
1854 67fd6849 2022-02-13 stsp 
1855 67fd6849 2022-02-13 stsp 	return add_meta(m, v);
1856 67fd6849 2022-02-13 stsp }
1857 67fd6849 2022-02-13 stsp 
1858 e6bcace5 2021-06-22 stsp const struct got_error *
1859 e6bcace5 2021-06-22 stsp got_pack_create(uint8_t *packsha1, FILE *packfile,
1860 e6bcace5 2021-06-22 stsp     struct got_object_id **theirs, int ntheirs,
1861 e6bcace5 2021-06-22 stsp     struct got_object_id **ours, int nours,
1862 f8a36e22 2021-08-26 stsp     struct got_repository *repo, int loose_obj_only, int allow_empty,
1863 05118f5a 2021-06-22 stsp     got_pack_progress_cb progress_cb, void *progress_arg,
1864 05118f5a 2021-06-22 stsp     got_cancel_cb cancel_cb, void *cancel_arg)
1865 e6bcace5 2021-06-22 stsp {
1866 e6bcace5 2021-06-22 stsp 	const struct got_error *err;
1867 67fd6849 2022-02-13 stsp 	int delta_cache_fd = -1;
1868 74881701 2021-10-15 stsp 	FILE *delta_cache = NULL;
1869 67fd6849 2022-02-13 stsp 	struct got_object_idset *idset;
1870 211cfef0 2022-01-05 stsp 	struct got_ratelimit rl;
1871 67fd6849 2022-02-13 stsp 	struct got_pack_metavec deltify, reuse;
1872 b8af7c06 2022-03-15 stsp 	int ncolored = 0, nfound = 0, ntrees = 0;
1873 f5e78e05 2022-05-04 stsp 	size_t ndeltify;
1874 e6bcace5 2021-06-22 stsp 
1875 67fd6849 2022-02-13 stsp 	memset(&deltify, 0, sizeof(deltify));
1876 67fd6849 2022-02-13 stsp 	memset(&reuse, 0, sizeof(reuse));
1877 67fd6849 2022-02-13 stsp 
1878 211cfef0 2022-01-05 stsp 	got_ratelimit_init(&rl, 0, 500);
1879 211cfef0 2022-01-05 stsp 
1880 67fd6849 2022-02-13 stsp 	idset = got_object_idset_alloc();
1881 67fd6849 2022-02-13 stsp 	if (idset == NULL)
1882 67fd6849 2022-02-13 stsp 		return got_error_from_errno("got_object_idset_alloc");
1883 67fd6849 2022-02-13 stsp 
1884 b8af7c06 2022-03-15 stsp 	err = load_object_ids(&ncolored, &nfound, &ntrees, idset, theirs,
1885 b8af7c06 2022-03-15 stsp 	    ntheirs, ours, nours, repo, loose_obj_only,
1886 b8af7c06 2022-03-15 stsp 	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
1887 e6bcace5 2021-06-22 stsp 	if (err)
1888 e6bcace5 2021-06-22 stsp 		return err;
1889 e6bcace5 2021-06-22 stsp 
1890 28526235 2022-02-13 stsp 	if (progress_cb) {
1891 b8af7c06 2022-03-15 stsp 		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
1892 b8af7c06 2022-03-15 stsp 		    0L, nours, got_object_idset_num_elements(idset), 0, 0);
1893 28526235 2022-02-13 stsp 		if (err)
1894 28526235 2022-02-13 stsp 			goto done;
1895 28526235 2022-02-13 stsp 	}
1896 28526235 2022-02-13 stsp 
1897 67fd6849 2022-02-13 stsp 	if (got_object_idset_num_elements(idset) == 0 && !allow_empty) {
1898 05118f5a 2021-06-22 stsp 		err = got_error(GOT_ERR_CANNOT_PACK);
1899 05118f5a 2021-06-22 stsp 		goto done;
1900 05118f5a 2021-06-22 stsp 	}
1901 74881701 2021-10-15 stsp 
1902 67fd6849 2022-02-13 stsp 	delta_cache_fd = got_opentempfd();
1903 67fd6849 2022-02-13 stsp 	if (delta_cache_fd == -1) {
1904 74881701 2021-10-15 stsp 		err = got_error_from_errno("got_opentemp");
1905 74881701 2021-10-15 stsp 		goto done;
1906 74881701 2021-10-15 stsp 	}
1907 74881701 2021-10-15 stsp 
1908 67fd6849 2022-02-13 stsp 	reuse.metasz = 64;
1909 67fd6849 2022-02-13 stsp 	reuse.meta = calloc(reuse.metasz,
1910 67fd6849 2022-02-13 stsp 	    sizeof(struct got_pack_meta *));
1911 67fd6849 2022-02-13 stsp 	if (reuse.meta == NULL) {
1912 67fd6849 2022-02-13 stsp 		err = got_error_from_errno("calloc");
1913 67fd6849 2022-02-13 stsp 		goto done;
1914 67fd6849 2022-02-13 stsp 	}
1915 67fd6849 2022-02-13 stsp 
1916 b8af7c06 2022-03-15 stsp 	err = search_deltas(&reuse, idset, delta_cache_fd, ncolored, nfound,
1917 b8af7c06 2022-03-15 stsp 	    ntrees, nours, repo, progress_cb, progress_arg, &rl,
1918 b8af7c06 2022-03-15 stsp 	    cancel_cb, cancel_arg);
1919 67fd6849 2022-02-13 stsp 	if (err)
1920 67fd6849 2022-02-13 stsp 		goto done;
1921 67fd6849 2022-02-13 stsp 
1922 67fd6849 2022-02-13 stsp 	delta_cache = fdopen(delta_cache_fd, "a+");
1923 67fd6849 2022-02-13 stsp 	if (delta_cache == NULL) {
1924 67fd6849 2022-02-13 stsp 		err = got_error_from_errno("fdopen");
1925 67fd6849 2022-02-13 stsp 		goto done;
1926 67fd6849 2022-02-13 stsp 	}
1927 67fd6849 2022-02-13 stsp 	delta_cache_fd = -1;
1928 67fd6849 2022-02-13 stsp 
1929 67fd6849 2022-02-13 stsp 	if (fseeko(delta_cache, 0L, SEEK_END) == -1) {
1930 67fd6849 2022-02-13 stsp 		err = got_error_from_errno("fseeko");
1931 67fd6849 2022-02-13 stsp 		goto done;
1932 67fd6849 2022-02-13 stsp 	}
1933 67fd6849 2022-02-13 stsp 
1934 f5e78e05 2022-05-04 stsp 	ndeltify = got_object_idset_num_elements(idset) - reuse.nmeta;
1935 f5e78e05 2022-05-04 stsp 	if (ndeltify > 0) {
1936 f5e78e05 2022-05-04 stsp 		deltify.meta = calloc(ndeltify, sizeof(struct got_pack_meta *));
1937 f5e78e05 2022-05-04 stsp 		if (deltify.meta == NULL) {
1938 f5e78e05 2022-05-04 stsp 			err = got_error_from_errno("calloc");
1939 f5e78e05 2022-05-04 stsp 			goto done;
1940 f5e78e05 2022-05-04 stsp 		}
1941 f5e78e05 2022-05-04 stsp 		deltify.metasz = ndeltify;
1942 67fd6849 2022-02-13 stsp 
1943 f5e78e05 2022-05-04 stsp 		err = got_object_idset_for_each(idset, add_meta_idset_cb,
1944 f5e78e05 2022-05-04 stsp 		    &deltify);
1945 f8a36e22 2021-08-26 stsp 		if (err)
1946 f8a36e22 2021-08-26 stsp 			goto done;
1947 f5e78e05 2022-05-04 stsp 		if (deltify.nmeta > 0) {
1948 f5e78e05 2022-05-04 stsp 			err = pick_deltas(deltify.meta, deltify.nmeta,
1949 f5e78e05 2022-05-04 stsp 			    ncolored, nfound, ntrees, nours, reuse.nmeta,
1950 f5e78e05 2022-05-04 stsp 			    delta_cache, repo, progress_cb, progress_arg, &rl,
1951 f5e78e05 2022-05-04 stsp 			    cancel_cb, cancel_arg);
1952 f5e78e05 2022-05-04 stsp 			if (err)
1953 f5e78e05 2022-05-04 stsp 				goto done;
1954 f5e78e05 2022-05-04 stsp 		}
1955 f8a36e22 2021-08-26 stsp 	}
1956 e6bcace5 2021-06-22 stsp 
1957 2d9e6abf 2022-05-04 stsp 	if (fflush(delta_cache) == EOF) {
1958 2d9e6abf 2022-05-04 stsp 		err = got_error_from_errno("fflush");
1959 2d9e6abf 2022-05-04 stsp 		goto done;
1960 2d9e6abf 2022-05-04 stsp 	}
1961 67fd6849 2022-02-13 stsp 	err = genpack(packsha1, packfile, delta_cache, deltify.meta,
1962 b8af7c06 2022-03-15 stsp 	    deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees,
1963 b8af7c06 2022-03-15 stsp 	    nours, repo, progress_cb, progress_arg, &rl,
1964 b8af7c06 2022-03-15 stsp 	    cancel_cb, cancel_arg);
1965 e6bcace5 2021-06-22 stsp 	if (err)
1966 e6bcace5 2021-06-22 stsp 		goto done;
1967 e6bcace5 2021-06-22 stsp done:
1968 67fd6849 2022-02-13 stsp 	free_nmeta(deltify.meta, deltify.nmeta);
1969 67fd6849 2022-02-13 stsp 	free_nmeta(reuse.meta, reuse.nmeta);
1970 67fd6849 2022-02-13 stsp 	got_object_idset_free(idset);
1971 67fd6849 2022-02-13 stsp 	if (delta_cache_fd != -1 && close(delta_cache_fd) == -1 && err == NULL)
1972 67fd6849 2022-02-13 stsp 		err = got_error_from_errno("close");
1973 74881701 2021-10-15 stsp 	if (delta_cache && fclose(delta_cache) == EOF && err == NULL)
1974 74881701 2021-10-15 stsp 		err = got_error_from_errno("fclose");
1975 e6bcace5 2021-06-22 stsp 	return err;
1976 e6bcace5 2021-06-22 stsp }