Gotweb

Blame

Date:: Wed Mar 8 09:27:14 2023 UTC
Message:: portable: configure: split out dependencies Rather than assume all dependencies are required for all programs, split them out. This will make packaging easier, as well as splitting the code to use subprojects. Note that due to the use of config.h semantics, in most cases the got_compat.h header file is now at the top of the .c file it is included in, so that it can handle the system header inclusion properly.
Actions:: History | Blob | Raw File
  1 142012ed 2022-10-20 thomas /*
  2 142012ed 2022-10-20 thomas  * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
  3 142012ed 2022-10-20 thomas  * Copyright (c) 2020, 2022 Stefan Sperling <stsp@openbsd.org>
  4 142012ed 2022-10-20 thomas  *
  5 142012ed 2022-10-20 thomas  * Permission to use, copy, modify, and distribute this software for any
  6 142012ed 2022-10-20 thomas  * purpose with or without fee is hereby granted, provided that the above
  7 142012ed 2022-10-20 thomas  * copyright notice and this permission notice appear in all copies.
  8 142012ed 2022-10-20 thomas  *
  9 142012ed 2022-10-20 thomas  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 10 142012ed 2022-10-20 thomas  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 11 142012ed 2022-10-20 thomas  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 12 142012ed 2022-10-20 thomas  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 13 142012ed 2022-10-20 thomas  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 14 142012ed 2022-10-20 thomas  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 15 142012ed 2022-10-20 thomas  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 16 142012ed 2022-10-20 thomas  */
 17 4fccd2fe 2023-03-08 thomas 
 18 4fccd2fe 2023-03-08 thomas #include "got_compat.h"
 19 142012ed 2022-10-20 thomas 
 20 142012ed 2022-10-20 thomas #include <sys/queue.h>
 21 142012ed 2022-10-20 thomas #include <sys/stat.h>
 22 142012ed 2022-10-20 thomas #include <sys/time.h>
 23 142012ed 2022-10-20 thomas #include <sys/types.h>
 24 142012ed 2022-10-20 thomas #include <sys/uio.h>
 25 142012ed 2022-10-20 thomas #include <sys/mman.h>
 26 142012ed 2022-10-20 thomas 
 27 142012ed 2022-10-20 thomas #include <stdint.h>
 28 142012ed 2022-10-20 thomas #include <errno.h>
 29 142012ed 2022-10-20 thomas #include <imsg.h>
 30 142012ed 2022-10-20 thomas #include <limits.h>
 31 142012ed 2022-10-20 thomas #include <signal.h>
 32 142012ed 2022-10-20 thomas #include <stdio.h>
 33 142012ed 2022-10-20 thomas #include <stdlib.h>
 34 142012ed 2022-10-20 thomas #include <string.h>
 35 142012ed 2022-10-20 thomas #include <ctype.h>
 36 142012ed 2022-10-20 thomas #include <fcntl.h>
 37 142012ed 2022-10-20 thomas #include <unistd.h>
 38 142012ed 2022-10-20 thomas #include <zlib.h>
 39 142012ed 2022-10-20 thomas #include <err.h>
 40 142012ed 2022-10-20 thomas #include <assert.h>
 41 142012ed 2022-10-20 thomas #include <dirent.h>
 42 142012ed 2022-10-20 thomas 
 43 142012ed 2022-10-20 thomas #include "got_error.h"
 44 142012ed 2022-10-20 thomas #include "got_object.h"
 45 142012ed 2022-10-20 thomas 
 46 be288a59 2023-02-23 thomas #include "got_lib_hash.h"
 47 142012ed 2022-10-20 thomas #include "got_lib_delta.h"
 48 142012ed 2022-10-20 thomas #include "got_lib_inflate.h"
 49 142012ed 2022-10-20 thomas #include "got_lib_object.h"
 50 142012ed 2022-10-20 thomas #include "got_lib_object_parse.h"
 51 142012ed 2022-10-20 thomas #include "got_lib_object_idset.h"
 52 142012ed 2022-10-20 thomas #include "got_lib_privsep.h"
 53 142012ed 2022-10-20 thomas #include "got_lib_pack.h"
 54 142012ed 2022-10-20 thomas #include "got_lib_ratelimit.h"
 55 142012ed 2022-10-20 thomas #include "got_lib_pack_index.h"
 56 142012ed 2022-10-20 thomas #include "got_lib_delta_cache.h"
 57 142012ed 2022-10-20 thomas 
 58 142012ed 2022-10-20 thomas struct got_indexed_object {
 59 142012ed 2022-10-20 thomas 	struct got_object_id id;
 60 142012ed 2022-10-20 thomas 
 61 142012ed 2022-10-20 thomas 	/*
 62 142012ed 2022-10-20 thomas 	 * Has this object been fully resolved?
 63 142012ed 2022-10-20 thomas 	 * If so, we know its ID, otherwise we don't and 'id' is invalid.
 64 142012ed 2022-10-20 thomas 	 */
 65 142012ed 2022-10-20 thomas 	int valid;
 66 142012ed 2022-10-20 thomas 
 67 142012ed 2022-10-20 thomas 	/* Offset of type+size field for this object in pack file. */
 68 142012ed 2022-10-20 thomas 	off_t off;
 69 142012ed 2022-10-20 thomas 
 70 142012ed 2022-10-20 thomas 	/* Type+size values parsed from pack file. */
 71 142012ed 2022-10-20 thomas 	uint8_t type;
 72 142012ed 2022-10-20 thomas 	uint64_t size;
 73 142012ed 2022-10-20 thomas 
 74 142012ed 2022-10-20 thomas 	/* Length of on-disk type+size data. */
 75 142012ed 2022-10-20 thomas 	size_t tslen;
 76 142012ed 2022-10-20 thomas 
 77 142012ed 2022-10-20 thomas 	/* Length of object data following type+size. */
 78 142012ed 2022-10-20 thomas 	size_t len;
 79 142012ed 2022-10-20 thomas 
 80 142012ed 2022-10-20 thomas 	uint32_t crc;
 81 142012ed 2022-10-20 thomas 
 82 142012ed 2022-10-20 thomas 	union {
 83 142012ed 2022-10-20 thomas 		struct {
 84 142012ed 2022-10-20 thomas 			/* For ref deltas. */
 85 142012ed 2022-10-20 thomas 			struct got_object_id ref_id;
 86 142012ed 2022-10-20 thomas 		} ref;
 87 142012ed 2022-10-20 thomas 		struct {
 88 142012ed 2022-10-20 thomas 			/* For offset deltas. */
 89 142012ed 2022-10-20 thomas 			off_t base_offset;
 90 142012ed 2022-10-20 thomas 			size_t base_offsetlen;
 91 142012ed 2022-10-20 thomas 		} ofs;
 92 142012ed 2022-10-20 thomas 	} delta;
 93 142012ed 2022-10-20 thomas };
 94 142012ed 2022-10-20 thomas 
 95 142012ed 2022-10-20 thomas static void
 96 142012ed 2022-10-20 thomas putbe32(char *b, uint32_t n)
 97 142012ed 2022-10-20 thomas {
 98 142012ed 2022-10-20 thomas 	b[0] = n >> 24;
 99 142012ed 2022-10-20 thomas 	b[1] = n >> 16;
100 142012ed 2022-10-20 thomas 	b[2] = n >> 8;
101 142012ed 2022-10-20 thomas 	b[3] = n >> 0;
102 142012ed 2022-10-20 thomas }
103 142012ed 2022-10-20 thomas 
104 142012ed 2022-10-20 thomas static const struct got_error *
105 142012ed 2022-10-20 thomas get_obj_type_label(const char **label, int obj_type)
106 142012ed 2022-10-20 thomas {
107 142012ed 2022-10-20 thomas 	const struct got_error *err = NULL;
108 142012ed 2022-10-20 thomas 
109 142012ed 2022-10-20 thomas 	switch (obj_type) {
110 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_BLOB:
111 142012ed 2022-10-20 thomas 		*label = GOT_OBJ_LABEL_BLOB;
112 142012ed 2022-10-20 thomas 		break;
113 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_TREE:
114 142012ed 2022-10-20 thomas 		*label = GOT_OBJ_LABEL_TREE;
115 142012ed 2022-10-20 thomas 		break;
116 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_COMMIT:
117 142012ed 2022-10-20 thomas 		*label = GOT_OBJ_LABEL_COMMIT;
118 142012ed 2022-10-20 thomas 		break;
119 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_TAG:
120 142012ed 2022-10-20 thomas 		*label = GOT_OBJ_LABEL_TAG;
121 142012ed 2022-10-20 thomas 		break;
122 142012ed 2022-10-20 thomas 	default:
123 142012ed 2022-10-20 thomas 		*label = NULL;
124 142012ed 2022-10-20 thomas 		err = got_error(GOT_ERR_OBJ_TYPE);
125 142012ed 2022-10-20 thomas 		break;
126 142012ed 2022-10-20 thomas 	}
127 142012ed 2022-10-20 thomas 
128 142012ed 2022-10-20 thomas 	return err;
129 142012ed 2022-10-20 thomas }
130 142012ed 2022-10-20 thomas 
131 142012ed 2022-10-20 thomas static const struct got_error *
132 b16893ba 2023-02-24 thomas read_checksum(uint32_t *crc, struct got_hash *ctx, int fd, size_t len)
133 142012ed 2022-10-20 thomas {
134 142012ed 2022-10-20 thomas 	uint8_t buf[8192];
135 142012ed 2022-10-20 thomas 	size_t n;
136 142012ed 2022-10-20 thomas 	ssize_t r;
137 142012ed 2022-10-20 thomas 
138 142012ed 2022-10-20 thomas 	for (n = len; n > 0; n -= r){
139 142012ed 2022-10-20 thomas 		r = read(fd, buf, n > sizeof(buf) ? sizeof(buf) : n);
140 142012ed 2022-10-20 thomas 		if (r == -1)
141 142012ed 2022-10-20 thomas 			return got_error_from_errno("read");
142 142012ed 2022-10-20 thomas 		if (r == 0)
143 142012ed 2022-10-20 thomas 			break;
144 142012ed 2022-10-20 thomas 		if (crc)
145 142012ed 2022-10-20 thomas 			*crc = crc32(*crc, buf, r);
146 b16893ba 2023-02-24 thomas 		if (ctx)
147 b16893ba 2023-02-24 thomas 			got_hash_update(ctx, buf, r);
148 142012ed 2022-10-20 thomas 	}
149 142012ed 2022-10-20 thomas 
150 142012ed 2022-10-20 thomas 	return NULL;
151 142012ed 2022-10-20 thomas }
152 142012ed 2022-10-20 thomas 
153 142012ed 2022-10-20 thomas static const struct got_error *
154 b16893ba 2023-02-24 thomas read_file_digest(struct got_hash *ctx, FILE *f, size_t len)
155 142012ed 2022-10-20 thomas {
156 142012ed 2022-10-20 thomas 	uint8_t buf[8192];
157 142012ed 2022-10-20 thomas 	size_t n, r;
158 142012ed 2022-10-20 thomas 
159 142012ed 2022-10-20 thomas 	for (n = len; n > 0; n -= r) {
160 142012ed 2022-10-20 thomas 		r = fread(buf, 1, n > sizeof(buf) ? sizeof(buf) : n, f);
161 142012ed 2022-10-20 thomas 		if (r == 0) {
162 142012ed 2022-10-20 thomas 			if (feof(f))
163 142012ed 2022-10-20 thomas 				return NULL;
164 142012ed 2022-10-20 thomas 			return got_ferror(f, GOT_ERR_IO);
165 142012ed 2022-10-20 thomas 		}
166 b16893ba 2023-02-24 thomas 		got_hash_update(ctx, buf, r);
167 142012ed 2022-10-20 thomas 	}
168 142012ed 2022-10-20 thomas 
169 142012ed 2022-10-20 thomas 	return NULL;
170 142012ed 2022-10-20 thomas }
171 142012ed 2022-10-20 thomas 
172 142012ed 2022-10-20 thomas static const struct got_error *
173 142012ed 2022-10-20 thomas read_packed_object(struct got_pack *pack, struct got_indexed_object *obj,
174 b16893ba 2023-02-24 thomas     FILE *tmpfile, struct got_hash *pack_sha1_ctx)
175 142012ed 2022-10-20 thomas {
176 142012ed 2022-10-20 thomas 	const struct got_error *err = NULL;
177 b16893ba 2023-02-24 thomas 	struct got_hash ctx;
178 142012ed 2022-10-20 thomas 	uint8_t *data = NULL;
179 142012ed 2022-10-20 thomas 	size_t datalen = 0;
180 142012ed 2022-10-20 thomas 	ssize_t n;
181 142012ed 2022-10-20 thomas 	char *header;
182 142012ed 2022-10-20 thomas 	size_t headerlen;
183 142012ed 2022-10-20 thomas 	const char *obj_label;
184 142012ed 2022-10-20 thomas 	size_t mapoff = obj->off;
185 142012ed 2022-10-20 thomas 	struct got_inflate_checksum csum;
186 142012ed 2022-10-20 thomas 
187 142012ed 2022-10-20 thomas 	memset(&csum, 0, sizeof(csum));
188 b16893ba 2023-02-24 thomas 	csum.input_ctx = pack_sha1_ctx;
189 142012ed 2022-10-20 thomas 	csum.input_crc = &obj->crc;
190 142012ed 2022-10-20 thomas 
191 142012ed 2022-10-20 thomas 	err = got_pack_parse_object_type_and_size(&obj->type, &obj->size,
192 142012ed 2022-10-20 thomas 	    &obj->tslen, pack, obj->off);
193 142012ed 2022-10-20 thomas 	if (err)
194 142012ed 2022-10-20 thomas 		return err;
195 142012ed 2022-10-20 thomas 
196 142012ed 2022-10-20 thomas 	if (pack->map) {
197 142012ed 2022-10-20 thomas 		obj->crc = crc32(obj->crc, pack->map + mapoff, obj->tslen);
198 b16893ba 2023-02-24 thomas 		got_hash_update(pack_sha1_ctx, pack->map + mapoff, obj->tslen);
199 142012ed 2022-10-20 thomas 		mapoff += obj->tslen;
200 142012ed 2022-10-20 thomas 	} else {
201 142012ed 2022-10-20 thomas 		/* XXX Seek back and get the CRC of on-disk type+size bytes. */
202 142012ed 2022-10-20 thomas 		if (lseek(pack->fd, obj->off, SEEK_SET) == -1)
203 142012ed 2022-10-20 thomas 			return got_error_from_errno("lseek");
204 142012ed 2022-10-20 thomas 		err = read_checksum(&obj->crc, pack_sha1_ctx,
205 142012ed 2022-10-20 thomas 		    pack->fd, obj->tslen);
206 142012ed 2022-10-20 thomas 		if (err)
207 142012ed 2022-10-20 thomas 			return err;
208 142012ed 2022-10-20 thomas 	}
209 142012ed 2022-10-20 thomas 
210 142012ed 2022-10-20 thomas 	switch (obj->type) {
211 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_BLOB:
212 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_COMMIT:
213 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_TREE:
214 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_TAG:
215 142012ed 2022-10-20 thomas 		if (obj->size > GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
216 142012ed 2022-10-20 thomas 			if (fseek(tmpfile, 0L, SEEK_SET) == -1) {
217 142012ed 2022-10-20 thomas 				err = got_error_from_errno("fseek");
218 142012ed 2022-10-20 thomas 				break;
219 142012ed 2022-10-20 thomas 			}
220 142012ed 2022-10-20 thomas 			if (pack->map) {
221 142012ed 2022-10-20 thomas 				err = got_inflate_to_file_mmap(&datalen,
222 142012ed 2022-10-20 thomas 				    &obj->len, &csum, pack->map, mapoff,
223 142012ed 2022-10-20 thomas 				    pack->filesize - mapoff, tmpfile);
224 142012ed 2022-10-20 thomas 			} else {
225 142012ed 2022-10-20 thomas 				err = got_inflate_to_file_fd(&datalen,
226 142012ed 2022-10-20 thomas 				    &obj->len, &csum, pack->fd, tmpfile);
227 142012ed 2022-10-20 thomas 			}
228 142012ed 2022-10-20 thomas 		} else {
229 142012ed 2022-10-20 thomas 			if (pack->map) {
230 142012ed 2022-10-20 thomas 				err = got_inflate_to_mem_mmap(&data, &datalen,
231 142012ed 2022-10-20 thomas 				    &obj->len, &csum, pack->map, mapoff,
232 142012ed 2022-10-20 thomas 				    pack->filesize - mapoff);
233 142012ed 2022-10-20 thomas 			} else {
234 142012ed 2022-10-20 thomas 				err = got_inflate_to_mem_fd(&data, &datalen,
235 142012ed 2022-10-20 thomas 				    &obj->len, &csum, obj->size, pack->fd);
236 142012ed 2022-10-20 thomas 			}
237 142012ed 2022-10-20 thomas 		}
238 142012ed 2022-10-20 thomas 		if (err)
239 142012ed 2022-10-20 thomas 			break;
240 b16893ba 2023-02-24 thomas 		got_hash_init(&ctx, GOT_HASH_SHA1);
241 142012ed 2022-10-20 thomas 		err = get_obj_type_label(&obj_label, obj->type);
242 142012ed 2022-10-20 thomas 		if (err) {
243 142012ed 2022-10-20 thomas 			free(data);
244 142012ed 2022-10-20 thomas 			break;
245 142012ed 2022-10-20 thomas 		}
246 142012ed 2022-10-20 thomas 		if (asprintf(&header, "%s %lld", obj_label,
247 142012ed 2022-10-20 thomas 		    (long long)obj->size) == -1) {
248 142012ed 2022-10-20 thomas 			err = got_error_from_errno("asprintf");
249 142012ed 2022-10-20 thomas 			free(data);
250 142012ed 2022-10-20 thomas 			break;
251 142012ed 2022-10-20 thomas 		}
252 142012ed 2022-10-20 thomas 		headerlen = strlen(header) + 1;
253 b16893ba 2023-02-24 thomas 		got_hash_update(&ctx, header, headerlen);
254 142012ed 2022-10-20 thomas 		if (obj->size > GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
255 b16893ba 2023-02-24 thomas 			err = read_file_digest(&ctx, tmpfile, datalen);
256 142012ed 2022-10-20 thomas 			if (err) {
257 142012ed 2022-10-20 thomas 				free(header);
258 142012ed 2022-10-20 thomas 				free(data);
259 142012ed 2022-10-20 thomas 				break;
260 142012ed 2022-10-20 thomas 			}
261 142012ed 2022-10-20 thomas 		} else
262 b16893ba 2023-02-24 thomas 			got_hash_update(&ctx, data, datalen);
263 b16893ba 2023-02-24 thomas 		got_hash_final_object_id(&ctx, &obj->id);
264 142012ed 2022-10-20 thomas 		free(header);
265 142012ed 2022-10-20 thomas 		free(data);
266 142012ed 2022-10-20 thomas 		break;
267 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_REF_DELTA:
268 142012ed 2022-10-20 thomas 		memset(obj->id.sha1, 0xff, SHA1_DIGEST_LENGTH);
269 142012ed 2022-10-20 thomas 		if (pack->map) {
270 142012ed 2022-10-20 thomas 			if (mapoff + SHA1_DIGEST_LENGTH >= pack->filesize) {
271 142012ed 2022-10-20 thomas 				err = got_error(GOT_ERR_BAD_PACKFILE);
272 66e6097f 2022-10-27 thomas 				break;
273 66e6097f 2022-10-27 thomas 			}
274 66e6097f 2022-10-27 thomas 			if (mapoff + SHA1_DIGEST_LENGTH > SIZE_MAX) {
275 66e6097f 2022-10-27 thomas 				err = got_error_fmt(GOT_ERR_RANGE,
276 66e6097f 2022-10-27 thomas 				    "mapoff %lld would overflow size_t",
277 66e6097f 2022-10-27 thomas 				    (long long)mapoff + SHA1_DIGEST_LENGTH);
278 142012ed 2022-10-20 thomas 				break;
279 142012ed 2022-10-20 thomas 			}
280 142012ed 2022-10-20 thomas 			memcpy(obj->delta.ref.ref_id.sha1, pack->map + mapoff,
281 142012ed 2022-10-20 thomas 			    SHA1_DIGEST_LENGTH);
282 142012ed 2022-10-20 thomas 			obj->crc = crc32(obj->crc, pack->map + mapoff,
283 142012ed 2022-10-20 thomas 			    SHA1_DIGEST_LENGTH);
284 b16893ba 2023-02-24 thomas 			got_hash_update(pack_sha1_ctx, pack->map + mapoff,
285 142012ed 2022-10-20 thomas 			    SHA1_DIGEST_LENGTH);
286 142012ed 2022-10-20 thomas 			mapoff += SHA1_DIGEST_LENGTH;
287 142012ed 2022-10-20 thomas 			err = got_inflate_to_mem_mmap(NULL, &datalen,
288 142012ed 2022-10-20 thomas 			    &obj->len, &csum, pack->map, mapoff,
289 142012ed 2022-10-20 thomas 			    pack->filesize - mapoff);
290 142012ed 2022-10-20 thomas 			if (err)
291 142012ed 2022-10-20 thomas 				break;
292 142012ed 2022-10-20 thomas 		} else {
293 142012ed 2022-10-20 thomas 			n = read(pack->fd, obj->delta.ref.ref_id.sha1,
294 142012ed 2022-10-20 thomas 			    SHA1_DIGEST_LENGTH);
295 142012ed 2022-10-20 thomas 			if (n == -1) {
296 142012ed 2022-10-20 thomas 				err = got_error_from_errno("read");
297 142012ed 2022-10-20 thomas 				break;
298 142012ed 2022-10-20 thomas 			}
299 142012ed 2022-10-20 thomas 			if (n < sizeof(obj->id)) {
300 142012ed 2022-10-20 thomas 				err = got_error(GOT_ERR_BAD_PACKFILE);
301 142012ed 2022-10-20 thomas 				break;
302 142012ed 2022-10-20 thomas 			}
303 142012ed 2022-10-20 thomas 			obj->crc = crc32(obj->crc, obj->delta.ref.ref_id.sha1,
304 142012ed 2022-10-20 thomas 			    SHA1_DIGEST_LENGTH);
305 b16893ba 2023-02-24 thomas 			got_hash_update(pack_sha1_ctx,
306 b16893ba 2023-02-24 thomas 			    obj->delta.ref.ref_id.sha1, SHA1_DIGEST_LENGTH);
307 142012ed 2022-10-20 thomas 			err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
308 142012ed 2022-10-20 thomas 			    &csum, obj->size, pack->fd);
309 142012ed 2022-10-20 thomas 			if (err)
310 142012ed 2022-10-20 thomas 				break;
311 142012ed 2022-10-20 thomas 		}
312 142012ed 2022-10-20 thomas 		obj->len += SHA1_DIGEST_LENGTH;
313 142012ed 2022-10-20 thomas 		break;
314 142012ed 2022-10-20 thomas 	case GOT_OBJ_TYPE_OFFSET_DELTA:
315 142012ed 2022-10-20 thomas 		memset(obj->id.sha1, 0xff, SHA1_DIGEST_LENGTH);
316 142012ed 2022-10-20 thomas 		err = got_pack_parse_offset_delta(&obj->delta.ofs.base_offset,
317 142012ed 2022-10-20 thomas 		    &obj->delta.ofs.base_offsetlen, pack, obj->off,
318 142012ed 2022-10-20 thomas 		    obj->tslen);
319 142012ed 2022-10-20 thomas 		if (err)
320 142012ed 2022-10-20 thomas 			break;
321 142012ed 2022-10-20 thomas 
322 142012ed 2022-10-20 thomas 		if (pack->map) {
323 6cf83e6a 2022-10-25 thomas 			if (mapoff + obj->delta.ofs.base_offsetlen >=
324 6cf83e6a 2022-10-25 thomas 			    pack->filesize) {
325 6cf83e6a 2022-10-25 thomas 				err = got_error(GOT_ERR_BAD_PACKFILE);
326 6cf83e6a 2022-10-25 thomas 				break;
327 6cf83e6a 2022-10-25 thomas 			}
328 6cf83e6a 2022-10-25 thomas 
329 66e6097f 2022-10-27 thomas 			if (mapoff + obj->delta.ofs.base_offsetlen >
330 66e6097f 2022-10-27 thomas 			    SIZE_MAX) {
331 66e6097f 2022-10-27 thomas 				err = got_error_fmt(GOT_ERR_RANGE,
332 66e6097f 2022-10-27 thomas 				    "mapoff %lld would overflow size_t",
333 66e6097f 2022-10-27 thomas 				    (long long)mapoff
334 66e6097f 2022-10-27 thomas 				    + obj->delta.ofs.base_offsetlen);
335 66e6097f 2022-10-27 thomas 			}
336 66e6097f 2022-10-27 thomas 
337 142012ed 2022-10-20 thomas 			obj->crc = crc32(obj->crc, pack->map + mapoff,
338 142012ed 2022-10-20 thomas 			    obj->delta.ofs.base_offsetlen);
339 b16893ba 2023-02-24 thomas 			got_hash_update(pack_sha1_ctx, pack->map + mapoff,
340 142012ed 2022-10-20 thomas 			    obj->delta.ofs.base_offsetlen);
341 142012ed 2022-10-20 thomas 			mapoff += obj->delta.ofs.base_offsetlen;
342 142012ed 2022-10-20 thomas 			err = got_inflate_to_mem_mmap(NULL, &datalen,
343 142012ed 2022-10-20 thomas 			    &obj->len, &csum, pack->map, mapoff,
344 142012ed 2022-10-20 thomas 			    pack->filesize - mapoff);
345 142012ed 2022-10-20 thomas 			if (err)
346 142012ed 2022-10-20 thomas 				break;
347 142012ed 2022-10-20 thomas 		} else {
348 142012ed 2022-10-20 thomas 			/*
349 142012ed 2022-10-20 thomas 			 * XXX Seek back and get CRC and SHA1 of on-disk
350 142012ed 2022-10-20 thomas 			 * offset bytes.
351 142012ed 2022-10-20 thomas 			 */
352 142012ed 2022-10-20 thomas 			if (lseek(pack->fd, obj->off + obj->tslen, SEEK_SET)
353 142012ed 2022-10-20 thomas 			    == -1) {
354 142012ed 2022-10-20 thomas 				err = got_error_from_errno("lseek");
355 142012ed 2022-10-20 thomas 				break;
356 142012ed 2022-10-20 thomas 			}
357 142012ed 2022-10-20 thomas 			err = read_checksum(&obj->crc, pack_sha1_ctx,
358 142012ed 2022-10-20 thomas 			    pack->fd, obj->delta.ofs.base_offsetlen);
359 142012ed 2022-10-20 thomas 			if (err)
360 142012ed 2022-10-20 thomas 				break;
361 142012ed 2022-10-20 thomas 
362 142012ed 2022-10-20 thomas 			err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
363 142012ed 2022-10-20 thomas 			    &csum, obj->size, pack->fd);
364 142012ed 2022-10-20 thomas 			if (err)
365 142012ed 2022-10-20 thomas 				break;
366 142012ed 2022-10-20 thomas 		}
367 142012ed 2022-10-20 thomas 		obj->len += obj->delta.ofs.base_offsetlen;
368 142012ed 2022-10-20 thomas 		break;
369 142012ed 2022-10-20 thomas 	default:
370 142012ed 2022-10-20 thomas 		err = got_error(GOT_ERR_OBJ_TYPE);
371 142012ed 2022-10-20 thomas 		break;
372 142012ed 2022-10-20 thomas 	}
373 142012ed 2022-10-20 thomas 
374 142012ed 2022-10-20 thomas 	return err;
375 142012ed 2022-10-20 thomas }
376 142012ed 2022-10-20 thomas 
377 3efd8e31 2022-10-23 thomas const struct got_error *
378 b16893ba 2023-02-24 thomas got_pack_hwrite(int fd, void *buf, int len, struct got_hash *ctx)
379 142012ed 2022-10-20 thomas {
380 142012ed 2022-10-20 thomas 	ssize_t w;
381 142012ed 2022-10-20 thomas 
382 b16893ba 2023-02-24 thomas 	got_hash_update(ctx, buf, len);
383 142012ed 2022-10-20 thomas 
384 142012ed 2022-10-20 thomas 	w = write(fd, buf, len);
385 142012ed 2022-10-20 thomas 	if (w == -1)
386 142012ed 2022-10-20 thomas 		return got_error_from_errno("write");
387 142012ed 2022-10-20 thomas 	if (w != len)
388 142012ed 2022-10-20 thomas 		return got_error(GOT_ERR_IO);
389 142012ed 2022-10-20 thomas 
390 142012ed 2022-10-20 thomas 	return NULL;
391 142012ed 2022-10-20 thomas }
392 142012ed 2022-10-20 thomas 
393 142012ed 2022-10-20 thomas static const struct got_error *
394 142012ed 2022-10-20 thomas resolve_deltified_object(struct got_pack *pack, struct got_packidx *packidx,
395 142012ed 2022-10-20 thomas     struct got_indexed_object *obj, FILE *tmpfile, FILE *delta_base_file,
396 142012ed 2022-10-20 thomas     FILE *delta_accum_file)
397 142012ed 2022-10-20 thomas {
398 142012ed 2022-10-20 thomas 	const struct got_error *err = NULL;
399 142012ed 2022-10-20 thomas 	struct got_delta_chain deltas;
400 142012ed 2022-10-20 thomas 	struct got_delta *delta;
401 142012ed 2022-10-20 thomas 	uint8_t *buf = NULL;
402 142012ed 2022-10-20 thomas 	size_t len = 0;
403 b16893ba 2023-02-24 thomas 	struct got_hash ctx;
404 142012ed 2022-10-20 thomas 	char *header = NULL;
405 142012ed 2022-10-20 thomas 	size_t headerlen;
406 142012ed 2022-10-20 thomas 	uint64_t max_size;
407 142012ed 2022-10-20 thomas 	int base_obj_type;
408 142012ed 2022-10-20 thomas 	const char *obj_label;
409 142012ed 2022-10-20 thomas 
410 142012ed 2022-10-20 thomas 	deltas.nentries = 0;
411 142012ed 2022-10-20 thomas 	STAILQ_INIT(&deltas.entries);
412 142012ed 2022-10-20 thomas 
413 142012ed 2022-10-20 thomas 	err = got_pack_resolve_delta_chain(&deltas, packidx, pack,
414 142012ed 2022-10-20 thomas 	    obj->off, obj->tslen, obj->type, obj->size,
415 142012ed 2022-10-20 thomas 	    GOT_DELTA_CHAIN_RECURSION_MAX);
416 142012ed 2022-10-20 thomas 	if (err)
417 142012ed 2022-10-20 thomas 		goto done;
418 142012ed 2022-10-20 thomas 
419 142012ed 2022-10-20 thomas 	err = got_pack_get_delta_chain_max_size(&max_size, &deltas, pack);
420 142012ed 2022-10-20 thomas 	if (err)
421 142012ed 2022-10-20 thomas 		goto done;
422 142012ed 2022-10-20 thomas 	if (max_size > GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
423 142012ed 2022-10-20 thomas 		rewind(tmpfile);
424 142012ed 2022-10-20 thomas 		rewind(delta_base_file);
425 142012ed 2022-10-20 thomas 		rewind(delta_accum_file);
426 142012ed 2022-10-20 thomas 		err = got_pack_dump_delta_chain_to_file(&len, &deltas,
427 142012ed 2022-10-20 thomas 		    pack, tmpfile, delta_base_file, delta_accum_file);
428 142012ed 2022-10-20 thomas 		if (err)
429 142012ed 2022-10-20 thomas 			goto done;
430 142012ed 2022-10-20 thomas 	} else {
431 142012ed 2022-10-20 thomas 		err = got_pack_dump_delta_chain_to_mem(&buf, &len,
432 142012ed 2022-10-20 thomas 		    &deltas, pack);
433 142012ed 2022-10-20 thomas 	}
434 142012ed 2022-10-20 thomas 	if (err)
435 142012ed 2022-10-20 thomas 		goto done;
436 142012ed 2022-10-20 thomas 
437 142012ed 2022-10-20 thomas 	err = got_delta_chain_get_base_type(&base_obj_type, &deltas);
438 142012ed 2022-10-20 thomas 	if (err)
439 142012ed 2022-10-20 thomas 		goto done;
440 142012ed 2022-10-20 thomas 	err = get_obj_type_label(&obj_label, base_obj_type);
441 142012ed 2022-10-20 thomas 	if (err)
442 142012ed 2022-10-20 thomas 		goto done;
443 142012ed 2022-10-20 thomas 	if (asprintf(&header, "%s %zd", obj_label, len) == -1) {
444 142012ed 2022-10-20 thomas 		err = got_error_from_errno("asprintf");
445 142012ed 2022-10-20 thomas 		goto done;
446 142012ed 2022-10-20 thomas 	}
447 142012ed 2022-10-20 thomas 	headerlen = strlen(header) + 1;
448 b16893ba 2023-02-24 thomas 	got_hash_init(&ctx, GOT_HASH_SHA1);
449 b16893ba 2023-02-24 thomas 	got_hash_update(&ctx, header, headerlen);
450 142012ed 2022-10-20 thomas 	if (max_size > GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
451 b16893ba 2023-02-24 thomas 		err = read_file_digest(&ctx, tmpfile, len);
452 142012ed 2022-10-20 thomas 		if (err)
453 142012ed 2022-10-20 thomas 			goto done;
454 142012ed 2022-10-20 thomas 	} else
455 b16893ba 2023-02-24 thomas 		got_hash_update(&ctx, buf, len);
456 b16893ba 2023-02-24 thomas 	got_hash_final_object_id(&ctx, &obj->id);
457 142012ed 2022-10-20 thomas done:
458 142012ed 2022-10-20 thomas 	free(buf);
459 142012ed 2022-10-20 thomas 	free(header);
460 142012ed 2022-10-20 thomas 	while (!STAILQ_EMPTY(&deltas.entries)) {
461 142012ed 2022-10-20 thomas 		delta = STAILQ_FIRST(&deltas.entries);
462 142012ed 2022-10-20 thomas 		STAILQ_REMOVE_HEAD(&deltas.entries, entry);
463 142012ed 2022-10-20 thomas 		free(delta);
464 142012ed 2022-10-20 thomas 	}
465 142012ed 2022-10-20 thomas 	return err;
466 142012ed 2022-10-20 thomas }
467 142012ed 2022-10-20 thomas 
468 142012ed 2022-10-20 thomas /* Determine the slot in the pack index a given object ID should use. */
469 142012ed 2022-10-20 thomas static int
470 142012ed 2022-10-20 thomas find_object_idx(struct got_packidx *packidx, uint8_t *sha1)
471 142012ed 2022-10-20 thomas {
472 142012ed 2022-10-20 thomas 	u_int8_t id0 = sha1[0];
473 142012ed 2022-10-20 thomas 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
474 142012ed 2022-10-20 thomas 	int left = 0, right = nindexed - 1;
475 142012ed 2022-10-20 thomas 	int cmp = 0, i = 0;
476 142012ed 2022-10-20 thomas 
477 142012ed 2022-10-20 thomas 	if (id0 > 0)
478 142012ed 2022-10-20 thomas 		left = be32toh(packidx->hdr.fanout_table[id0 - 1]);
479 142012ed 2022-10-20 thomas 
480 142012ed 2022-10-20 thomas 	while (left <= right) {
481 142012ed 2022-10-20 thomas 		struct got_packidx_object_id *oid;
482 142012ed 2022-10-20 thomas 
483 142012ed 2022-10-20 thomas 		i = ((left + right) / 2);
484 142012ed 2022-10-20 thomas 		oid = &packidx->hdr.sorted_ids[i];
485 142012ed 2022-10-20 thomas 
486 142012ed 2022-10-20 thomas 		cmp = memcmp(sha1, oid->sha1, SHA1_DIGEST_LENGTH);
487 142012ed 2022-10-20 thomas 		if (cmp == 0)
488 142012ed 2022-10-20 thomas 			return -1; /* object already indexed */
489 142012ed 2022-10-20 thomas 		else if (cmp > 0)
490 142012ed 2022-10-20 thomas 			left = i + 1;
491 142012ed 2022-10-20 thomas 		else if (cmp < 0)
492 142012ed 2022-10-20 thomas 			right = i - 1;
493 142012ed 2022-10-20 thomas 	}
494 142012ed 2022-10-20 thomas 
495 142012ed 2022-10-20 thomas 	return left;
496 142012ed 2022-10-20 thomas }
497 142012ed 2022-10-20 thomas 
498 142012ed 2022-10-20 thomas #if 0
499 142012ed 2022-10-20 thomas static void
500 142012ed 2022-10-20 thomas print_packidx(struct got_packidx *packidx)
501 142012ed 2022-10-20 thomas {
502 142012ed 2022-10-20 thomas 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
503 142012ed 2022-10-20 thomas 	int i;
504 142012ed 2022-10-20 thomas 
505 142012ed 2022-10-20 thomas 	fprintf(stderr, "object IDs:\n");
506 142012ed 2022-10-20 thomas 	for (i = 0; i < nindexed; i++) {
507 142012ed 2022-10-20 thomas 		char hex[SHA1_DIGEST_STRING_LENGTH];
508 142012ed 2022-10-20 thomas 		got_sha1_digest_to_str(packidx->hdr.sorted_ids[i].sha1,
509 142012ed 2022-10-20 thomas 		    hex, sizeof(hex));
510 142012ed 2022-10-20 thomas 		fprintf(stderr, "%s\n", hex);
511 142012ed 2022-10-20 thomas 	}
512 142012ed 2022-10-20 thomas 	fprintf(stderr, "\n");
513 142012ed 2022-10-20 thomas 
514 142012ed 2022-10-20 thomas 	fprintf(stderr, "object offsets:\n");
515 142012ed 2022-10-20 thomas 	for (i = 0; i < nindexed; i++) {
516 142012ed 2022-10-20 thomas 		uint32_t offset = be32toh(packidx->hdr.offsets[i]);
517 142012ed 2022-10-20 thomas 		if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
518 142012ed 2022-10-20 thomas 			int j = offset & GOT_PACKIDX_OFFSET_VAL_MASK;
519 142012ed 2022-10-20 thomas 			fprintf(stderr, "%u -> %llu\n", offset,
520 142012ed 2022-10-20 thomas 			    be64toh(packidx->hdr.large_offsets[j]));
521 142012ed 2022-10-20 thomas 		} else
522 142012ed 2022-10-20 thomas 			fprintf(stderr, "%u\n", offset);
523 142012ed 2022-10-20 thomas 	}
524 142012ed 2022-10-20 thomas 	fprintf(stderr, "\n");
525 142012ed 2022-10-20 thomas 
526 142012ed 2022-10-20 thomas 	fprintf(stderr, "fanout table:");
527 142012ed 2022-10-20 thomas 	for (i = 0; i <= 0xff; i++)
528 142012ed 2022-10-20 thomas 		fprintf(stderr, " %u", be32toh(packidx->hdr.fanout_table[i]));
529 142012ed 2022-10-20 thomas 	fprintf(stderr, "\n");
530 142012ed 2022-10-20 thomas }
531 142012ed 2022-10-20 thomas #endif
532 142012ed 2022-10-20 thomas 
533 142012ed 2022-10-20 thomas static void
534 142012ed 2022-10-20 thomas add_indexed_object(struct got_packidx *packidx, uint32_t idx,
535 142012ed 2022-10-20 thomas     struct got_indexed_object *obj)
536 142012ed 2022-10-20 thomas {
537 142012ed 2022-10-20 thomas 	int i;
538 142012ed 2022-10-20 thomas 
539 142012ed 2022-10-20 thomas 	memcpy(packidx->hdr.sorted_ids[idx].sha1, obj->id.sha1,
540 142012ed 2022-10-20 thomas 	    SHA1_DIGEST_LENGTH);
541 142012ed 2022-10-20 thomas 	packidx->hdr.crc32[idx] = htobe32(obj->crc);
542 142012ed 2022-10-20 thomas 	if (obj->off < GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX)
543 142012ed 2022-10-20 thomas 		packidx->hdr.offsets[idx] = htobe32(obj->off);
544 142012ed 2022-10-20 thomas 	else {
545 142012ed 2022-10-20 thomas 		packidx->hdr.offsets[idx] = htobe32(packidx->nlargeobj |
546 142012ed 2022-10-20 thomas 		    GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX);
547 142012ed 2022-10-20 thomas 		packidx->hdr.large_offsets[packidx->nlargeobj] =
548 142012ed 2022-10-20 thomas 		    htobe64(obj->off);
549 142012ed 2022-10-20 thomas 		packidx->nlargeobj++;
550 142012ed 2022-10-20 thomas 	}
551 142012ed 2022-10-20 thomas 
552 142012ed 2022-10-20 thomas 	for (i = obj->id.sha1[0]; i <= 0xff; i++) {
553 142012ed 2022-10-20 thomas 		uint32_t n = be32toh(packidx->hdr.fanout_table[i]);
554 142012ed 2022-10-20 thomas 		packidx->hdr.fanout_table[i] = htobe32(n + 1);
555 142012ed 2022-10-20 thomas 	}
556 142012ed 2022-10-20 thomas }
557 142012ed 2022-10-20 thomas 
558 142012ed 2022-10-20 thomas static int
559 142012ed 2022-10-20 thomas indexed_obj_cmp(const void *pa, const void *pb)
560 142012ed 2022-10-20 thomas {
561 142012ed 2022-10-20 thomas 	struct got_indexed_object *a, *b;
562 142012ed 2022-10-20 thomas 
563 142012ed 2022-10-20 thomas 	a = (struct got_indexed_object *)pa;
564 142012ed 2022-10-20 thomas 	b = (struct got_indexed_object *)pb;
565 142012ed 2022-10-20 thomas 	return got_object_id_cmp(&a->id, &b->id);
566 142012ed 2022-10-20 thomas }
567 142012ed 2022-10-20 thomas 
568 142012ed 2022-10-20 thomas static void
569 eebe1fbb 2022-10-20 thomas make_packidx(struct got_packidx *packidx, uint32_t nobj,
570 142012ed 2022-10-20 thomas     struct got_indexed_object *objects)
571 142012ed 2022-10-20 thomas {
572 142012ed 2022-10-20 thomas 	struct got_indexed_object *obj;
573 142012ed 2022-10-20 thomas 	int i;
574 142012ed 2022-10-20 thomas 	uint32_t idx = 0;
575 142012ed 2022-10-20 thomas 
576 142012ed 2022-10-20 thomas 	qsort(objects, nobj, sizeof(struct got_indexed_object),
577 142012ed 2022-10-20 thomas 	    indexed_obj_cmp);
578 142012ed 2022-10-20 thomas 
579 142012ed 2022-10-20 thomas 	memset(packidx->hdr.fanout_table, 0,
580 142012ed 2022-10-20 thomas 	    GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS * sizeof(uint32_t));
581 142012ed 2022-10-20 thomas 	packidx->nlargeobj = 0;
582 142012ed 2022-10-20 thomas 
583 142012ed 2022-10-20 thomas 	for (i = 0; i < nobj; i++) {
584 142012ed 2022-10-20 thomas 		obj = &objects[i];
585 142012ed 2022-10-20 thomas 		if (obj->valid)
586 142012ed 2022-10-20 thomas 			add_indexed_object(packidx, idx++, obj);
587 142012ed 2022-10-20 thomas 	}
588 142012ed 2022-10-20 thomas }
589 142012ed 2022-10-20 thomas 
590 142012ed 2022-10-20 thomas static void
591 eebe1fbb 2022-10-20 thomas update_packidx(struct got_packidx *packidx, uint32_t nobj,
592 142012ed 2022-10-20 thomas     struct got_indexed_object *obj)
593 142012ed 2022-10-20 thomas {
594 142012ed 2022-10-20 thomas 	int idx;
595 142012ed 2022-10-20 thomas 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
596 142012ed 2022-10-20 thomas 
597 142012ed 2022-10-20 thomas 	idx = find_object_idx(packidx, obj->id.sha1);
598 aac3e701 2023-02-03 thomas 	if (idx == -1)
599 142012ed 2022-10-20 thomas 		return; /* object already indexed */
600 142012ed 2022-10-20 thomas 
601 142012ed 2022-10-20 thomas 	memmove(&packidx->hdr.sorted_ids[idx + 1],
602 142012ed 2022-10-20 thomas 	    &packidx->hdr.sorted_ids[idx],
603 142012ed 2022-10-20 thomas 	    sizeof(struct got_packidx_object_id) * (nindexed - idx));
604 142012ed 2022-10-20 thomas 	memmove(&packidx->hdr.offsets[idx + 1], &packidx->hdr.offsets[idx],
605 142012ed 2022-10-20 thomas 	    sizeof(uint32_t) * (nindexed - idx));
606 142012ed 2022-10-20 thomas 
607 142012ed 2022-10-20 thomas 	add_indexed_object(packidx, idx, obj);
608 142012ed 2022-10-20 thomas }
609 142012ed 2022-10-20 thomas 
610 142012ed 2022-10-20 thomas static const struct got_error *
611 eebe1fbb 2022-10-20 thomas report_progress(uint32_t nobj_total, uint32_t nobj_indexed, uint32_t nobj_loose,
612 eebe1fbb 2022-10-20 thomas     uint32_t nobj_resolved, struct got_ratelimit *rl,
613 142012ed 2022-10-20 thomas     got_pack_index_progress_cb progress_cb, void *progress_arg)
614 142012ed 2022-10-20 thomas {
615 142012ed 2022-10-20 thomas 	const struct got_error *err;
616 142012ed 2022-10-20 thomas 	int elapsed = 0;
617 142012ed 2022-10-20 thomas 
618 142012ed 2022-10-20 thomas 	if (rl) {
619 142012ed 2022-10-20 thomas 		err = got_ratelimit_check(&elapsed, rl);
620 142012ed 2022-10-20 thomas 		if (err || !elapsed)
621 142012ed 2022-10-20 thomas 			return err;
622 142012ed 2022-10-20 thomas 	}
623 142012ed 2022-10-20 thomas 
624 142012ed 2022-10-20 thomas 	return progress_cb(progress_arg, nobj_total, nobj_indexed, nobj_loose,
625 142012ed 2022-10-20 thomas 	    nobj_resolved);
626 142012ed 2022-10-20 thomas }
627 142012ed 2022-10-20 thomas 
628 142012ed 2022-10-20 thomas const struct got_error *
629 142012ed 2022-10-20 thomas got_pack_index(struct got_pack *pack, int idxfd, FILE *tmpfile,
630 142012ed 2022-10-20 thomas     FILE *delta_base_file, FILE *delta_accum_file, uint8_t *pack_sha1_expected,
631 aecd2225 2022-10-20 thomas     got_pack_index_progress_cb progress_cb, void *progress_arg,
632 aecd2225 2022-10-20 thomas     struct got_ratelimit *rl)
633 142012ed 2022-10-20 thomas {
634 142012ed 2022-10-20 thomas 	const struct got_error *err;
635 142012ed 2022-10-20 thomas 	struct got_packfile_hdr hdr;
636 142012ed 2022-10-20 thomas 	struct got_packidx packidx;
637 142012ed 2022-10-20 thomas 	char buf[8];
638 142012ed 2022-10-20 thomas 	char pack_sha1[SHA1_DIGEST_LENGTH];
639 eebe1fbb 2022-10-20 thomas 	uint32_t nobj, nvalid, nloose, nresolved = 0, i;
640 142012ed 2022-10-20 thomas 	struct got_indexed_object *objects = NULL, *obj;
641 b16893ba 2023-02-24 thomas 	struct got_hash ctx;
642 142012ed 2022-10-20 thomas 	uint8_t packidx_hash[SHA1_DIGEST_LENGTH];
643 142012ed 2022-10-20 thomas 	ssize_t r, w;
644 142012ed 2022-10-20 thomas 	int pass, have_ref_deltas = 0, first_delta_idx = -1;
645 142012ed 2022-10-20 thomas 	size_t mapoff = 0;
646 142012ed 2022-10-20 thomas 	int p_indexed = 0, last_p_indexed = -1;
647 142012ed 2022-10-20 thomas 	int p_resolved = 0, last_p_resolved = -1;
648 142012ed 2022-10-20 thomas 
649 142012ed 2022-10-20 thomas 	/* Require that pack file header and SHA1 trailer are present. */
650 142012ed 2022-10-20 thomas 	if (pack->filesize < sizeof(hdr) + SHA1_DIGEST_LENGTH)
651 142012ed 2022-10-20 thomas 		return got_error_msg(GOT_ERR_BAD_PACKFILE,
652 142012ed 2022-10-20 thomas 		    "short pack file");
653 142012ed 2022-10-20 thomas 
654 142012ed 2022-10-20 thomas 	if (pack->map) {
655 142012ed 2022-10-20 thomas 		memcpy(&hdr, pack->map, sizeof(hdr));
656 142012ed 2022-10-20 thomas 		mapoff += sizeof(hdr);
657 142012ed 2022-10-20 thomas 	} else {
658 142012ed 2022-10-20 thomas 		r = read(pack->fd, &hdr, sizeof(hdr));
659 142012ed 2022-10-20 thomas 		if (r == -1)
660 142012ed 2022-10-20 thomas 			return got_error_from_errno("read");
661 142012ed 2022-10-20 thomas 		if (r < sizeof(hdr))
662 142012ed 2022-10-20 thomas 			return got_error_msg(GOT_ERR_BAD_PACKFILE,
663 142012ed 2022-10-20 thomas 			    "short pack file");
664 142012ed 2022-10-20 thomas 	}
665 142012ed 2022-10-20 thomas 
666 142012ed 2022-10-20 thomas 	if (hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE))
667 142012ed 2022-10-20 thomas 		return got_error_msg(GOT_ERR_BAD_PACKFILE,
668 142012ed 2022-10-20 thomas 		    "bad packfile signature");
669 142012ed 2022-10-20 thomas 	if (hdr.version != htobe32(GOT_PACKFILE_VERSION))
670 142012ed 2022-10-20 thomas 		return got_error_msg(GOT_ERR_BAD_PACKFILE,
671 142012ed 2022-10-20 thomas 		    "bad packfile version");
672 142012ed 2022-10-20 thomas 	nobj = be32toh(hdr.nobjects);
673 142012ed 2022-10-20 thomas 	if (nobj == 0)
674 142012ed 2022-10-20 thomas 		return got_error_msg(GOT_ERR_BAD_PACKFILE,
675 142012ed 2022-10-20 thomas 		    "bad packfile with zero objects");
676 142012ed 2022-10-20 thomas 
677 142012ed 2022-10-20 thomas 	/* We compute the SHA1 of pack file contents and verify later on. */
678 b16893ba 2023-02-24 thomas 	got_hash_init(&ctx, GOT_HASH_SHA1);
679 b16893ba 2023-02-24 thomas 	got_hash_update(&ctx, &hdr, sizeof(hdr));
680 142012ed 2022-10-20 thomas 
681 142012ed 2022-10-20 thomas 	/*
682 142012ed 2022-10-20 thomas 	 * Create an in-memory pack index which will grow as objects
683 142012ed 2022-10-20 thomas 	 * IDs in the pack file are discovered. Only fields used to
684 142012ed 2022-10-20 thomas 	 * read deltified objects will be needed by the pack.c library
685 142012ed 2022-10-20 thomas 	 * code, so setting up just a pack index header is sufficient.
686 142012ed 2022-10-20 thomas 	 */
687 142012ed 2022-10-20 thomas 	memset(&packidx, 0, sizeof(packidx));
688 142012ed 2022-10-20 thomas 	packidx.hdr.magic = malloc(sizeof(uint32_t));
689 142012ed 2022-10-20 thomas 	if (packidx.hdr.magic == NULL)
690 142012ed 2022-10-20 thomas 		return got_error_from_errno("malloc");
691 142012ed 2022-10-20 thomas 	*packidx.hdr.magic = htobe32(GOT_PACKIDX_V2_MAGIC);
692 142012ed 2022-10-20 thomas 	packidx.hdr.version = malloc(sizeof(uint32_t));
693 142012ed 2022-10-20 thomas 	if (packidx.hdr.version == NULL) {
694 142012ed 2022-10-20 thomas 		err = got_error_from_errno("malloc");
695 142012ed 2022-10-20 thomas 		goto done;
696 142012ed 2022-10-20 thomas 	}
697 142012ed 2022-10-20 thomas 	*packidx.hdr.version = htobe32(GOT_PACKIDX_VERSION);
698 142012ed 2022-10-20 thomas 	packidx.hdr.fanout_table = calloc(GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS,
699 142012ed 2022-10-20 thomas 	    sizeof(uint32_t));
700 142012ed 2022-10-20 thomas 	if (packidx.hdr.fanout_table == NULL) {
701 142012ed 2022-10-20 thomas 		err = got_error_from_errno("calloc");
702 142012ed 2022-10-20 thomas 		goto done;
703 142012ed 2022-10-20 thomas 	}
704 142012ed 2022-10-20 thomas 	packidx.hdr.sorted_ids = calloc(nobj,
705 142012ed 2022-10-20 thomas 	    sizeof(struct got_packidx_object_id));
706 142012ed 2022-10-20 thomas 	if (packidx.hdr.sorted_ids == NULL) {
707 142012ed 2022-10-20 thomas 		err = got_error_from_errno("calloc");
708 142012ed 2022-10-20 thomas 		goto done;
709 142012ed 2022-10-20 thomas 	}
710 142012ed 2022-10-20 thomas 	packidx.hdr.crc32 = calloc(nobj, sizeof(uint32_t));
711 142012ed 2022-10-20 thomas 	if (packidx.hdr.crc32 == NULL) {
712 142012ed 2022-10-20 thomas 		err = got_error_from_errno("calloc");
713 142012ed 2022-10-20 thomas 		goto done;
714 142012ed 2022-10-20 thomas 	}
715 142012ed 2022-10-20 thomas 	packidx.hdr.offsets = calloc(nobj, sizeof(uint32_t));
716 142012ed 2022-10-20 thomas 	if (packidx.hdr.offsets == NULL) {
717 142012ed 2022-10-20 thomas 		err = got_error_from_errno("calloc");
718 142012ed 2022-10-20 thomas 		goto done;
719 142012ed 2022-10-20 thomas 	}
720 142012ed 2022-10-20 thomas 	/* Large offsets table is empty for pack files < 2 GB. */
721 142012ed 2022-10-20 thomas 	if (pack->filesize >= GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
722 142012ed 2022-10-20 thomas 		packidx.hdr.large_offsets = calloc(nobj, sizeof(uint64_t));
723 142012ed 2022-10-20 thomas 		if (packidx.hdr.large_offsets == NULL) {
724 142012ed 2022-10-20 thomas 			err = got_error_from_errno("calloc");
725 142012ed 2022-10-20 thomas 			goto done;
726 142012ed 2022-10-20 thomas 		}
727 142012ed 2022-10-20 thomas 	}
728 142012ed 2022-10-20 thomas 
729 142012ed 2022-10-20 thomas 	nvalid = 0;
730 142012ed 2022-10-20 thomas 	nloose = 0;
731 142012ed 2022-10-20 thomas 	objects = calloc(nobj, sizeof(struct got_indexed_object));
732 142012ed 2022-10-20 thomas 	if (objects == NULL)
733 142012ed 2022-10-20 thomas 		return got_error_from_errno("calloc");
734 142012ed 2022-10-20 thomas 
735 142012ed 2022-10-20 thomas 	/*
736 142012ed 2022-10-20 thomas 	 * First pass: locate all objects and identify un-deltified objects.
737 142012ed 2022-10-20 thomas 	 *
738 142012ed 2022-10-20 thomas 	 * When this pass has completed we will know offset, type, size, and
739 142012ed 2022-10-20 thomas 	 * CRC information for all objects in this pack file. We won't know
740 142012ed 2022-10-20 thomas 	 * any of the actual object IDs of deltified objects yet since we
741 142012ed 2022-10-20 thomas 	 * will not yet attempt to combine deltas.
742 142012ed 2022-10-20 thomas 	 */
743 142012ed 2022-10-20 thomas 	pass = 1;
744 142012ed 2022-10-20 thomas 	for (i = 0; i < nobj; i++) {
745 142012ed 2022-10-20 thomas 		/* Don't send too many progress privsep messages. */
746 142012ed 2022-10-20 thomas 		p_indexed = ((i + 1) * 100) / nobj;
747 142012ed 2022-10-20 thomas 		if (p_indexed != last_p_indexed) {
748 142012ed 2022-10-20 thomas 			err = report_progress(nobj, i + 1, nloose, 0,
749 aecd2225 2022-10-20 thomas 			    rl, progress_cb, progress_arg);
750 142012ed 2022-10-20 thomas 			if (err)
751 142012ed 2022-10-20 thomas 				goto done;
752 142012ed 2022-10-20 thomas 			last_p_indexed = p_indexed;
753 142012ed 2022-10-20 thomas 		}
754 142012ed 2022-10-20 thomas 
755 142012ed 2022-10-20 thomas 		obj = &objects[i];
756 142012ed 2022-10-20 thomas 		obj->crc = crc32(0L, NULL, 0);
757 142012ed 2022-10-20 thomas 
758 142012ed 2022-10-20 thomas 		/* Store offset to type+size information for this object. */
759 142012ed 2022-10-20 thomas 		if (pack->map) {
760 142012ed 2022-10-20 thomas 			obj->off = mapoff;
761 142012ed 2022-10-20 thomas 		} else {
762 142012ed 2022-10-20 thomas 			obj->off = lseek(pack->fd, 0, SEEK_CUR);
763 142012ed 2022-10-20 thomas 			if (obj->off == -1) {
764 142012ed 2022-10-20 thomas 				err = got_error_from_errno("lseek");
765 142012ed 2022-10-20 thomas 				goto done;
766 142012ed 2022-10-20 thomas 			}
767 142012ed 2022-10-20 thomas 		}
768 142012ed 2022-10-20 thomas 
769 142012ed 2022-10-20 thomas 		err = read_packed_object(pack, obj, tmpfile, &ctx);
770 142012ed 2022-10-20 thomas 		if (err)
771 142012ed 2022-10-20 thomas 			goto done;
772 142012ed 2022-10-20 thomas 
773 142012ed 2022-10-20 thomas 		if (pack->map) {
774 142012ed 2022-10-20 thomas 			mapoff += obj->tslen + obj->len;
775 142012ed 2022-10-20 thomas 		} else {
776 142012ed 2022-10-20 thomas 			if (lseek(pack->fd, obj->off + obj->tslen + obj->len,
777 142012ed 2022-10-20 thomas 			    SEEK_SET) == -1) {
778 142012ed 2022-10-20 thomas 				err = got_error_from_errno("lseek");
779 142012ed 2022-10-20 thomas 				goto done;
780 142012ed 2022-10-20 thomas 			}
781 142012ed 2022-10-20 thomas 		}
782 142012ed 2022-10-20 thomas 
783 142012ed 2022-10-20 thomas 		if (obj->type == GOT_OBJ_TYPE_BLOB ||
784 142012ed 2022-10-20 thomas 		    obj->type == GOT_OBJ_TYPE_TREE ||
785 142012ed 2022-10-20 thomas 		    obj->type == GOT_OBJ_TYPE_COMMIT ||
786 142012ed 2022-10-20 thomas 		    obj->type == GOT_OBJ_TYPE_TAG) {
787 142012ed 2022-10-20 thomas 			obj->valid = 1;
788 142012ed 2022-10-20 thomas 			nloose++;
789 142012ed 2022-10-20 thomas 		} else {
790 142012ed 2022-10-20 thomas 			if (first_delta_idx == -1)
791 142012ed 2022-10-20 thomas 				first_delta_idx = i;
792 142012ed 2022-10-20 thomas 			if (obj->type == GOT_OBJ_TYPE_REF_DELTA)
793 142012ed 2022-10-20 thomas 				have_ref_deltas = 1;
794 142012ed 2022-10-20 thomas 		}
795 142012ed 2022-10-20 thomas 	}
796 142012ed 2022-10-20 thomas 	nvalid = nloose;
797 142012ed 2022-10-20 thomas 
798 142012ed 2022-10-20 thomas 	/*
799 142012ed 2022-10-20 thomas 	 * Having done a full pass over the pack file and can now
800 142012ed 2022-10-20 thomas 	 * verify its checksum.
801 142012ed 2022-10-20 thomas 	 */
802 b16893ba 2023-02-24 thomas 	got_hash_final(&ctx, pack_sha1);
803 3efd8e31 2022-10-23 thomas 
804 142012ed 2022-10-20 thomas 	if (memcmp(pack_sha1_expected, pack_sha1, SHA1_DIGEST_LENGTH) != 0) {
805 3efd8e31 2022-10-23 thomas 		err = got_error(GOT_ERR_PACKFILE_CSUM);
806 142012ed 2022-10-20 thomas 		goto done;
807 142012ed 2022-10-20 thomas 	}
808 142012ed 2022-10-20 thomas 
809 142012ed 2022-10-20 thomas 	/* Verify the SHA1 checksum stored at the end of the pack file. */
810 142012ed 2022-10-20 thomas 	if (pack->map) {
811 66e6097f 2022-10-27 thomas 		if (pack->filesize > SIZE_MAX) {
812 66e6097f 2022-10-27 thomas 			err = got_error_fmt(GOT_ERR_RANGE,
813 66e6097f 2022-10-27 thomas 			    "filesize %lld overflows size_t",
814 66e6097f 2022-10-27 thomas 			    (long long)pack->filesize);
815 66e6097f 2022-10-27 thomas 			goto done;
816 66e6097f 2022-10-27 thomas 		}
817 66e6097f 2022-10-27 thomas 
818 142012ed 2022-10-20 thomas 		memcpy(pack_sha1_expected, pack->map +
819 142012ed 2022-10-20 thomas 		    pack->filesize - SHA1_DIGEST_LENGTH,
820 142012ed 2022-10-20 thomas 		    SHA1_DIGEST_LENGTH);
821 142012ed 2022-10-20 thomas 	} else {
822 142012ed 2022-10-20 thomas 		ssize_t n;
823 142012ed 2022-10-20 thomas 		if (lseek(pack->fd, -SHA1_DIGEST_LENGTH, SEEK_END) == -1) {
824 142012ed 2022-10-20 thomas 			err = got_error_from_errno("lseek");
825 142012ed 2022-10-20 thomas 			goto done;
826 142012ed 2022-10-20 thomas 		}
827 142012ed 2022-10-20 thomas 		n = read(pack->fd, pack_sha1_expected, SHA1_DIGEST_LENGTH);
828 142012ed 2022-10-20 thomas 		if (n == -1) {
829 142012ed 2022-10-20 thomas 			err = got_error_from_errno("read");
830 142012ed 2022-10-20 thomas 			goto done;
831 142012ed 2022-10-20 thomas 		}
832 142012ed 2022-10-20 thomas 		if (n != SHA1_DIGEST_LENGTH) {
833 142012ed 2022-10-20 thomas 			err = got_error(GOT_ERR_IO);
834 142012ed 2022-10-20 thomas 			goto done;
835 142012ed 2022-10-20 thomas 		}
836 142012ed 2022-10-20 thomas 	}
837 142012ed 2022-10-20 thomas 	if (memcmp(pack_sha1, pack_sha1_expected, SHA1_DIGEST_LENGTH) != 0) {
838 142012ed 2022-10-20 thomas 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
839 142012ed 2022-10-20 thomas 		    "bad checksum in pack file trailer");
840 142012ed 2022-10-20 thomas 		goto done;
841 142012ed 2022-10-20 thomas 	}
842 142012ed 2022-10-20 thomas 
843 142012ed 2022-10-20 thomas 	if (first_delta_idx == -1)
844 142012ed 2022-10-20 thomas 		first_delta_idx = 0;
845 142012ed 2022-10-20 thomas 
846 142012ed 2022-10-20 thomas 	/* In order to resolve ref deltas we need an in-progress pack index. */
847 142012ed 2022-10-20 thomas 	if (have_ref_deltas)
848 142012ed 2022-10-20 thomas 		make_packidx(&packidx, nobj, objects);
849 142012ed 2022-10-20 thomas 
850 142012ed 2022-10-20 thomas 	/*
851 142012ed 2022-10-20 thomas 	 * Second pass: We can now resolve deltas to compute the IDs of
852 142012ed 2022-10-20 thomas 	 * objects which appear in deltified form. Because deltas can be
853 142012ed 2022-10-20 thomas 	 * chained this pass may require a couple of iterations until all
854 142012ed 2022-10-20 thomas 	 * IDs of deltified objects have been discovered.
855 142012ed 2022-10-20 thomas 	 */
856 142012ed 2022-10-20 thomas 	pass++;
857 142012ed 2022-10-20 thomas 	while (nvalid != nobj) {
858 142012ed 2022-10-20 thomas 		int n = 0;
859 142012ed 2022-10-20 thomas 		/*
860 142012ed 2022-10-20 thomas 		 * This loop will only run once unless the pack file
861 142012ed 2022-10-20 thomas 		 * contains ref deltas which refer to objects located
862 142012ed 2022-10-20 thomas 		 * later in the pack file, which is unusual.
863 142012ed 2022-10-20 thomas 		 * Offset deltas can always be resolved in one pass
864 142012ed 2022-10-20 thomas 		 * unless the packfile is corrupt.
865 142012ed 2022-10-20 thomas 		 */
866 142012ed 2022-10-20 thomas 		for (i = first_delta_idx; i < nobj; i++) {
867 142012ed 2022-10-20 thomas 			obj = &objects[i];
868 142012ed 2022-10-20 thomas 			if (obj->type != GOT_OBJ_TYPE_REF_DELTA &&
869 142012ed 2022-10-20 thomas 			    obj->type != GOT_OBJ_TYPE_OFFSET_DELTA)
870 142012ed 2022-10-20 thomas 				continue;
871 142012ed 2022-10-20 thomas 
872 142012ed 2022-10-20 thomas 			if (obj->valid)
873 142012ed 2022-10-20 thomas 				continue;
874 142012ed 2022-10-20 thomas 
875 142012ed 2022-10-20 thomas 			if (pack->map == NULL && lseek(pack->fd,
876 142012ed 2022-10-20 thomas 			    obj->off + obj->tslen, SEEK_SET) == -1) {
877 142012ed 2022-10-20 thomas 				err = got_error_from_errno("lseek");
878 142012ed 2022-10-20 thomas 				goto done;
879 142012ed 2022-10-20 thomas 			}
880 142012ed 2022-10-20 thomas 
881 142012ed 2022-10-20 thomas 			err = resolve_deltified_object(pack, &packidx, obj,
882 142012ed 2022-10-20 thomas 			    tmpfile, delta_base_file, delta_accum_file);
883 142012ed 2022-10-20 thomas 			if (err) {
884 142012ed 2022-10-20 thomas 				if (err->code != GOT_ERR_NO_OBJ)
885 142012ed 2022-10-20 thomas 					goto done;
886 142012ed 2022-10-20 thomas 				/*
887 142012ed 2022-10-20 thomas 				 * We cannot resolve this object yet because
888 142012ed 2022-10-20 thomas 				 * a delta base is unknown. Try again later.
889 142012ed 2022-10-20 thomas 				 */
890 142012ed 2022-10-20 thomas 				continue;
891 142012ed 2022-10-20 thomas 			}
892 142012ed 2022-10-20 thomas 
893 142012ed 2022-10-20 thomas 			obj->valid = 1;
894 142012ed 2022-10-20 thomas 			n++;
895 142012ed 2022-10-20 thomas 			if (have_ref_deltas)
896 142012ed 2022-10-20 thomas 				update_packidx(&packidx, nobj, obj);
897 142012ed 2022-10-20 thomas 			/* Don't send too many progress privsep messages. */
898 142012ed 2022-10-20 thomas 			p_resolved = ((nresolved + n) * 100) / nobj;
899 142012ed 2022-10-20 thomas 			if (p_resolved != last_p_resolved) {
900 142012ed 2022-10-20 thomas 				err = report_progress(nobj, nobj,
901 aecd2225 2022-10-20 thomas 				    nloose, nresolved + n, rl,
902 142012ed 2022-10-20 thomas 				    progress_cb, progress_arg);
903 142012ed 2022-10-20 thomas 				if (err)
904 142012ed 2022-10-20 thomas 					goto done;
905 142012ed 2022-10-20 thomas 				last_p_resolved = p_resolved;
906 142012ed 2022-10-20 thomas 			}
907 142012ed 2022-10-20 thomas 
908 142012ed 2022-10-20 thomas 		}
909 142012ed 2022-10-20 thomas 		if (pass++ > 3 && n == 0) {
910 142012ed 2022-10-20 thomas 			err = got_error_msg(GOT_ERR_BAD_PACKFILE,
911 142012ed 2022-10-20 thomas 			    "could not resolve any of deltas; packfile could "
912 142012ed 2022-10-20 thomas 			    "be corrupt");
913 142012ed 2022-10-20 thomas 			goto done;
914 142012ed 2022-10-20 thomas 		}
915 142012ed 2022-10-20 thomas 		nresolved += n;
916 88056de1 2023-01-19 thomas 		nvalid += n;
917 142012ed 2022-10-20 thomas 	}
918 142012ed 2022-10-20 thomas 
919 142012ed 2022-10-20 thomas 	if (nloose + nresolved != nobj) {
920 142012ed 2022-10-20 thomas 		static char msg[64];
921 142012ed 2022-10-20 thomas 		snprintf(msg, sizeof(msg), "discovered only %d of %d objects",
922 142012ed 2022-10-20 thomas 		    nloose + nresolved, nobj);
923 142012ed 2022-10-20 thomas 		err = got_error_msg(GOT_ERR_BAD_PACKFILE, msg);
924 142012ed 2022-10-20 thomas 		goto done;
925 142012ed 2022-10-20 thomas 	}
926 142012ed 2022-10-20 thomas 
927 142012ed 2022-10-20 thomas 	err = report_progress(nobj, nobj, nloose, nresolved, NULL,
928 142012ed 2022-10-20 thomas 	    progress_cb, progress_arg);
929 142012ed 2022-10-20 thomas 	if (err)
930 142012ed 2022-10-20 thomas 		goto done;
931 142012ed 2022-10-20 thomas 
932 142012ed 2022-10-20 thomas 	make_packidx(&packidx, nobj, objects);
933 142012ed 2022-10-20 thomas 
934 142012ed 2022-10-20 thomas 	free(objects);
935 142012ed 2022-10-20 thomas 	objects = NULL;
936 142012ed 2022-10-20 thomas 
937 b16893ba 2023-02-24 thomas 	got_hash_init(&ctx, GOT_HASH_SHA1);
938 142012ed 2022-10-20 thomas 	putbe32(buf, GOT_PACKIDX_V2_MAGIC);
939 142012ed 2022-10-20 thomas 	putbe32(buf + 4, GOT_PACKIDX_VERSION);
940 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, buf, 8, &ctx);
941 142012ed 2022-10-20 thomas 	if (err)
942 142012ed 2022-10-20 thomas 		goto done;
943 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, packidx.hdr.fanout_table,
944 142012ed 2022-10-20 thomas 	    GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS * sizeof(uint32_t), &ctx);
945 142012ed 2022-10-20 thomas 	if (err)
946 142012ed 2022-10-20 thomas 		goto done;
947 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, packidx.hdr.sorted_ids,
948 142012ed 2022-10-20 thomas 	    nobj * SHA1_DIGEST_LENGTH, &ctx);
949 142012ed 2022-10-20 thomas 	if (err)
950 142012ed 2022-10-20 thomas 		goto done;
951 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, packidx.hdr.crc32,
952 3efd8e31 2022-10-23 thomas 	    nobj * sizeof(uint32_t), &ctx);
953 142012ed 2022-10-20 thomas 	if (err)
954 142012ed 2022-10-20 thomas 		goto done;
955 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, packidx.hdr.offsets,
956 3efd8e31 2022-10-23 thomas 	    nobj * sizeof(uint32_t), &ctx);
957 142012ed 2022-10-20 thomas 	if (err)
958 142012ed 2022-10-20 thomas 		goto done;
959 142012ed 2022-10-20 thomas 	if (packidx.nlargeobj > 0) {
960 3efd8e31 2022-10-23 thomas 		err = got_pack_hwrite(idxfd, packidx.hdr.large_offsets,
961 142012ed 2022-10-20 thomas 		    packidx.nlargeobj * sizeof(uint64_t), &ctx);
962 142012ed 2022-10-20 thomas 		if (err)
963 142012ed 2022-10-20 thomas 			goto done;
964 142012ed 2022-10-20 thomas 	}
965 3efd8e31 2022-10-23 thomas 	err = got_pack_hwrite(idxfd, pack_sha1, SHA1_DIGEST_LENGTH, &ctx);
966 142012ed 2022-10-20 thomas 	if (err)
967 142012ed 2022-10-20 thomas 		goto done;
968 142012ed 2022-10-20 thomas 
969 b16893ba 2023-02-24 thomas 	got_hash_final(&ctx, packidx_hash);
970 142012ed 2022-10-20 thomas 	w = write(idxfd, packidx_hash, sizeof(packidx_hash));
971 142012ed 2022-10-20 thomas 	if (w == -1) {
972 142012ed 2022-10-20 thomas 		err = got_error_from_errno("write");
973 142012ed 2022-10-20 thomas 		goto done;
974 142012ed 2022-10-20 thomas 	}
975 142012ed 2022-10-20 thomas 	if (w != sizeof(packidx_hash)) {
976 142012ed 2022-10-20 thomas 		err = got_error(GOT_ERR_IO);
977 142012ed 2022-10-20 thomas 		goto done;
978 142012ed 2022-10-20 thomas 	}
979 142012ed 2022-10-20 thomas done:
980 142012ed 2022-10-20 thomas 	free(objects);
981 142012ed 2022-10-20 thomas 	free(packidx.hdr.magic);
982 142012ed 2022-10-20 thomas 	free(packidx.hdr.version);
983 142012ed 2022-10-20 thomas 	free(packidx.hdr.fanout_table);
984 142012ed 2022-10-20 thomas 	free(packidx.hdr.sorted_ids);
985 142012ed 2022-10-20 thomas 	free(packidx.hdr.offsets);
986 142012ed 2022-10-20 thomas 	free(packidx.hdr.large_offsets);
987 142012ed 2022-10-20 thomas 	return err;
988 142012ed 2022-10-20 thomas }