Gotweb

Blob

Date:: Wed May 4 13:43:24 2022 UTC
Message:: store deltas in compressed form while packing, both in memory and cache file This reduces memory and disk space consumption during packing. with tweaks + memleak on error fix from op@ ok op@
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/uio.h>
22 #include <sys/socket.h>
23 #include <sys/wait.h>
24 #include <sys/resource.h>
25 #include <sys/mman.h>
26 
27 #include <errno.h>
28 #include <fcntl.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <stdint.h>
33 #include <sha1.h>
34 #include <unistd.h>
35 #include <zlib.h>
36 #include <ctype.h>
37 #include <libgen.h>
38 #include <limits.h>
39 #include <imsg.h>
40 #include <time.h>
41 
42 #include "got_error.h"
43 #include "got_object.h"
44 #include "got_repository.h"
45 #include "got_opentemp.h"
46 #include "got_path.h"
47 
48 #include "got_lib_sha1.h"
49 #include "got_lib_delta.h"
50 #include "got_lib_inflate.h"
51 #include "got_lib_object.h"
52 #include "got_lib_privsep.h"
53 #include "got_lib_object_idcache.h"
54 #include "got_lib_object_cache.h"
55 #include "got_lib_object_parse.h"
56 #include "got_lib_pack.h"
57 #include "got_lib_repository.h"
58 
59 #ifndef MIN
60 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
61 #endif
62 
63 struct got_object_id *
64 got_object_get_id(struct got_object *obj)
65 {
66 	return &obj->id;
67 }
68 
69 const struct got_error *
70 got_object_get_id_str(char **outbuf, struct got_object *obj)
71 {
72 	return got_object_id_str(outbuf, &obj->id);
73 }
74 
75 const struct got_error *
76 got_object_get_type(int *type, struct got_repository *repo,
77     struct got_object_id *id)
78 {
79 	const struct got_error *err = NULL;
80 	struct got_object *obj;
81 
82 	err = got_object_open(&obj, repo, id);
83 	if (err)
84 		return err;
85 
86 	switch (obj->type) {
87 	case GOT_OBJ_TYPE_COMMIT:
88 	case GOT_OBJ_TYPE_TREE:
89 	case GOT_OBJ_TYPE_BLOB:
90 	case GOT_OBJ_TYPE_TAG:
91 		*type = obj->type;
92 		break;
93 	default:
94 		err = got_error(GOT_ERR_OBJ_TYPE);
95 		break;
96 	}
97 
98 	got_object_close(obj);
99 	return err;
100 }
101 
102 const struct got_error *
103 got_object_get_path(char **path, struct got_object_id *id,
104     struct got_repository *repo)
105 {
106 	const struct got_error *err = NULL;
107 	char *hex = NULL;
108 	char *path_objects;
109 
110 	*path = NULL;
111 
112 	path_objects = got_repo_get_path_objects(repo);
113 	if (path_objects == NULL)
114 		return got_error_from_errno("got_repo_get_path_objects");
115 
116 	err = got_object_id_str(&hex, id);
117 	if (err)
118 		goto done;
119 
120 	if (asprintf(path, "%s/%.2x/%s", path_objects,
121 	    id->sha1[0], hex + 2) == -1)
122 		err = got_error_from_errno("asprintf");
123 
124 done:
125 	free(hex);
126 	free(path_objects);
127 	return err;
128 }
129 
130 const struct got_error *
131 got_object_open_loose_fd(int *fd, struct got_object_id *id,
132     struct got_repository *repo)
133 {
134 	const struct got_error *err = NULL;
135 	char *path;
136 
137 	err = got_object_get_path(&path, id, repo);
138 	if (err)
139 		return err;
140 	*fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
141 	if (*fd == -1) {
142 		err = got_error_from_errno2("open", path);
143 		goto done;
144 	}
145 done:
146 	free(path);
147 	return err;
148 }
149 
150 static const struct got_error *
151 request_packed_object(struct got_object **obj, struct got_pack *pack, int idx,
152     struct got_object_id *id)
153 {
154 	const struct got_error *err = NULL;
155 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
156 
157 	err = got_privsep_send_packed_obj_req(ibuf, idx, id);
158 	if (err)
159 		return err;
160 
161 	err = got_privsep_recv_obj(obj, ibuf);
162 	if (err)
163 		return err;
164 
165 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
166 
167 	return NULL;
168 }
169 
170 /* Create temporary files used during delta application. */
171 static const struct got_error *
172 pack_child_send_tempfiles(struct imsgbuf *ibuf, struct got_pack *pack)
173 {
174 	const struct got_error *err;
175 	int basefd, accumfd;
176 
177 	/* 
178 	 * For performance reasons, the child will keep reusing the
179 	 * same temporary files during every object request.
180 	 * Opening and closing new files for every object request is
181 	 * too expensive during operations such as 'gotadmin pack'.
182 	 */
183 	if (pack->child_has_tempfiles)
184 		return NULL;
185 
186 	basefd = got_opentempfd();
187 	if (basefd == -1)
188 		return got_error_from_errno("got_opentempfd");
189 
190 	err = got_privsep_send_tmpfd(ibuf, basefd);
191 	if (err)
192 		return err;
193 
194 	accumfd = got_opentempfd();
195 	if (accumfd == -1)
196 		return got_error_from_errno("got_opentempfd");
197 
198 	err = got_privsep_send_tmpfd(ibuf, accumfd);
199 	if (err)
200 		return err;
201 
202 	pack->child_has_tempfiles = 1;
203 	return NULL;
204 }
205 
206 static const struct got_error *
207 request_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen,
208     int outfd, struct got_pack *pack, int idx, struct got_object_id *id)
209 {
210 	const struct got_error *err = NULL;
211 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
212 	int outfd_child;
213 
214 	 err = pack_child_send_tempfiles(ibuf, pack);
215 	 if (err)
216 		return err;
217 
218 	outfd_child = dup(outfd);
219 	if (outfd_child == -1)
220 		return got_error_from_errno("dup");
221 
222 	err = got_privsep_send_packed_raw_obj_req(ibuf, idx, id);
223 	if (err) {
224 		close(outfd_child);
225 		return err;
226 	}
227 
228 	err = got_privsep_send_raw_obj_outfd(ibuf, outfd_child);
229 	if (err)
230 		return err;
231 
232 	err = got_privsep_recv_raw_obj(outbuf, size, hdrlen, ibuf);
233 	if (err)
234 		return err;
235 
236 	return NULL;
237 }
238 
239 static void
240 set_max_datasize(void)
241 {
242 	struct rlimit rl;
243 
244 	if (getrlimit(RLIMIT_DATA, &rl) != 0)
245 		return;
246 
247 	rl.rlim_cur = rl.rlim_max;
248 	setrlimit(RLIMIT_DATA, &rl);
249 }
250 
251 static const struct got_error *
252 start_pack_privsep_child(struct got_pack *pack, struct got_packidx *packidx)
253 {
254 	const struct got_error *err = NULL;
255 	int imsg_fds[2];
256 	pid_t pid;
257 	struct imsgbuf *ibuf;
258 
259 	ibuf = calloc(1, sizeof(*ibuf));
260 	if (ibuf == NULL)
261 		return got_error_from_errno("calloc");
262 
263 	pack->privsep_child = calloc(1, sizeof(*pack->privsep_child));
264 	if (pack->privsep_child == NULL) {
265 		err = got_error_from_errno("calloc");
266 		free(ibuf);
267 		return err;
268 	}
269 	pack->child_has_tempfiles = 0;
270 	pack->child_has_delta_outfd = 0;
271 
272 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
273 		err = got_error_from_errno("socketpair");
274 		goto done;
275 	}
276 
277 	pid = fork();
278 	if (pid == -1) {
279 		err = got_error_from_errno("fork");
280 		goto done;
281 	} else if (pid == 0) {
282 		set_max_datasize();
283 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_PACK,
284 		    pack->path_packfile);
285 		/* not reached */
286 	}
287 
288 	if (close(imsg_fds[1]) == -1)
289 		return got_error_from_errno("close");
290 	pack->privsep_child->imsg_fd = imsg_fds[0];
291 	pack->privsep_child->pid = pid;
292 	imsg_init(ibuf, imsg_fds[0]);
293 	pack->privsep_child->ibuf = ibuf;
294 
295 	err = got_privsep_init_pack_child(ibuf, pack, packidx);
296 	if (err) {
297 		const struct got_error *child_err;
298 		err = got_privsep_send_stop(pack->privsep_child->imsg_fd);
299 		child_err = got_privsep_wait_for_child(
300 		    pack->privsep_child->pid);
301 		if (child_err && err == NULL)
302 			err = child_err;
303 	}
304 done:
305 	if (err) {
306 		free(ibuf);
307 		free(pack->privsep_child);
308 		pack->privsep_child = NULL;
309 	}
310 	return err;
311 }
312 
313 static const struct got_error *
314 read_packed_object_privsep(struct got_object **obj,
315     struct got_repository *repo, struct got_pack *pack,
316     struct got_packidx *packidx, int idx, struct got_object_id *id)
317 {
318 	const struct got_error *err = NULL;
319 
320 	if (pack->privsep_child == NULL) {
321 		err = start_pack_privsep_child(pack, packidx);
322 		if (err)
323 			return err;
324 	}
325 
326 	return request_packed_object(obj, pack, idx, id);
327 }
328 
329 static const struct got_error *
330 read_packed_object_raw_privsep(uint8_t **outbuf, off_t *size, size_t *hdrlen,
331     int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
332     struct got_object_id *id)
333 {
334 	const struct got_error *err = NULL;
335 
336 	if (pack->privsep_child == NULL) {
337 		err = start_pack_privsep_child(pack, packidx);
338 		if (err)
339 			return err;
340 	}
341 
342 	return request_packed_object_raw(outbuf, size, hdrlen, outfd, pack,
343 	    idx, id);
344 }
345 
346 const struct got_error *
347 got_object_open_packed(struct got_object **obj, struct got_object_id *id,
348     struct got_repository *repo)
349 {
350 	const struct got_error *err = NULL;
351 	struct got_pack *pack = NULL;
352 	struct got_packidx *packidx = NULL;
353 	int idx;
354 	char *path_packfile;
355 
356 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
357 	if (err)
358 		return err;
359 
360 	err = got_packidx_get_packfile_path(&path_packfile,
361 	    packidx->path_packidx);
362 	if (err)
363 		return err;
364 
365 	pack = got_repo_get_cached_pack(repo, path_packfile);
366 	if (pack == NULL) {
367 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
368 		if (err)
369 			goto done;
370 	}
371 
372 	err = read_packed_object_privsep(obj, repo, pack, packidx, idx, id);
373 	if (err)
374 		goto done;
375 done:
376 	free(path_packfile);
377 	return err;
378 }
379 
380 const struct got_error *
381 got_object_open_from_packfile(struct got_object **obj, struct got_object_id *id,
382     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
383     struct got_repository *repo)
384 {
385 	return read_packed_object_privsep(obj, repo, pack, packidx,
386 	    obj_idx, id);
387 }
388 
389 const struct got_error *
390 got_object_read_raw_delta(uint64_t *base_size, uint64_t *result_size,
391     off_t *delta_size, off_t *delta_compressed_size, off_t *delta_offset,
392     off_t *delta_out_offset, struct got_object_id **base_id, int delta_cache_fd,
393     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
394     struct got_repository *repo)
395 {
396 	const struct got_error *err = NULL;
397 	struct got_pack *pack = NULL;
398 	char *path_packfile;
399 
400 	*base_size = 0;
401 	*result_size = 0;
402 	*delta_size = 0;
403 	*delta_compressed_size = 0;
404 	*delta_offset = 0;
405 	*delta_out_offset = 0;
406 
407 	err = got_packidx_get_packfile_path(&path_packfile,
408 	    packidx->path_packidx);
409 	if (err)
410 		return err;
411 
412 	pack = got_repo_get_cached_pack(repo, path_packfile);
413 	if (pack == NULL) {
414 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
415 		if (err)
416 			return err;
417 	}
418 
419 	if (pack->privsep_child == NULL) {
420 		err = start_pack_privsep_child(pack, packidx);
421 		if (err)
422 			return err;
423 	}
424 
425 	if (!pack->child_has_delta_outfd) {
426 		int outfd_child;
427 		outfd_child = dup(delta_cache_fd);
428 		if (outfd_child == -1)
429 			return got_error_from_errno("dup");
430 		err = got_privsep_send_raw_delta_outfd(
431 		    pack->privsep_child->ibuf, outfd_child);
432 		if (err)
433 			return err;
434 		pack->child_has_delta_outfd = 1;
435 	}
436 
437 	err = got_privsep_send_raw_delta_req(pack->privsep_child->ibuf,
438 	    obj_idx, id);
439 	if (err)
440 		return err;
441 
442 	return got_privsep_recv_raw_delta(base_size, result_size, delta_size,
443 	    delta_compressed_size, delta_offset, delta_out_offset, base_id,
444 	    pack->privsep_child->ibuf);
445 }
446 
447 static const struct got_error *
448 request_object(struct got_object **obj, struct got_object_id *id,
449     struct got_repository *repo, int fd)
450 {
451 	const struct got_error *err = NULL;
452 	struct imsgbuf *ibuf;
453 
454 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf;
455 
456 	err = got_privsep_send_obj_req(ibuf, fd, id);
457 	if (err)
458 		return err;
459 
460 	return got_privsep_recv_obj(obj, ibuf);
461 }
462 
463 static const struct got_error *
464 request_raw_object(uint8_t **outbuf, off_t *size, size_t *hdrlen, int outfd,
465     struct got_object_id *id, struct got_repository *repo, int infd)
466 {
467 	const struct got_error *err = NULL;
468 	struct imsgbuf *ibuf;
469 	int outfd_child;
470 
471 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf;
472 
473 	outfd_child = dup(outfd);
474 	if (outfd_child == -1)
475 		return got_error_from_errno("dup");
476 
477 	err = got_privsep_send_raw_obj_req(ibuf, infd, id);
478 	if (err)
479 		return err;
480 
481 	err = got_privsep_send_raw_obj_outfd(ibuf, outfd_child);
482 	if (err)
483 		return err;
484 
485 	return got_privsep_recv_raw_obj(outbuf, size, hdrlen, ibuf);
486 }
487 
488 static const struct got_error *
489 start_read_object_child(struct got_repository *repo)
490 {
491 	const struct got_error *err = NULL;
492 	int imsg_fds[2];
493 	pid_t pid;
494 	struct imsgbuf *ibuf;
495 
496 	ibuf = calloc(1, sizeof(*ibuf));
497 	if (ibuf == NULL)
498 		return got_error_from_errno("calloc");
499 
500 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
501 		err = got_error_from_errno("socketpair");
502 		free(ibuf);
503 		return err;
504 	}
505 
506 	pid = fork();
507 	if (pid == -1) {
508 		err = got_error_from_errno("fork");
509 		free(ibuf);
510 		return err;
511 	}
512 	else if (pid == 0) {
513 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_OBJECT,
514 		    repo->path);
515 		/* not reached */
516 	}
517 
518 	if (close(imsg_fds[1]) == -1) {
519 		err = got_error_from_errno("close");
520 		free(ibuf);
521 		return err;
522 	}
523 
524 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd =
525 	    imsg_fds[0];
526 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].pid = pid;
527 	imsg_init(ibuf, imsg_fds[0]);
528 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf = ibuf;
529 
530 	return NULL;
531 }
532 
533 const struct got_error *
534 got_object_read_header_privsep(struct got_object **obj,
535     struct got_object_id *id, struct got_repository *repo, int obj_fd)
536 {
537 	const struct got_error *err;
538 
539 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd != -1)
540 		return request_object(obj, id, repo, obj_fd);
541 
542 	err = start_read_object_child(repo);
543 	if (err) {
544 		close(obj_fd);
545 		return err;
546 	}
547 
548 	return request_object(obj, id, repo, obj_fd);
549 }
550 
551 static const struct got_error *
552 read_object_raw_privsep(uint8_t **outbuf, off_t *size, size_t *hdrlen,
553     int outfd, struct got_object_id *id, struct got_repository *repo,
554     int obj_fd)
555 {
556 	const struct got_error *err;
557 
558 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd != -1)
559 		return request_raw_object(outbuf, size, hdrlen, outfd, id,
560 		    repo, obj_fd);
561 
562 	err = start_read_object_child(repo);
563 	if (err)
564 		return err;
565 
566 	return request_raw_object(outbuf, size, hdrlen, outfd, id, repo,
567 	    obj_fd);
568 }
569 
570 const struct got_error *
571 got_object_open(struct got_object **obj, struct got_repository *repo,
572     struct got_object_id *id)
573 {
574 	const struct got_error *err = NULL;
575 	int fd;
576 
577 	*obj = got_repo_get_cached_object(repo, id);
578 	if (*obj != NULL) {
579 		(*obj)->refcnt++;
580 		return NULL;
581 	}
582 
583 	err = got_object_open_packed(obj, id, repo);
584 	if (err && err->code != GOT_ERR_NO_OBJ)
585 		return err;
586 	if (*obj) {
587 		(*obj)->refcnt++;
588 		return got_repo_cache_object(repo, id, *obj);
589 	}
590 
591 	err = got_object_open_loose_fd(&fd, id, repo);
592 	if (err) {
593 		if (err->code == GOT_ERR_ERRNO && errno == ENOENT)
594 			err = got_error_no_obj(id);
595 		return err;
596 	}
597 
598 	err = got_object_read_header_privsep(obj, id, repo, fd);
599 	if (err)
600 		return err;
601 
602 	memcpy((*obj)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH);
603 
604 	(*obj)->refcnt++;
605 	return got_repo_cache_object(repo, id, *obj);
606 }
607 
608 /* *outfd must be initialized to -1 by caller */
609 const struct got_error *
610 got_object_raw_open(struct got_raw_object **obj, int *outfd,
611     struct got_repository *repo, struct got_object_id *id)
612 {
613 	const struct got_error *err = NULL;
614 	struct got_packidx *packidx = NULL;
615 	int idx;
616 	uint8_t *outbuf = NULL;
617 	off_t size = 0;
618 	size_t hdrlen = 0;
619 	char *path_packfile = NULL;
620 
621 	*obj = got_repo_get_cached_raw_object(repo, id);
622 	if (*obj != NULL) {
623 		(*obj)->refcnt++;
624 		return NULL;
625 	}
626 
627 	if (*outfd == -1) {
628 		*outfd = got_opentempfd();
629 		if (*outfd == -1)
630 			return got_error_from_errno("got_opentempfd");
631 	}
632 
633 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
634 	if (err == NULL) {
635 		struct got_pack *pack = NULL;
636 
637 		err = got_packidx_get_packfile_path(&path_packfile,
638 		    packidx->path_packidx);
639 		if (err)
640 			goto done;
641 
642 		pack = got_repo_get_cached_pack(repo, path_packfile);
643 		if (pack == NULL) {
644 			err = got_repo_cache_pack(&pack, repo, path_packfile,
645 			    packidx);
646 			if (err)
647 				goto done;
648 		}
649 		err = read_packed_object_raw_privsep(&outbuf, &size, &hdrlen,
650 		    *outfd, pack, packidx, idx, id);
651 		if (err)
652 			goto done;
653 	} else if (err->code == GOT_ERR_NO_OBJ) {
654 		int fd;
655 
656 		err = got_object_open_loose_fd(&fd, id, repo);
657 		if (err)
658 			goto done;
659 		err = read_object_raw_privsep(&outbuf, &size, &hdrlen, *outfd,
660 		    id, repo, fd);
661 		if (err)
662 			goto done;
663 	}
664 
665 	*obj = calloc(1, sizeof(**obj));
666 	if (*obj == NULL) {
667 		err = got_error_from_errno("calloc");
668 		goto done;
669 	}
670 	(*obj)->fd = -1;
671 
672 	if (outbuf) {
673 		(*obj)->data = outbuf;
674 	} else {
675 		struct stat sb;
676 		if (fstat(*outfd, &sb) == -1) {
677 			err = got_error_from_errno("fstat");
678 			goto done;
679 		}
680 
681 		if (sb.st_size != hdrlen + size) {
682 			err = got_error(GOT_ERR_PRIVSEP_LEN);
683 			goto done;
684 		}
685 #ifndef GOT_PACK_NO_MMAP
686 		if (hdrlen + size > 0) {
687 			(*obj)->data = mmap(NULL, hdrlen + size, PROT_READ,
688 			    MAP_PRIVATE, *outfd, 0);
689 			if ((*obj)->data == MAP_FAILED) {
690 				if (errno != ENOMEM) {
691 					err = got_error_from_errno("mmap");
692 					goto done;
693 				}
694 				(*obj)->data = NULL;
695 			} else {
696 				(*obj)->fd = *outfd;
697 				*outfd = -1;
698 			}
699 		}
700 #endif
701 		if (*outfd != -1) {
702 			(*obj)->f = fdopen(*outfd, "r");
703 			if ((*obj)->f == NULL) {
704 				err = got_error_from_errno("fdopen");
705 				goto done;
706 			}
707 			*outfd = -1;
708 		}
709 	}
710 	(*obj)->hdrlen = hdrlen;
711 	(*obj)->size = size;
712 	err = got_repo_cache_raw_object(repo, id, *obj);
713 done:
714 	free(path_packfile);
715 	if (err) {
716 		if (*obj) {
717 			got_object_raw_close(*obj);
718 			*obj = NULL;
719 		}
720 		free(outbuf);
721 	} else
722 		(*obj)->refcnt++;
723 	return err;
724 }
725 
726 const struct got_error *
727 got_object_open_by_id_str(struct got_object **obj, struct got_repository *repo,
728     const char *id_str)
729 {
730 	struct got_object_id id;
731 
732 	if (!got_parse_sha1_digest(id.sha1, id_str))
733 		return got_error_path(id_str, GOT_ERR_BAD_OBJ_ID_STR);
734 
735 	return got_object_open(obj, repo, &id);
736 }
737 
738 const struct got_error *
739 got_object_resolve_id_str(struct got_object_id **id,
740     struct got_repository *repo, const char *id_str)
741 {
742 	const struct got_error *err = NULL;
743 	struct got_object *obj;
744 
745 	err = got_object_open_by_id_str(&obj, repo, id_str);
746 	if (err)
747 		return err;
748 
749 	*id = got_object_id_dup(got_object_get_id(obj));
750 	got_object_close(obj);
751 	if (*id == NULL)
752 		return got_error_from_errno("got_object_id_dup");
753 
754 	return NULL;
755 }
756 
757 static const struct got_error *
758 request_packed_commit(struct got_commit_object **commit, struct got_pack *pack,
759     int pack_idx, struct got_object_id *id)
760 {
761 	const struct got_error *err = NULL;
762 
763 	err = got_privsep_send_commit_req(pack->privsep_child->ibuf, -1, id,
764 	    pack_idx);
765 	if (err)
766 		return err;
767 
768 	err = got_privsep_recv_commit(commit, pack->privsep_child->ibuf);
769 	if (err)
770 		return err;
771 
772 	(*commit)->flags |= GOT_COMMIT_FLAG_PACKED;
773 	return NULL;
774 }
775 
776 static const struct got_error *
777 read_packed_commit_privsep(struct got_commit_object **commit,
778     struct got_pack *pack, struct got_packidx *packidx, int idx,
779     struct got_object_id *id)
780 {
781 	const struct got_error *err = NULL;
782 
783 	if (pack->privsep_child)
784 		return request_packed_commit(commit, pack, idx, id);
785 
786 	err = start_pack_privsep_child(pack, packidx);
787 	if (err)
788 		return err;
789 
790 	return request_packed_commit(commit, pack, idx, id);
791 }
792 
793 static const struct got_error *
794 request_commit(struct got_commit_object **commit, struct got_repository *repo,
795     int fd, struct got_object_id *id)
796 {
797 	const struct got_error *err = NULL;
798 	struct imsgbuf *ibuf;
799 
800 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf;
801 
802 	err = got_privsep_send_commit_req(ibuf, fd, id, -1);
803 	if (err)
804 		return err;
805 
806 	return got_privsep_recv_commit(commit, ibuf);
807 }
808 
809 static const struct got_error *
810 read_commit_privsep(struct got_commit_object **commit, int obj_fd,
811     struct got_object_id *id, struct got_repository *repo)
812 {
813 	const struct got_error *err;
814 	int imsg_fds[2];
815 	pid_t pid;
816 	struct imsgbuf *ibuf;
817 
818 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].imsg_fd != -1)
819 		return request_commit(commit, repo, obj_fd, id);
820 
821 	ibuf = calloc(1, sizeof(*ibuf));
822 	if (ibuf == NULL)
823 		return got_error_from_errno("calloc");
824 
825 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
826 		err = got_error_from_errno("socketpair");
827 		free(ibuf);
828 		return err;
829 	}
830 
831 	pid = fork();
832 	if (pid == -1) {
833 		err = got_error_from_errno("fork");
834 		free(ibuf);
835 		return err;
836 	}
837 	else if (pid == 0) {
838 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_COMMIT,
839 		    repo->path);
840 		/* not reached */
841 	}
842 
843 	if (close(imsg_fds[1]) == -1) {
844 		err = got_error_from_errno("close");
845 		free(ibuf);
846 		return err;
847 	}
848 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].imsg_fd =
849 	    imsg_fds[0];
850 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].pid = pid;
851 	imsg_init(ibuf, imsg_fds[0]);
852 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf = ibuf;
853 
854 	return request_commit(commit, repo, obj_fd, id);
855 }
856 
857 
858 static const struct got_error *
859 open_commit(struct got_commit_object **commit,
860     struct got_repository *repo, struct got_object_id *id, int check_cache)
861 {
862 	const struct got_error *err = NULL;
863 	struct got_packidx *packidx = NULL;
864 	int idx;
865 	char *path_packfile = NULL;
866 
867 	if (check_cache) {
868 		*commit = got_repo_get_cached_commit(repo, id);
869 		if (*commit != NULL) {
870 			(*commit)->refcnt++;
871 			return NULL;
872 		}
873 	} else
874 		*commit = NULL;
875 
876 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
877 	if (err == NULL) {
878 		struct got_pack *pack = NULL;
879 
880 		err = got_packidx_get_packfile_path(&path_packfile,
881 		    packidx->path_packidx);
882 		if (err)
883 			return err;
884 
885 		pack = got_repo_get_cached_pack(repo, path_packfile);
886 		if (pack == NULL) {
887 			err = got_repo_cache_pack(&pack, repo, path_packfile,
888 			    packidx);
889 			if (err)
890 				goto done;
891 		}
892 		err = read_packed_commit_privsep(commit, pack,
893 		    packidx, idx, id);
894 	} else if (err->code == GOT_ERR_NO_OBJ) {
895 		int fd;
896 
897 		err = got_object_open_loose_fd(&fd, id, repo);
898 		if (err)
899 			return err;
900 		err = read_commit_privsep(commit, fd, id, repo);
901 	}
902 
903 	if (err == NULL) {
904 		(*commit)->refcnt++;
905 		err = got_repo_cache_commit(repo, id, *commit);
906 	}
907 done:
908 	free(path_packfile);
909 	return err;
910 }
911 
912 const struct got_error *
913 got_object_open_as_commit(struct got_commit_object **commit,
914     struct got_repository *repo, struct got_object_id *id)
915 {
916 	*commit = got_repo_get_cached_commit(repo, id);
917 	if (*commit != NULL) {
918 		(*commit)->refcnt++;
919 		return NULL;
920 	}
921 
922 	return open_commit(commit, repo, id, 0);
923 }
924 
925 const struct got_error *
926 got_object_commit_open(struct got_commit_object **commit,
927     struct got_repository *repo, struct got_object *obj)
928 {
929 	return open_commit(commit, repo, got_object_get_id(obj), 1);
930 }
931 
932 const struct got_error *
933 got_object_qid_alloc(struct got_object_qid **qid, struct got_object_id *id)
934 {
935 	*qid = calloc(1, sizeof(**qid));
936 	if (*qid == NULL)
937 		return got_error_from_errno("calloc");
938 
939 	memcpy(&(*qid)->id, id, sizeof((*qid)->id));
940 	return NULL;
941 }
942 
943 const struct got_error *
944 got_object_id_queue_copy(const struct got_object_id_queue *src,
945     struct got_object_id_queue *dest)
946 {
947 	const struct got_error *err;
948 	struct got_object_qid *qid;
949 
950 	STAILQ_FOREACH(qid, src, entry) {
951 		struct got_object_qid *new;
952 		/*
953 		 * Deep-copy the object ID only. Let the caller deal
954 		 * with setting up the new->data pointer if needed.
955 		 */
956 		err = got_object_qid_alloc(&new, &qid->id); 
957 		if (err) {
958 			got_object_id_queue_free(dest);
959 			return err;
960 		}
961 		STAILQ_INSERT_TAIL(dest, new, entry);
962 	}
963 
964 	return NULL;
965 }
966 
967 static const struct got_error *
968 request_packed_tree(struct got_tree_object **tree, struct got_pack *pack,
969     int pack_idx, struct got_object_id *id)
970 {
971 	const struct got_error *err = NULL;
972 
973 	err = got_privsep_send_tree_req(pack->privsep_child->ibuf, -1, id,
974 	    pack_idx);
975 	if (err)
976 		return err;
977 
978 	return got_privsep_recv_tree(tree, pack->privsep_child->ibuf);
979 }
980 
981 static const struct got_error *
982 read_packed_tree_privsep(struct got_tree_object **tree,
983     struct got_pack *pack, struct got_packidx *packidx, int idx,
984     struct got_object_id *id)
985 {
986 	const struct got_error *err = NULL;
987 
988 	if (pack->privsep_child)
989 		return request_packed_tree(tree, pack, idx, id);
990 
991 	err = start_pack_privsep_child(pack, packidx);
992 	if (err)
993 		return err;
994 
995 	return request_packed_tree(tree, pack, idx, id);
996 }
997 
998 static const struct got_error *
999 request_tree(struct got_tree_object **tree, struct got_repository *repo,
1000     int fd, struct got_object_id *id)
1001 {
1002 	const struct got_error *err = NULL;
1003 	struct imsgbuf *ibuf;
1004 
1005 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].ibuf;
1006 
1007 	err = got_privsep_send_tree_req(ibuf, fd, id, -1);
1008 	if (err)
1009 		return err;
1010 
1011 	return got_privsep_recv_tree(tree, ibuf);
1012 }
1013 
1014 const struct got_error *
1015 read_tree_privsep(struct got_tree_object **tree, int obj_fd,
1016     struct got_object_id *id, struct got_repository *repo)
1017 {
1018 	const struct got_error *err;
1019 	int imsg_fds[2];
1020 	pid_t pid;
1021 	struct imsgbuf *ibuf;
1022 
1023 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].imsg_fd != -1)
1024 		return request_tree(tree, repo, obj_fd, id);
1025 
1026 	ibuf = calloc(1, sizeof(*ibuf));
1027 	if (ibuf == NULL)
1028 		return got_error_from_errno("calloc");
1029 
1030 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1031 		err = got_error_from_errno("socketpair");
1032 		free(ibuf);
1033 		return err;
1034 	}
1035 
1036 	pid = fork();
1037 	if (pid == -1) {
1038 		err = got_error_from_errno("fork");
1039 		free(ibuf);
1040 		return err;
1041 	}
1042 	else if (pid == 0) {
1043 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_TREE,
1044 		    repo->path);
1045 		/* not reached */
1046 	}
1047 
1048 	if (close(imsg_fds[1]) == -1) {
1049 		err = got_error_from_errno("close");
1050 		free(ibuf);
1051 		return err;
1052 	}
1053 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].imsg_fd =
1054 	    imsg_fds[0];
1055 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].pid = pid;
1056 	imsg_init(ibuf, imsg_fds[0]);
1057 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].ibuf = ibuf;
1058 
1059 
1060 	return request_tree(tree, repo, obj_fd, id);
1061 }
1062 
1063 static const struct got_error *
1064 open_tree(struct got_tree_object **tree, struct got_repository *repo,
1065     struct got_object_id *id, int check_cache)
1066 {
1067 	const struct got_error *err = NULL;
1068 	struct got_packidx *packidx = NULL;
1069 	int idx;
1070 	char *path_packfile = NULL;
1071 
1072 	if (check_cache) {
1073 		*tree = got_repo_get_cached_tree(repo, id);
1074 		if (*tree != NULL) {
1075 			(*tree)->refcnt++;
1076 			return NULL;
1077 		}
1078 	} else
1079 		*tree = NULL;
1080 
1081 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1082 	if (err == NULL) {
1083 		struct got_pack *pack = NULL;
1084 
1085 		err = got_packidx_get_packfile_path(&path_packfile,
1086 		    packidx->path_packidx);
1087 		if (err)
1088 			return err;
1089 
1090 		pack = got_repo_get_cached_pack(repo, path_packfile);
1091 		if (pack == NULL) {
1092 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1093 			    packidx);
1094 			if (err)
1095 				goto done;
1096 		}
1097 		err = read_packed_tree_privsep(tree, pack,
1098 		    packidx, idx, id);
1099 	} else if (err->code == GOT_ERR_NO_OBJ) {
1100 		int fd;
1101 
1102 		err = got_object_open_loose_fd(&fd, id, repo);
1103 		if (err)
1104 			return err;
1105 		err = read_tree_privsep(tree, fd, id, repo);
1106 	}
1107 
1108 	if (err == NULL) {
1109 		(*tree)->refcnt++;
1110 		err = got_repo_cache_tree(repo, id, *tree);
1111 	}
1112 done:
1113 	free(path_packfile);
1114 	return err;
1115 }
1116 
1117 const struct got_error *
1118 got_object_open_as_tree(struct got_tree_object **tree,
1119     struct got_repository *repo, struct got_object_id *id)
1120 {
1121 	*tree = got_repo_get_cached_tree(repo, id);
1122 	if (*tree != NULL) {
1123 		(*tree)->refcnt++;
1124 		return NULL;
1125 	}
1126 
1127 	return open_tree(tree, repo, id, 0);
1128 }
1129 
1130 const struct got_error *
1131 got_object_tree_open(struct got_tree_object **tree,
1132     struct got_repository *repo, struct got_object *obj)
1133 {
1134 	return open_tree(tree, repo, got_object_get_id(obj), 1);
1135 }
1136 
1137 int
1138 got_object_tree_get_nentries(struct got_tree_object *tree)
1139 {
1140 	return tree->nentries;
1141 }
1142 
1143 struct got_tree_entry *
1144 got_object_tree_get_first_entry(struct got_tree_object *tree)
1145 {
1146 	return got_object_tree_get_entry(tree, 0);
1147 }
1148 
1149 struct got_tree_entry *
1150 got_object_tree_get_last_entry(struct got_tree_object *tree)
1151 {
1152 	return got_object_tree_get_entry(tree, tree->nentries - 1);
1153 }
1154 
1155 struct got_tree_entry *
1156 got_object_tree_get_entry(struct got_tree_object *tree, int i)
1157 {
1158 	if (i < 0 || i >= tree->nentries)
1159 		return NULL;
1160 	return &tree->entries[i];
1161 }
1162 
1163 mode_t
1164 got_tree_entry_get_mode(struct got_tree_entry *te)
1165 {
1166 	return te->mode;
1167 }
1168 
1169 const char *
1170 got_tree_entry_get_name(struct got_tree_entry *te)
1171 {
1172 	return &te->name[0];
1173 }
1174 
1175 struct got_object_id *
1176 got_tree_entry_get_id(struct got_tree_entry *te)
1177 {
1178 	return &te->id;
1179 }
1180 
1181 const struct got_error *
1182 got_object_blob_read_to_str(char **s, struct got_blob_object *blob)
1183 {
1184 	const struct got_error *err = NULL;
1185 	size_t len, totlen, hdrlen, offset;
1186 
1187 	*s = NULL;
1188 
1189 	hdrlen = got_object_blob_get_hdrlen(blob);
1190 	totlen = 0;
1191 	offset = 0;
1192 	do {
1193 		char *p;
1194 
1195 		err = got_object_blob_read_block(&len, blob);
1196 		if (err)
1197 			return err;
1198 
1199 		if (len == 0)
1200 			break;
1201 
1202 		totlen += len - hdrlen;
1203 		p = realloc(*s, totlen + 1);
1204 		if (p == NULL) {
1205 			err = got_error_from_errno("realloc");
1206 			free(*s);
1207 			*s = NULL;
1208 			return err;
1209 		}
1210 		*s = p;
1211 		/* Skip blob object header first time around. */
1212 		memcpy(*s + offset,
1213 		    got_object_blob_get_read_buf(blob) + hdrlen, len - hdrlen);
1214 		hdrlen = 0;
1215 		offset = totlen;
1216 	} while (len > 0);
1217 
1218 	(*s)[totlen] = '\0';
1219 	return NULL;
1220 }
1221 
1222 const struct got_error *
1223 got_tree_entry_get_symlink_target(char **link_target, struct got_tree_entry *te,
1224     struct got_repository *repo)
1225 {
1226 	const struct got_error *err = NULL;
1227 	struct got_blob_object *blob = NULL;
1228 
1229 	*link_target = NULL;
1230 
1231 	if (!got_object_tree_entry_is_symlink(te))
1232 		return got_error(GOT_ERR_TREE_ENTRY_TYPE);
1233 
1234 	err = got_object_open_as_blob(&blob, repo,
1235 	    got_tree_entry_get_id(te), PATH_MAX);
1236 	if (err)
1237 		return err;
1238 
1239 	err = got_object_blob_read_to_str(link_target, blob);
1240 	got_object_blob_close(blob);
1241 	if (err) {
1242 		free(*link_target);
1243 		*link_target = NULL;
1244 	}
1245 	return err;
1246 }
1247 
1248 int
1249 got_tree_entry_get_index(struct got_tree_entry *te)
1250 {
1251 	return te->idx;
1252 }
1253 
1254 struct got_tree_entry *
1255 got_tree_entry_get_next(struct got_tree_object *tree,
1256     struct got_tree_entry *te)
1257 {
1258 	return got_object_tree_get_entry(tree, te->idx + 1);
1259 }
1260 
1261 struct got_tree_entry *
1262 got_tree_entry_get_prev(struct got_tree_object *tree,
1263     struct got_tree_entry *te)
1264 {
1265 	return got_object_tree_get_entry(tree, te->idx - 1);
1266 }
1267 
1268 static const struct got_error *
1269 request_packed_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd,
1270     struct got_pack *pack, struct got_packidx *packidx, int idx,
1271     struct got_object_id *id)
1272 {
1273 	const struct got_error *err = NULL;
1274 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
1275 	int outfd_child;
1276 
1277 	 err = pack_child_send_tempfiles(ibuf, pack);
1278 	 if (err)
1279 		return err;
1280 
1281 	outfd_child = dup(outfd);
1282 	if (outfd_child == -1)
1283 		return got_error_from_errno("dup");
1284 
1285 	err = got_privsep_send_blob_req(pack->privsep_child->ibuf, -1, id, idx);
1286 	if (err)
1287 		return err;
1288 
1289 	err = got_privsep_send_blob_outfd(pack->privsep_child->ibuf,
1290 	    outfd_child);
1291 	if (err) {
1292 		return err;
1293 	}
1294 
1295 	err = got_privsep_recv_blob(outbuf, size, hdrlen,
1296 	    pack->privsep_child->ibuf);
1297 	if (err)
1298 		return err;
1299 
1300 	if (lseek(outfd, SEEK_SET, 0) == -1)
1301 		err = got_error_from_errno("lseek");
1302 
1303 	return err;
1304 }
1305 
1306 static const struct got_error *
1307 read_packed_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen,
1308     int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
1309     struct got_object_id *id)
1310 {
1311 	const struct got_error *err = NULL;
1312 
1313 	if (pack->privsep_child == NULL) {
1314 		err = start_pack_privsep_child(pack, packidx);
1315 		if (err)
1316 			return err;
1317 	}
1318 
1319 	return request_packed_blob(outbuf, size, hdrlen, outfd, pack, packidx,
1320 	    idx, id);
1321 }
1322 
1323 static const struct got_error *
1324 request_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd,
1325     int infd, struct got_object_id *id, struct imsgbuf *ibuf)
1326 {
1327 	const struct got_error *err = NULL;
1328 	int outfd_child;
1329 
1330 	outfd_child = dup(outfd);
1331 	if (outfd_child == -1)
1332 		return got_error_from_errno("dup");
1333 
1334 	err = got_privsep_send_blob_req(ibuf, infd, id, -1);
1335 	if (err)
1336 		return err;
1337 
1338 	err = got_privsep_send_blob_outfd(ibuf, outfd_child);
1339 	if (err)
1340 		return err;
1341 
1342 	err = got_privsep_recv_blob(outbuf, size, hdrlen, ibuf);
1343 	if (err)
1344 		return err;
1345 
1346 	if (lseek(outfd, SEEK_SET, 0) == -1)
1347 		return got_error_from_errno("lseek");
1348 
1349 	return err;
1350 }
1351 
1352 static const struct got_error *
1353 read_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen,
1354     int outfd, int infd, struct got_object_id *id, struct got_repository *repo)
1355 {
1356 	const struct got_error *err;
1357 	int imsg_fds[2];
1358 	pid_t pid;
1359 	struct imsgbuf *ibuf;
1360 
1361 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) {
1362 		ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf;
1363 		return request_blob(outbuf, size, hdrlen, outfd, infd, id,
1364 		    ibuf);
1365 	}
1366 
1367 	ibuf = calloc(1, sizeof(*ibuf));
1368 	if (ibuf == NULL)
1369 		return got_error_from_errno("calloc");
1370 
1371 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1372 		err = got_error_from_errno("socketpair");
1373 		free(ibuf);
1374 		return err;
1375 	}
1376 
1377 	pid = fork();
1378 	if (pid == -1) {
1379 		err = got_error_from_errno("fork");
1380 		free(ibuf);
1381 		return err;
1382 	}
1383 	else if (pid == 0) {
1384 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_BLOB,
1385 		    repo->path);
1386 		/* not reached */
1387 	}
1388 
1389 	if (close(imsg_fds[1]) == -1) {
1390 		err = got_error_from_errno("close");
1391 		free(ibuf);
1392 		return err;
1393 	}
1394 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd =
1395 	    imsg_fds[0];
1396 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].pid = pid;
1397 	imsg_init(ibuf, imsg_fds[0]);
1398 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf;
1399 
1400 	return request_blob(outbuf, size, hdrlen, outfd, infd, id, ibuf);
1401 }
1402 
1403 static const struct got_error *
1404 open_blob(struct got_blob_object **blob, struct got_repository *repo,
1405     struct got_object_id *id, size_t blocksize)
1406 {
1407 	const struct got_error *err = NULL;
1408 	struct got_packidx *packidx = NULL;
1409 	int idx;
1410 	char *path_packfile = NULL;
1411 	uint8_t *outbuf;
1412 	int outfd;
1413 	size_t size, hdrlen;
1414 	struct stat sb;
1415 
1416 	*blob = calloc(1, sizeof(**blob));
1417 	if (*blob == NULL)
1418 		return got_error_from_errno("calloc");
1419 
1420 	outfd = got_opentempfd();
1421 	if (outfd == -1)
1422 		return got_error_from_errno("got_opentempfd");
1423 
1424 	(*blob)->read_buf = malloc(blocksize);
1425 	if ((*blob)->read_buf == NULL) {
1426 		err = got_error_from_errno("malloc");
1427 		goto done;
1428 	}
1429 
1430 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1431 	if (err == NULL) {
1432 		struct got_pack *pack = NULL;
1433 
1434 		err = got_packidx_get_packfile_path(&path_packfile,
1435 		    packidx->path_packidx);
1436 		if (err)
1437 			goto done;
1438 
1439 		pack = got_repo_get_cached_pack(repo, path_packfile);
1440 		if (pack == NULL) {
1441 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1442 			    packidx);
1443 			if (err)
1444 				goto done;
1445 		}
1446 		err = read_packed_blob_privsep(&outbuf, &size, &hdrlen, outfd,
1447 		    pack, packidx, idx, id);
1448 	} else if (err->code == GOT_ERR_NO_OBJ) {
1449 		int infd;
1450 
1451 		err = got_object_open_loose_fd(&infd, id, repo);
1452 		if (err)
1453 			goto done;
1454 		err = read_blob_privsep(&outbuf, &size, &hdrlen, outfd, infd,
1455 		    id, repo);
1456 	}
1457 	if (err)
1458 		goto done;
1459 
1460 	if (hdrlen > size) {
1461 		err = got_error(GOT_ERR_BAD_OBJ_HDR);
1462 		goto done;
1463 	}
1464 
1465 	if (outbuf) {
1466 		if (close(outfd) == -1 && err == NULL)
1467 			err = got_error_from_errno("close");
1468 		outfd = -1;
1469 		(*blob)->f = fmemopen(outbuf, size, "rb");
1470 		if ((*blob)->f == NULL) {
1471 			err = got_error_from_errno("fmemopen");
1472 			free(outbuf);
1473 			goto done;
1474 		}
1475 		(*blob)->data = outbuf;
1476 	} else {
1477 		if (fstat(outfd, &sb) == -1) {
1478 			err = got_error_from_errno("fstat");
1479 			goto done;
1480 		}
1481 
1482 		if (sb.st_size != size) {
1483 			err = got_error(GOT_ERR_PRIVSEP_LEN);
1484 			goto done;
1485 		}
1486 
1487 		(*blob)->f = fdopen(outfd, "rb");
1488 		if ((*blob)->f == NULL) {
1489 			err = got_error_from_errno("fdopen");
1490 			close(outfd);
1491 			outfd = -1;
1492 			goto done;
1493 		}
1494 	}
1495 
1496 	(*blob)->hdrlen = hdrlen;
1497 	(*blob)->blocksize = blocksize;
1498 	memcpy(&(*blob)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH);
1499 
1500 done:
1501 	free(path_packfile);
1502 	if (err) {
1503 		if (*blob) {
1504 			got_object_blob_close(*blob);
1505 			*blob = NULL;
1506 		} else if (outfd != -1)
1507 			close(outfd);
1508 	}
1509 	return err;
1510 }
1511 
1512 const struct got_error *
1513 got_object_open_as_blob(struct got_blob_object **blob,
1514     struct got_repository *repo, struct got_object_id *id,
1515     size_t blocksize)
1516 {
1517 	return open_blob(blob, repo, id, blocksize);
1518 }
1519 
1520 const struct got_error *
1521 got_object_blob_open(struct got_blob_object **blob,
1522     struct got_repository *repo, struct got_object *obj, size_t blocksize)
1523 {
1524 	return open_blob(blob, repo, got_object_get_id(obj), blocksize);
1525 }
1526 
1527 const struct got_error *
1528 got_object_blob_close(struct got_blob_object *blob)
1529 {
1530 	const struct got_error *err = NULL;
1531 	free(blob->read_buf);
1532 	if (blob->f && fclose(blob->f) == EOF)
1533 		err = got_error_from_errno("fclose");
1534 	free(blob->data);
1535 	free(blob);
1536 	return err;
1537 }
1538 
1539 void
1540 got_object_blob_rewind(struct got_blob_object *blob)
1541 {
1542 	if (blob->f)
1543 		rewind(blob->f);
1544 }
1545 
1546 char *
1547 got_object_blob_id_str(struct got_blob_object *blob, char *buf, size_t size)
1548 {
1549 	return got_sha1_digest_to_str(blob->id.sha1, buf, size);
1550 }
1551 
1552 size_t
1553 got_object_blob_get_hdrlen(struct got_blob_object *blob)
1554 {
1555 	return blob->hdrlen;
1556 }
1557 
1558 const uint8_t *
1559 got_object_blob_get_read_buf(struct got_blob_object *blob)
1560 {
1561 	return blob->read_buf;
1562 }
1563 
1564 const struct got_error *
1565 got_object_blob_read_block(size_t *outlenp, struct got_blob_object *blob)
1566 {
1567 	size_t n;
1568 
1569 	n = fread(blob->read_buf, 1, blob->blocksize, blob->f);
1570 	if (n == 0 && ferror(blob->f))
1571 		return got_ferror(blob->f, GOT_ERR_IO);
1572 	*outlenp = n;
1573 	return NULL;
1574 }
1575 
1576 const struct got_error *
1577 got_object_blob_dump_to_file(off_t *filesize, int *nlines,
1578     off_t **line_offsets, FILE *outfile, struct got_blob_object *blob)
1579 {
1580 	const struct got_error *err = NULL;
1581 	size_t n, len, hdrlen;
1582 	const uint8_t *buf;
1583 	int i;
1584 	const int alloc_chunksz = 512;
1585 	size_t nalloc = 0;
1586 	off_t off = 0, total_len = 0;
1587 
1588 	if (line_offsets)
1589 		*line_offsets = NULL;
1590 	if (filesize)
1591 		*filesize = 0;
1592 	if (nlines)
1593 		*nlines = 0;
1594 
1595 	hdrlen = got_object_blob_get_hdrlen(blob);
1596 	do {
1597 		err = got_object_blob_read_block(&len, blob);
1598 		if (err)
1599 			return err;
1600 		if (len == 0)
1601 			break;
1602 		buf = got_object_blob_get_read_buf(blob);
1603 		i = hdrlen;
1604 		if (nlines) {
1605 			if (line_offsets && *line_offsets == NULL) {
1606 				/* Have some data but perhaps no '\n'. */
1607 				*nlines = 1;
1608 				nalloc = alloc_chunksz;
1609 				*line_offsets = calloc(nalloc,
1610 				    sizeof(**line_offsets));
1611 				if (*line_offsets == NULL)
1612 					return got_error_from_errno("calloc");
1613 
1614 				/* Skip forward over end of first line. */
1615 				while (i < len) {
1616 					if (buf[i] == '\n')
1617 						break;
1618 					i++;
1619 				}
1620 			}
1621 			/* Scan '\n' offsets in remaining chunk of data. */
1622 			while (i < len) {
1623 				if (buf[i] != '\n') {
1624 					i++;
1625 					continue;
1626 				}
1627 				(*nlines)++;
1628 				if (line_offsets && nalloc < *nlines) {
1629 					size_t n = *nlines + alloc_chunksz;
1630 					off_t *o = recallocarray(*line_offsets,
1631 					    nalloc, n, sizeof(**line_offsets));
1632 					if (o == NULL) {
1633 						free(*line_offsets);
1634 						*line_offsets = NULL;
1635 						return got_error_from_errno(
1636 						    "recallocarray");
1637 					}
1638 					*line_offsets = o;
1639 					nalloc = n;
1640 				}
1641 				if (line_offsets) {
1642 					off = total_len + i - hdrlen + 1;
1643 					(*line_offsets)[*nlines - 1] = off;
1644 				}
1645 				i++;
1646 			}
1647 		}
1648 		/* Skip blob object header first time around. */
1649 		n = fwrite(buf + hdrlen, 1, len - hdrlen, outfile);
1650 		if (n != len - hdrlen)
1651 			return got_ferror(outfile, GOT_ERR_IO);
1652 		total_len += len - hdrlen;
1653 		hdrlen = 0;
1654 	} while (len != 0);
1655 
1656 	if (fflush(outfile) != 0)
1657 		return got_error_from_errno("fflush");
1658 	rewind(outfile);
1659 
1660 	if (filesize)
1661 		*filesize = total_len;
1662 
1663 	return NULL;
1664 }
1665 
1666 static const struct got_error *
1667 request_packed_tag(struct got_tag_object **tag, struct got_pack *pack,
1668     int pack_idx, struct got_object_id *id)
1669 {
1670 	const struct got_error *err = NULL;
1671 
1672 	err = got_privsep_send_tag_req(pack->privsep_child->ibuf, -1, id,
1673 	    pack_idx);
1674 	if (err)
1675 		return err;
1676 
1677 	return got_privsep_recv_tag(tag, pack->privsep_child->ibuf);
1678 }
1679 
1680 static const struct got_error *
1681 read_packed_tag_privsep(struct got_tag_object **tag,
1682     struct got_pack *pack, struct got_packidx *packidx, int idx,
1683     struct got_object_id *id)
1684 {
1685 	const struct got_error *err = NULL;
1686 
1687 	if (pack->privsep_child)
1688 		return request_packed_tag(tag, pack, idx, id);
1689 
1690 	err = start_pack_privsep_child(pack, packidx);
1691 	if (err)
1692 		return err;
1693 
1694 	return request_packed_tag(tag, pack, idx, id);
1695 }
1696 
1697 static const struct got_error *
1698 request_tag(struct got_tag_object **tag, struct got_repository *repo,
1699     int fd, struct got_object_id *id)
1700 {
1701 	const struct got_error *err = NULL;
1702 	struct imsgbuf *ibuf;
1703 
1704 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].ibuf;
1705 
1706 	err = got_privsep_send_tag_req(ibuf, fd, id, -1);
1707 	if (err)
1708 		return err;
1709 
1710 	return got_privsep_recv_tag(tag, ibuf);
1711 }
1712 
1713 static const struct got_error *
1714 read_tag_privsep(struct got_tag_object **tag, int obj_fd,
1715     struct got_object_id *id, struct got_repository *repo)
1716 {
1717 	const struct got_error *err;
1718 	int imsg_fds[2];
1719 	pid_t pid;
1720 	struct imsgbuf *ibuf;
1721 
1722 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].imsg_fd != -1)
1723 		return request_tag(tag, repo, obj_fd, id);
1724 
1725 	ibuf = calloc(1, sizeof(*ibuf));
1726 	if (ibuf == NULL)
1727 		return got_error_from_errno("calloc");
1728 
1729 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1730 		err = got_error_from_errno("socketpair");
1731 		free(ibuf);
1732 		return err;
1733 	}
1734 
1735 	pid = fork();
1736 	if (pid == -1) {
1737 		err = got_error_from_errno("fork");
1738 		free(ibuf);
1739 		return err;
1740 	}
1741 	else if (pid == 0) {
1742 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_TAG,
1743 		    repo->path);
1744 		/* not reached */
1745 	}
1746 
1747 	if (close(imsg_fds[1]) == -1) {
1748 		err = got_error_from_errno("close");
1749 		free(ibuf);
1750 		return err;
1751 	}
1752 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].imsg_fd =
1753 	    imsg_fds[0];
1754 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].pid = pid;
1755 	imsg_init(ibuf, imsg_fds[0]);
1756 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].ibuf = ibuf;
1757 
1758 	return request_tag(tag, repo, obj_fd, id);
1759 }
1760 
1761 static const struct got_error *
1762 open_tag(struct got_tag_object **tag, struct got_repository *repo,
1763     struct got_object_id *id, int check_cache)
1764 {
1765 	const struct got_error *err = NULL;
1766 	struct got_packidx *packidx = NULL;
1767 	int idx;
1768 	char *path_packfile = NULL;
1769 	struct got_object *obj = NULL;
1770 	int obj_type = GOT_OBJ_TYPE_ANY;
1771 
1772 	if (check_cache) {
1773 		*tag = got_repo_get_cached_tag(repo, id);
1774 		if (*tag != NULL) {
1775 			(*tag)->refcnt++;
1776 			return NULL;
1777 		}
1778 	} else
1779 		*tag = NULL;
1780 
1781 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1782 	if (err == NULL) {
1783 		struct got_pack *pack = NULL;
1784 
1785 		err = got_packidx_get_packfile_path(&path_packfile,
1786 		    packidx->path_packidx);
1787 		if (err)
1788 			return err;
1789 
1790 		pack = got_repo_get_cached_pack(repo, path_packfile);
1791 		if (pack == NULL) {
1792 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1793 			    packidx);
1794 			if (err)
1795 				goto done;
1796 		}
1797 
1798 		/* Beware of "lightweight" tags: Check object type first. */
1799 		err = read_packed_object_privsep(&obj, repo, pack, packidx,
1800 		    idx, id);
1801 		if (err)
1802 			goto done;
1803 		obj_type = obj->type;
1804 		got_object_close(obj);
1805 		if (obj_type != GOT_OBJ_TYPE_TAG) {
1806 			err = got_error(GOT_ERR_OBJ_TYPE);
1807 			goto done;
1808 		}
1809 		err = read_packed_tag_privsep(tag, pack, packidx, idx, id);
1810 	} else if (err->code == GOT_ERR_NO_OBJ) {
1811 		int fd;
1812 
1813 		err = got_object_open_loose_fd(&fd, id, repo);
1814 		if (err)
1815 			return err;
1816 		err = got_object_read_header_privsep(&obj, id, repo, fd);
1817 		if (err)
1818 			return err;
1819 		obj_type = obj->type;
1820 		got_object_close(obj);
1821 		if (obj_type != GOT_OBJ_TYPE_TAG)
1822 			return got_error(GOT_ERR_OBJ_TYPE);
1823 
1824 		err = got_object_open_loose_fd(&fd, id, repo);
1825 		if (err)
1826 			return err;
1827 		err = read_tag_privsep(tag, fd, id, repo);
1828 	}
1829 
1830 	if (err == NULL) {
1831 		(*tag)->refcnt++;
1832 		err = got_repo_cache_tag(repo, id, *tag);
1833 	}
1834 done:
1835 	free(path_packfile);
1836 	return err;
1837 }
1838 
1839 const struct got_error *
1840 got_object_open_as_tag(struct got_tag_object **tag,
1841     struct got_repository *repo, struct got_object_id *id)
1842 {
1843 	*tag = got_repo_get_cached_tag(repo, id);
1844 	if (*tag != NULL) {
1845 		(*tag)->refcnt++;
1846 		return NULL;
1847 	}
1848 
1849 	return open_tag(tag, repo, id, 0);
1850 }
1851 
1852 const struct got_error *
1853 got_object_tag_open(struct got_tag_object **tag,
1854     struct got_repository *repo, struct got_object *obj)
1855 {
1856 	return open_tag(tag, repo, got_object_get_id(obj), 1);
1857 }
1858 
1859 const char *
1860 got_object_tag_get_name(struct got_tag_object *tag)
1861 {
1862 	return tag->tag;
1863 }
1864 
1865 int
1866 got_object_tag_get_object_type(struct got_tag_object *tag)
1867 {
1868 	return tag->obj_type;
1869 }
1870 
1871 struct got_object_id *
1872 got_object_tag_get_object_id(struct got_tag_object *tag)
1873 {
1874 	return &tag->id;
1875 }
1876 
1877 time_t
1878 got_object_tag_get_tagger_time(struct got_tag_object *tag)
1879 {
1880 	return tag->tagger_time;
1881 }
1882 
1883 time_t
1884 got_object_tag_get_tagger_gmtoff(struct got_tag_object *tag)
1885 {
1886 	return tag->tagger_gmtoff;
1887 }
1888 
1889 const char *
1890 got_object_tag_get_tagger(struct got_tag_object *tag)
1891 {
1892 	return tag->tagger;
1893 }
1894 
1895 const char *
1896 got_object_tag_get_message(struct got_tag_object *tag)
1897 {
1898 	return tag->tagmsg;
1899 }
1900 
1901 static struct got_tree_entry *
1902 find_entry_by_name(struct got_tree_object *tree, const char *name, size_t len)
1903 {
1904 	int i;
1905 
1906 	/* Note that tree entries are sorted in strncmp() order. */
1907 	for (i = 0; i < tree->nentries; i++) {
1908 		struct got_tree_entry *te = &tree->entries[i];
1909 		int cmp = strncmp(te->name, name, len);
1910 		if (cmp < 0)
1911 			continue;
1912 		if (cmp > 0)
1913 			break;
1914 		if (te->name[len] == '\0')
1915 			return te;
1916 	}
1917 	return NULL;
1918 }
1919 
1920 struct got_tree_entry *
1921 got_object_tree_find_entry(struct got_tree_object *tree, const char *name)
1922 {
1923 	return find_entry_by_name(tree, name, strlen(name));
1924 }
1925 
1926 const struct got_error *
1927 got_object_tree_find_path(struct got_object_id **id, mode_t *mode,
1928     struct got_repository *repo, struct got_tree_object *tree,
1929     const char *path)
1930 {
1931 	const struct got_error *err = NULL;
1932 	struct got_tree_object *subtree = NULL;
1933 	struct got_tree_entry *te = NULL;
1934 	const char *seg, *s;
1935 	size_t seglen;
1936 
1937 	*id = NULL;
1938 
1939 	s = path;
1940 	while (s[0] == '/')
1941 		s++;
1942 	seg = s;
1943 	seglen = 0;
1944 	subtree = tree;
1945 	while (*s) {
1946 		struct got_tree_object *next_tree;
1947 
1948 		if (*s != '/') {
1949 			s++;
1950 			seglen++;
1951 			if (*s)
1952 				continue;
1953 		}
1954 
1955 		te = find_entry_by_name(subtree, seg, seglen);
1956 		if (te == NULL) {
1957 			err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
1958 			goto done;
1959 		}
1960 
1961 		if (*s == '\0')
1962 			break;
1963 
1964 		seg = s + 1;
1965 		seglen = 0;
1966 		s++;
1967 		if (*s) {
1968 			err = got_object_open_as_tree(&next_tree, repo,
1969 			    &te->id);
1970 			te = NULL;
1971 			if (err)
1972 				goto done;
1973 			if (subtree != tree)
1974 				got_object_tree_close(subtree);
1975 			subtree = next_tree;
1976 		}
1977 	}
1978 
1979 	if (te) {
1980 		*id = got_object_id_dup(&te->id);
1981 		if (*id == NULL)
1982 			return got_error_from_errno("got_object_id_dup");
1983 		if (mode)
1984 			*mode = te->mode;
1985 	} else
1986 		err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
1987 done:
1988 	if (subtree && subtree != tree)
1989 		got_object_tree_close(subtree);
1990 	return err;
1991 }
1992 const struct got_error *
1993 got_object_id_by_path(struct got_object_id **id, struct got_repository *repo,
1994     struct got_commit_object *commit, const char *path)
1995 {
1996 	const struct got_error *err = NULL;
1997 	struct got_tree_object *tree = NULL;
1998 
1999 	*id = NULL;
2000 
2001 	/* Handle opening of root of commit's tree. */
2002 	if (got_path_is_root_dir(path)) {
2003 		*id = got_object_id_dup(commit->tree_id);
2004 		if (*id == NULL)
2005 			err = got_error_from_errno("got_object_id_dup");
2006 	} else {
2007 		err = got_object_open_as_tree(&tree, repo, commit->tree_id);
2008 		if (err)
2009 			goto done;
2010 		err = got_object_tree_find_path(id, NULL, repo, tree, path);
2011 	}
2012 done:
2013 	if (tree)
2014 		got_object_tree_close(tree);
2015 	return err;
2016 }
2017 
2018 /*
2019  * Normalize file mode bits to avoid false positive tree entry differences
2020  * in case tree entries have unexpected mode bits set.
2021  */
2022 static mode_t
2023 normalize_mode_for_comparison(mode_t mode)
2024 {
2025 	/*
2026 	 * For directories, the only relevant bit is the IFDIR bit.
2027 	 * This allows us to detect paths changing from a directory
2028 	 * to a file and vice versa.
2029 	 */
2030 	if (S_ISDIR(mode))
2031 		return mode & S_IFDIR;
2032 
2033 	/*
2034 	 * For symlinks, the only relevant bit is the IFLNK bit.
2035 	 * This allows us to detect paths changing from a symlinks
2036 	 * to a file or directory and vice versa.
2037 	 */
2038 	if (S_ISLNK(mode))
2039 		return mode & S_IFLNK;
2040 
2041 	/* For files, the only change we care about is the executable bit. */
2042 	return mode & S_IXUSR;
2043 }
2044 
2045 const struct got_error *
2046 got_object_tree_path_changed(int *changed,
2047     struct got_tree_object *tree01, struct got_tree_object *tree02,
2048     const char *path, struct got_repository *repo)
2049 {
2050 	const struct got_error *err = NULL;
2051 	struct got_tree_object *tree1 = NULL, *tree2 = NULL;
2052 	struct got_tree_entry *te1 = NULL, *te2 = NULL;
2053 	const char *seg, *s;
2054 	size_t seglen;
2055 
2056 	*changed = 0;
2057 
2058 	/* We not do support comparing the root path. */
2059 	if (got_path_is_root_dir(path))
2060 		return got_error_path(path, GOT_ERR_BAD_PATH);
2061 
2062 	tree1 = tree01;
2063 	tree2 = tree02;
2064 	s = path;
2065 	while (*s == '/')
2066 		s++;
2067 	seg = s;
2068 	seglen = 0;
2069 	while (*s) {
2070 		struct got_tree_object *next_tree1, *next_tree2;
2071 		mode_t mode1, mode2;
2072 
2073 		if (*s != '/') {
2074 			s++;
2075 			seglen++;
2076 			if (*s)
2077 				continue;
2078 		}
2079 
2080 		te1 = find_entry_by_name(tree1, seg, seglen);
2081 		if (te1 == NULL) {
2082 			err = got_error(GOT_ERR_NO_OBJ);
2083 			goto done;
2084 		}
2085 
2086 		if (tree2)
2087 			te2 = find_entry_by_name(tree2, seg, seglen);
2088 
2089 		if (te2) {
2090 			mode1 = normalize_mode_for_comparison(te1->mode);
2091 			mode2 = normalize_mode_for_comparison(te2->mode);
2092 			if (mode1 != mode2) {
2093 				*changed = 1;
2094 				goto done;
2095 			}
2096 
2097 			if (got_object_id_cmp(&te1->id, &te2->id) == 0) {
2098 				*changed = 0;
2099 				goto done;
2100 			}
2101 		}
2102 
2103 		if (*s == '\0') { /* final path element */
2104 			*changed = 1;
2105 			goto done;
2106 		}
2107 
2108 		seg = s + 1;
2109 		s++;
2110 		seglen = 0;
2111 		if (*s) {
2112 			err = got_object_open_as_tree(&next_tree1, repo,
2113 			    &te1->id);
2114 			te1 = NULL;
2115 			if (err)
2116 				goto done;
2117 			if (tree1 != tree01)
2118 				got_object_tree_close(tree1);
2119 			tree1 = next_tree1;
2120 
2121 			if (te2) {
2122 				err = got_object_open_as_tree(&next_tree2, repo,
2123 				    &te2->id);
2124 				te2 = NULL;
2125 				if (err)
2126 					goto done;
2127 				if (tree2 != tree02)
2128 					got_object_tree_close(tree2);
2129 				tree2 = next_tree2;
2130 			} else if (tree2) {
2131 				if (tree2 != tree02)
2132 					got_object_tree_close(tree2);
2133 				tree2 = NULL;
2134 			}
2135 		}
2136 	}
2137 done:
2138 	if (tree1 && tree1 != tree01)
2139 		got_object_tree_close(tree1);
2140 	if (tree2 && tree2 != tree02)
2141 		got_object_tree_close(tree2);
2142 	return err;
2143 }
2144 
2145 const struct got_error *
2146 got_object_tree_entry_dup(struct got_tree_entry **new_te,
2147     struct got_tree_entry *te)
2148 {
2149 	const struct got_error *err = NULL;
2150 
2151 	*new_te = calloc(1, sizeof(**new_te));
2152 	if (*new_te == NULL)
2153 		return got_error_from_errno("calloc");
2154 
2155 	(*new_te)->mode = te->mode;
2156 	memcpy((*new_te)->name, te->name, sizeof((*new_te)->name));
2157 	memcpy(&(*new_te)->id, &te->id, sizeof((*new_te)->id));
2158 	return err;
2159 }
2160 
2161 int
2162 got_object_tree_entry_is_submodule(struct got_tree_entry *te)
2163 {
2164 	return (te->mode & S_IFMT) == (S_IFDIR | S_IFLNK);
2165 }
2166 
2167 int
2168 got_object_tree_entry_is_symlink(struct got_tree_entry *te)
2169 {
2170 	/* S_IFDIR check avoids confusing symlinks with submodules. */
2171 	return ((te->mode & (S_IFDIR | S_IFLNK)) == S_IFLNK);
2172 }
2173 
2174 static const struct got_error *
2175 resolve_symlink(char **link_target, const char *path,
2176     struct got_commit_object *commit, struct got_repository *repo)
2177 {
2178 	const struct got_error *err = NULL;
2179 	char buf[PATH_MAX];
2180 	char *name, *parent_path = NULL;
2181 	struct got_object_id *tree_obj_id = NULL;
2182 	struct got_tree_object *tree = NULL;
2183 	struct got_tree_entry *te = NULL;
2184 
2185 	*link_target = NULL;
2186 
2187 	if (strlcpy(buf, path, sizeof(buf)) >= sizeof(buf))
2188 		return got_error(GOT_ERR_NO_SPACE);
2189 
2190 	name = basename(buf);
2191 	if (name == NULL)
2192 		return got_error_from_errno2("basename", path);
2193 
2194 	err = got_path_dirname(&parent_path, path);
2195 	if (err)
2196 		return err;
2197 
2198 	err = got_object_id_by_path(&tree_obj_id, repo, commit,
2199 	    parent_path);
2200 	if (err) {
2201 		if (err->code == GOT_ERR_NO_TREE_ENTRY) {
2202 			/* Display the complete path in error message. */
2203 			err = got_error_path(path, err->code);
2204 		}
2205 		goto done;
2206 	}
2207 
2208 	err = got_object_open_as_tree(&tree, repo, tree_obj_id);
2209 	if (err)
2210 		goto done;
2211 
2212 	te = got_object_tree_find_entry(tree, name);
2213 	if (te == NULL) {
2214 		err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
2215 		goto done;
2216 	}
2217 
2218 	if (got_object_tree_entry_is_symlink(te)) {
2219 		err = got_tree_entry_get_symlink_target(link_target, te, repo);
2220 		if (err)
2221 			goto done;
2222 		if (!got_path_is_absolute(*link_target)) {
2223 			char *abspath;
2224 			if (asprintf(&abspath, "%s/%s", parent_path,
2225 			    *link_target) == -1) {
2226 				err = got_error_from_errno("asprintf");
2227 				goto done;
2228 			}
2229 			free(*link_target);
2230 			*link_target = malloc(PATH_MAX);
2231 			if (*link_target == NULL) {
2232 				err = got_error_from_errno("malloc");
2233 				goto done;
2234 			}
2235 			err = got_canonpath(abspath, *link_target, PATH_MAX);
2236 			free(abspath);
2237 			if (err)
2238 				goto done;
2239 		}
2240 	}
2241 done:
2242 	free(tree_obj_id);
2243 	if (tree)
2244 		got_object_tree_close(tree);
2245 	if (err) {
2246 		free(*link_target);
2247 		*link_target = NULL;
2248 	}
2249 	return err;
2250 }
2251 
2252 const struct got_error *
2253 got_object_resolve_symlinks(char **link_target, const char *path,
2254     struct got_commit_object *commit, struct got_repository *repo)
2255 {
2256 	const struct got_error *err = NULL;
2257 	char *next_target = NULL;
2258 	int max_recursion = 40; /* matches Git */
2259 
2260 	*link_target = NULL;
2261 
2262 	do {
2263 		err = resolve_symlink(&next_target,
2264 		    *link_target ? *link_target : path, commit, repo);
2265 		if (err)
2266 			break;
2267 		if (next_target) {
2268 			free(*link_target);
2269 			if (--max_recursion == 0) {
2270 				err = got_error_path(path, GOT_ERR_RECURSION);
2271 				*link_target = NULL;
2272 				break;
2273 			}
2274 			*link_target = next_target;
2275 		}
2276 	} while (next_target);
2277 
2278 	return err;
2279 }
2280 
2281 const struct got_error *
2282 got_traverse_packed_commits(struct got_object_id_queue *traversed_commits,
2283     struct got_object_id *commit_id, const char *path,
2284     struct got_repository *repo)
2285 {
2286 	const struct got_error *err = NULL;
2287 	struct got_pack *pack = NULL;
2288 	struct got_packidx *packidx = NULL;
2289 	char *path_packfile = NULL;
2290 	struct got_commit_object *changed_commit = NULL;
2291 	struct got_object_id *changed_commit_id = NULL;
2292 	int idx;
2293 
2294 	err = got_repo_search_packidx(&packidx, &idx, repo, commit_id);
2295 	if (err) {
2296 		if (err->code != GOT_ERR_NO_OBJ)
2297 			return err;
2298 		return NULL;
2299 	}
2300 
2301 	err = got_packidx_get_packfile_path(&path_packfile,
2302 	    packidx->path_packidx);
2303 	if (err)
2304 		return err;
2305 
2306 	pack = got_repo_get_cached_pack(repo, path_packfile);
2307 	if (pack == NULL) {
2308 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
2309 		if (err)
2310 			goto done;
2311 	}
2312 
2313 	if (pack->privsep_child == NULL) {
2314 		err = start_pack_privsep_child(pack, packidx);
2315 		if (err)
2316 			goto done;
2317 	}
2318 
2319 	err = got_privsep_send_commit_traversal_request(
2320 	    pack->privsep_child->ibuf, commit_id, idx, path);
2321 	if (err)
2322 		goto done;
2323 
2324 	err = got_privsep_recv_traversed_commits(&changed_commit,
2325 	    &changed_commit_id, traversed_commits, pack->privsep_child->ibuf);
2326 	if (err)
2327 		goto done;
2328 
2329 	if (changed_commit) {
2330 		/*
2331 		 * Cache the commit in which the path was changed.
2332 		 * This commit might be opened again soon.
2333 		 */
2334 		changed_commit->refcnt++;
2335 		err = got_repo_cache_commit(repo, changed_commit_id,
2336 		    changed_commit);
2337 		got_object_commit_close(changed_commit);
2338 	}
2339 done:
2340 	free(path_packfile);
2341 	free(changed_commit_id);
2342 	return err;
2343 }