1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/mman.h>
23 #include <sys/resource.h>
24 
25 #include <ctype.h>
26 #include <fcntl.h>
27 #include <fnmatch.h>
28 #include <limits.h>
29 #include <dirent.h>
30 #include <stdlib.h>
31 #include <stdio.h>
32 #include <sha2.h>
33 #include <string.h>
34 #include <time.h>
35 #include <unistd.h>
36 #include <zlib.h>
37 #include <errno.h>
38 #include <libgen.h>
39 #include <stdint.h>
40 
41 #include "bloom.h"
42 
43 #include "got_error.h"
44 #include "got_reference.h"
45 #include "got_repository.h"
46 #include "got_path.h"
47 #include "got_cancel.h"
48 #include "got_object.h"
49 #include "got_opentemp.h"
50 
51 #include "got_lib_delta.h"
52 #include "got_lib_delta_cache.h"
53 #include "got_lib_inflate.h"
54 #include "got_lib_object.h"
55 #include "got_lib_object_parse.h"
56 #include "got_lib_object_create.h"
57 #include "got_lib_pack.h"
58 #include "got_lib_privsep.h"
59 #include "got_lib_hash.h"
60 #include "got_lib_object_cache.h"
61 #include "got_lib_repository.h"
62 #include "got_lib_gotconfig.h"
63 
64 #ifndef nitems
65 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
66 #endif
67 
68 #define GOT_PACK_NUM_TEMPFILES		GOT_PACK_CACHE_SIZE * 2
69 
70 RB_PROTOTYPE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
71     got_packidx_bloom_filter_cmp);
72 
73 static inline int
74 is_boolean_val(const char *val)
75 {
76 	return (strcasecmp(val, "true") == 0 ||
77 	    strcasecmp(val, "false") == 0 ||
78 	    strcasecmp(val, "on") == 0 ||
79     	    strcasecmp(val, "off") == 0 ||
80 	    strcasecmp(val, "yes") == 0 ||
81     	    strcasecmp(val, "no") == 0 ||
82 	    strcasecmp(val, "1") == 0 ||
83 	    strcasecmp(val, "0") == 0);
84 }
85 
86 static inline int
87 get_boolean_val(const char *val)
88 {
89 	return (strcasecmp(val, "true") == 0 ||
90 	    strcasecmp(val, "on") == 0 ||
91 	    strcasecmp(val, "yes") == 0 ||
92 	    strcasecmp(val, "1") == 0);
93 }
94 
95 const char *
96 got_repo_get_path(struct got_repository *repo)
97 {
98 	return repo->path;
99 }
100 
101 const char *
102 got_repo_get_path_git_dir(struct got_repository *repo)
103 {
104 	return repo->path_git_dir;
105 }
106 
107 int
108 got_repo_get_fd(struct got_repository *repo)
109 {
110 	return repo->gitdir_fd;
111 }
112 
113 const char *
114 got_repo_get_gitconfig_author_name(struct got_repository *repo)
115 {
116 	return repo->gitconfig_author_name;
117 }
118 
119 const char *
120 got_repo_get_gitconfig_author_email(struct got_repository *repo)
121 {
122 	return repo->gitconfig_author_email;
123 }
124 
125 const char *
126 got_repo_get_global_gitconfig_author_name(struct got_repository *repo)
127 {
128 	return repo->global_gitconfig_author_name;
129 }
130 
131 const char *
132 got_repo_get_global_gitconfig_author_email(struct got_repository *repo)
133 {
134 	return repo->global_gitconfig_author_email;
135 }
136 
137 const char *
138 got_repo_get_gitconfig_owner(struct got_repository *repo)
139 {
140 	return repo->gitconfig_owner;
141 }
142 
143 int
144 got_repo_has_extension(struct got_repository *repo, const char *ext)
145 {
146 	int i;
147 
148 	for (i = 0; i < repo->nextensions; ++i) {
149 		if (!strcasecmp(ext, repo->extnames[i]))
150 			return get_boolean_val(repo->extvals[i]);
151 	}
152 
153 	return 0;
154 }
155 
156 int
157 got_repo_is_bare(struct got_repository *repo)
158 {
159 	return (strcmp(repo->path, repo->path_git_dir) == 0);
160 }
161 
162 static char *
163 get_path_git_child(struct got_repository *repo, const char *basename)
164 {
165 	char *path_child;
166 
167 	if (asprintf(&path_child, "%s/%s", repo->path_git_dir,
168 	    basename) == -1)
169 		return NULL;
170 
171 	return path_child;
172 }
173 
174 char *
175 got_repo_get_path_objects(struct got_repository *repo)
176 {
177 	return get_path_git_child(repo, GOT_OBJECTS_DIR);
178 }
179 
180 char *
181 got_repo_get_path_objects_pack(struct got_repository *repo)
182 {
183 	return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR);
184 }
185 
186 char *
187 got_repo_get_path_refs(struct got_repository *repo)
188 {
189 	return get_path_git_child(repo, GOT_REFS_DIR);
190 }
191 
192 char *
193 got_repo_get_path_packed_refs(struct got_repository *repo)
194 {
195 	return get_path_git_child(repo, GOT_PACKED_REFS_FILE);
196 }
197 
198 static char *
199 get_path_head(struct got_repository *repo)
200 {
201 	return get_path_git_child(repo, GOT_HEAD_FILE);
202 }
203 
204 char *
205 got_repo_get_path_gitconfig(struct got_repository *repo)
206 {
207 	return get_path_git_child(repo, GOT_GITCONFIG);
208 }
209 
210 char *
211 got_repo_get_path_gotconfig(struct got_repository *repo)
212 {
213 	return get_path_git_child(repo, GOT_GOTCONFIG_FILENAME);
214 }
215 
216 const struct got_gotconfig *
217 got_repo_get_gotconfig(struct got_repository *repo)
218 {
219 	return repo->gotconfig;
220 }
221 
222 void
223 got_repo_get_gitconfig_remotes(int *nremotes,
224     const struct got_remote_repo **remotes, struct got_repository *repo)
225 {
226 	*nremotes = repo->ngitconfig_remotes;
227 	*remotes = repo->gitconfig_remotes;
228 }
229 
230 static int
231 is_git_repo(struct got_repository *repo)
232 {
233 	const char *path_git = got_repo_get_path_git_dir(repo);
234 	char *path_objects = got_repo_get_path_objects(repo);
235 	char *path_refs = got_repo_get_path_refs(repo);
236 	char *path_head = get_path_head(repo);
237 	int ret = 0;
238 	struct stat sb;
239 	struct got_reference *head_ref;
240 
241 	if (lstat(path_git, &sb) == -1)
242 		goto done;
243 	if (!S_ISDIR(sb.st_mode))
244 		goto done;
245 
246 	if (lstat(path_objects, &sb) == -1)
247 		goto done;
248 	if (!S_ISDIR(sb.st_mode))
249 		goto done;
250 
251 	if (lstat(path_refs, &sb) == -1)
252 		goto done;
253 	if (!S_ISDIR(sb.st_mode))
254 		goto done;
255 
256 	if (lstat(path_head, &sb) == -1)
257 		goto done;
258 	if (!S_ISREG(sb.st_mode))
259 		goto done;
260 
261 	/* Check if the HEAD reference can be opened. */
262 	if (got_ref_open(&head_ref, repo, GOT_REF_HEAD, 0) != NULL)
263 		goto done;
264 	got_ref_close(head_ref);
265 
266 	ret = 1;
267 done:
268 	free(path_objects);
269 	free(path_refs);
270 	free(path_head);
271 	return ret;
272 
273 }
274 
275 static const struct got_error *
276 close_tempfiles(int *fds, size_t nfds)
277 {
278 	const struct got_error *err = NULL;
279 	int i;
280 
281 	for (i = 0; i < nfds; i++) {
282 		if (fds[i] == -1)
283 			continue;
284 		if (close(fds[i]) == -1) {
285 			err = got_error_from_errno("close");
286 			break;
287 		}
288 	}
289 	free(fds);
290 	return err;
291 }
292 
293 static const struct got_error *
294 open_tempfiles(int **fds, size_t array_size, size_t nfds)
295 {
296 	const struct got_error *err = NULL;
297 	int i;
298 
299 	*fds = calloc(array_size, sizeof(**fds));
300 	if (*fds == NULL)
301 		return got_error_from_errno("calloc");
302 
303 	for (i = 0; i < array_size; i++)
304 		(*fds)[i] = -1;
305 
306 	for (i = 0; i < nfds; i++) {
307 		(*fds)[i] = got_opentempfd();
308 		if ((*fds)[i] == -1) {
309 			err = got_error_from_errno("got_opentempfd");
310 			close_tempfiles(*fds, nfds);
311 			*fds = NULL;
312 			return err;
313 		}
314 	}
315 
316 	return NULL;
317 }
318 
319 static const struct got_error *
320 get_pack_cache_size(int *pack_cache_size)
321 {
322 	struct rlimit rl;
323 
324 	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
325 		return got_error_from_errno("getrlimit");
326 
327 	*pack_cache_size = GOT_PACK_CACHE_SIZE;
328 	if (*pack_cache_size > rl.rlim_cur / 8)
329 		*pack_cache_size = rl.rlim_cur / 8;
330 
331 	return NULL;
332 }
333 
334 const struct got_error *
335 got_repo_pack_fds_open(int **pack_fds)
336 {
337 	const struct got_error *err;
338 	int nfds;
339 
340 	err = get_pack_cache_size(&nfds);
341 	if (err)
342 		return err;
343 
344 	/*
345 	 * We need one basefd and one accumfd per cached pack.
346 	 * Our constants should be set up in a way such that
347 	 * this error never triggers.
348 	 */
349 	if (nfds * 2 > GOT_PACK_NUM_TEMPFILES)
350 		return got_error(GOT_ERR_NO_SPACE);
351 
352 	return open_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES, nfds * 2);
353 }
354 
355 const struct got_error *
356 got_repo_pack_fds_close(int *pack_fds)
357 {
358 	return close_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES);
359 }
360 
361 const struct got_error *
362 got_repo_temp_fds_open(int **temp_fds)
363 {
364 	return open_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES,
365 	    GOT_REPO_NUM_TEMPFILES);
366 }
367 
368 void
369 got_repo_temp_fds_set(struct got_repository *repo, int *temp_fds)
370 {
371 	int i;
372 
373 	for (i = 0; i < GOT_REPO_NUM_TEMPFILES; i++)
374 		repo->tempfiles[i] = temp_fds[i];
375 }
376 
377 const struct got_error *
378 got_repo_temp_fds_get(int *fd, int *idx, struct got_repository *repo)
379 {
380 	int i;
381 
382 	*fd = -1;
383 	*idx = -1;
384 
385 	for (i = 0; i < nitems(repo->tempfiles); i++) {
386 		if (repo->tempfile_use_mask & (1 << i))
387 			continue;
388 		if (repo->tempfiles[i] != -1) {
389 			if (ftruncate(repo->tempfiles[i], 0L) == -1)
390 				return got_error_from_errno("ftruncate");
391 			*fd = repo->tempfiles[i];
392 			*idx = i;
393 			repo->tempfile_use_mask |= (1 << i);
394 			return NULL;
395 		}
396 	}
397 
398 	return got_error(GOT_ERR_REPO_TEMPFILE);
399 }
400 
401 void
402 got_repo_temp_fds_put(int idx, struct got_repository *repo)
403 {
404 	repo->tempfile_use_mask &= ~(1 << idx);
405 }
406 
407 const struct got_error *
408 got_repo_temp_fds_close(int *temp_fds)
409 {
410 	return close_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES);
411 }
412 
413 const struct got_error *
414 got_repo_cache_object(struct got_repository *repo, struct got_object_id *id,
415     struct got_object *obj)
416 {
417 #ifndef GOT_NO_OBJ_CACHE
418 	const struct got_error *err = NULL;
419 	err = got_object_cache_add(&repo->objcache, id, obj);
420 	if (err) {
421 		if (err->code == GOT_ERR_OBJ_EXISTS ||
422 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
423 			err = NULL;
424 		return err;
425 	}
426 	obj->refcnt++;
427 #endif
428 	return NULL;
429 }
430 
431 struct got_object *
432 got_repo_get_cached_object(struct got_repository *repo,
433     struct got_object_id *id)
434 {
435 	return (struct got_object *)got_object_cache_get(&repo->objcache, id);
436 }
437 
438 const struct got_error *
439 got_repo_cache_tree(struct got_repository *repo, struct got_object_id *id,
440     struct got_tree_object *tree)
441 {
442 #ifndef GOT_NO_OBJ_CACHE
443 	const struct got_error *err = NULL;
444 	err = got_object_cache_add(&repo->treecache, id, tree);
445 	if (err) {
446 		if (err->code == GOT_ERR_OBJ_EXISTS ||
447 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
448 			err = NULL;
449 		return err;
450 	}
451 	tree->refcnt++;
452 #endif
453 	return NULL;
454 }
455 
456 struct got_tree_object *
457 got_repo_get_cached_tree(struct got_repository *repo,
458     struct got_object_id *id)
459 {
460 	return (struct got_tree_object *)got_object_cache_get(
461 	    &repo->treecache, id);
462 }
463 
464 const struct got_error *
465 got_repo_cache_commit(struct got_repository *repo, struct got_object_id *id,
466     struct got_commit_object *commit)
467 {
468 #ifndef GOT_NO_OBJ_CACHE
469 	const struct got_error *err = NULL;
470 	err = got_object_cache_add(&repo->commitcache, id, commit);
471 	if (err) {
472 		if (err->code == GOT_ERR_OBJ_EXISTS ||
473 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
474 			err = NULL;
475 		return err;
476 	}
477 	commit->refcnt++;
478 #endif
479 	return NULL;
480 }
481 
482 struct got_commit_object *
483 got_repo_get_cached_commit(struct got_repository *repo,
484     struct got_object_id *id)
485 {
486 	return (struct got_commit_object *)got_object_cache_get(
487 	    &repo->commitcache, id);
488 }
489 
490 const struct got_error *
491 got_repo_cache_tag(struct got_repository *repo, struct got_object_id *id,
492     struct got_tag_object *tag)
493 {
494 #ifndef GOT_NO_OBJ_CACHE
495 	const struct got_error *err = NULL;
496 	err = got_object_cache_add(&repo->tagcache, id, tag);
497 	if (err) {
498 		if (err->code == GOT_ERR_OBJ_EXISTS ||
499 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
500 			err = NULL;
501 		return err;
502 	}
503 	tag->refcnt++;
504 #endif
505 	return NULL;
506 }
507 
508 struct got_tag_object *
509 got_repo_get_cached_tag(struct got_repository *repo, struct got_object_id *id)
510 {
511 	return (struct got_tag_object *)got_object_cache_get(
512 	    &repo->tagcache, id);
513 }
514 
515 const struct got_error *
516 got_repo_cache_raw_object(struct got_repository *repo, struct got_object_id *id,
517     struct got_raw_object *raw)
518 {
519 #ifndef GOT_NO_OBJ_CACHE
520 	const struct got_error *err = NULL;
521 	err = got_object_cache_add(&repo->rawcache, id, raw);
522 	if (err) {
523 		if (err->code == GOT_ERR_OBJ_EXISTS ||
524 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
525 			err = NULL;
526 		return err;
527 	}
528 	raw->refcnt++;
529 #endif
530 	return NULL;
531 }
532 
533 
534 struct got_raw_object *
535 got_repo_get_cached_raw_object(struct got_repository *repo,
536     struct got_object_id *id)
537 {
538 	return (struct got_raw_object *)got_object_cache_get(&repo->rawcache, id);
539 }
540 
541 
542 static const struct got_error *
543 open_repo(struct got_repository *repo, const char *path)
544 {
545 	const struct got_error *err = NULL;
546 
547 	repo->gitdir_fd = -1;
548 
549 	/* bare git repository? */
550 	repo->path_git_dir = strdup(path);
551 	if (repo->path_git_dir == NULL)
552 		return got_error_from_errno("strdup");
553 	if (is_git_repo(repo)) {
554 		repo->path = strdup(repo->path_git_dir);
555 		if (repo->path == NULL) {
556 			err = got_error_from_errno("strdup");
557 			goto done;
558 		}
559 		repo->gitdir_fd = open(repo->path_git_dir,
560 		    O_DIRECTORY | O_CLOEXEC);
561 		if (repo->gitdir_fd == -1) {
562 			err = got_error_from_errno2("open",
563 			    repo->path_git_dir);
564 			goto done;
565 		}
566 		return NULL;
567 	}
568 
569 	/* git repository with working tree? */
570 	free(repo->path_git_dir);
571 	repo->path_git_dir = NULL;
572 	if (asprintf(&repo->path_git_dir, "%s/%s", path, GOT_GIT_DIR) == -1) {
573 		err = got_error_from_errno("asprintf");
574 		goto done;
575 	}
576 	if (is_git_repo(repo)) {
577 		repo->path = strdup(path);
578 		if (repo->path == NULL) {
579 			err = got_error_from_errno("strdup");
580 			goto done;
581 		}
582 		repo->gitdir_fd = open(repo->path_git_dir,
583 		    O_DIRECTORY | O_CLOEXEC);
584 		if (repo->gitdir_fd == -1) {
585 			err = got_error_from_errno2("open",
586 			    repo->path_git_dir);
587 			goto done;
588 		}
589 		return NULL;
590 	}
591 
592 	err = got_error(GOT_ERR_NOT_GIT_REPO);
593 done:
594 	if (err) {
595 		free(repo->path);
596 		repo->path = NULL;
597 		free(repo->path_git_dir);
598 		repo->path_git_dir = NULL;
599 		if (repo->gitdir_fd != -1)
600 			close(repo->gitdir_fd);
601 		repo->gitdir_fd = -1;
602 
603 	}
604 	return err;
605 }
606 
607 static const struct got_error *
608 read_gitconfig(struct got_repository *repo, const char *global_gitconfig_path)
609 {
610 	const struct got_error *err = NULL;
611 	char *repo_gitconfig_path = NULL;
612 
613 	if (global_gitconfig_path) {
614 		/* Read settings from ~/.gitconfig. */
615 		int dummy_repo_version;
616 		err = got_repo_read_gitconfig(&dummy_repo_version,
617 		    &repo->global_gitconfig_author_name,
618 		    &repo->global_gitconfig_author_email,
619 		    NULL, NULL, NULL, NULL, NULL, NULL,
620 		    global_gitconfig_path);
621 		if (err)
622 			return err;
623 	}
624 
625 	/* Read repository's .git/config file. */
626 	repo_gitconfig_path = got_repo_get_path_gitconfig(repo);
627 	if (repo_gitconfig_path == NULL)
628 		return got_error_from_errno("got_repo_get_path_gitconfig");
629 
630 	err = got_repo_read_gitconfig(
631 	    &repo->gitconfig_repository_format_version,
632 	    &repo->gitconfig_author_name, &repo->gitconfig_author_email,
633 	    &repo->gitconfig_remotes, &repo->ngitconfig_remotes,
634 	    &repo->gitconfig_owner, &repo->extnames, &repo->extvals,
635 	    &repo->nextensions, repo_gitconfig_path);
636 	if (err)
637 		goto done;
638 
639 	if (getenv("GOT_IGNORE_GITCONFIG") != NULL) {
640 		int i;
641 
642 		for (i = 0; i < repo->ngitconfig_remotes; i++) {
643 			got_repo_free_remote_repo_data(
644 			    &repo->gitconfig_remotes[i]);
645 		}
646 		free(repo->gitconfig_remotes);
647 		repo->gitconfig_remotes = NULL;
648 		repo->ngitconfig_remotes = 0;
649 
650 		free(repo->gitconfig_author_name);
651 		repo->gitconfig_author_name = NULL;
652 		free(repo->gitconfig_author_email);
653 		repo->gitconfig_author_email = NULL;
654 
655 		free(repo->global_gitconfig_author_name);
656 		repo->global_gitconfig_author_name = NULL;
657 		free(repo->global_gitconfig_author_email);
658 		repo->global_gitconfig_author_email = NULL;
659 	}
660 
661 done:
662 	free(repo_gitconfig_path);
663 	return err;
664 }
665 
666 static const struct got_error *
667 read_gotconfig(struct got_repository *repo)
668 {
669 	const struct got_error *err = NULL;
670 	char *gotconfig_path;
671 
672 	gotconfig_path = got_repo_get_path_gotconfig(repo);
673 	if (gotconfig_path == NULL)
674 		return got_error_from_errno("got_repo_get_path_gotconfig");
675 
676 	err = got_gotconfig_read(&repo->gotconfig, gotconfig_path);
677 	free(gotconfig_path);
678 	return err;
679 }
680 
681 /* Supported repository format extensions. */
682 static const char *const repo_extensions[] = {
683 	"noop",			/* Got supports repository format version 1. */
684 	"preciousObjects",	/* Supported by gotadmin cleanup. */
685 	"worktreeConfig",	/* Got does not care about Git work trees. */
686 };
687 
688 const struct got_error *
689 got_repo_open(struct got_repository **repop, const char *path,
690     const char *global_gitconfig_path, int *pack_fds)
691 {
692 	struct got_repository *repo = NULL;
693 	const struct got_error *err = NULL;
694 	char *repo_path = NULL;
695 	size_t i, j = 0;
696 
697 	*repop = NULL;
698 
699 	repo = calloc(1, sizeof(*repo));
700 	if (repo == NULL)
701 		return got_error_from_errno("calloc");
702 
703 	RB_INIT(&repo->packidx_bloom_filters);
704 	TAILQ_INIT(&repo->packidx_paths);
705 
706 	for (i = 0; i < nitems(repo->privsep_children); i++) {
707 		memset(&repo->privsep_children[i], 0,
708 		    sizeof(repo->privsep_children[0]));
709 		repo->privsep_children[i].imsg_fd = -1;
710 	}
711 
712 	err = got_object_cache_init(&repo->objcache,
713 	    GOT_OBJECT_CACHE_TYPE_OBJ);
714 	if (err)
715 		goto done;
716 	err = got_object_cache_init(&repo->treecache,
717 	    GOT_OBJECT_CACHE_TYPE_TREE);
718 	if (err)
719 		goto done;
720 	err = got_object_cache_init(&repo->commitcache,
721 	    GOT_OBJECT_CACHE_TYPE_COMMIT);
722 	if (err)
723 		goto done;
724 	err = got_object_cache_init(&repo->tagcache,
725 	    GOT_OBJECT_CACHE_TYPE_TAG);
726 	if (err)
727 		goto done;
728 	err = got_object_cache_init(&repo->rawcache,
729 	    GOT_OBJECT_CACHE_TYPE_RAW);
730 	if (err)
731 		goto done;
732 
733 	err = get_pack_cache_size(&repo->pack_cache_size);
734 	if (err)
735 		goto done;
736 	for (i = 0; i < nitems(repo->packs); i++) {
737 		if (pack_fds != NULL && i < repo->pack_cache_size) {
738 			repo->packs[i].basefd = pack_fds[j++];
739 			repo->packs[i].accumfd = pack_fds[j++];
740 		} else {
741 			repo->packs[i].basefd = -1;
742 			repo->packs[i].accumfd = -1;
743 		}
744 	}
745 	for (i = 0; i < nitems(repo->tempfiles); i++)
746 		repo->tempfiles[i] = -1;
747 	repo->pinned_pack = -1;
748 	repo->pinned_packidx = -1;
749 	repo->pinned_pid = 0;
750 
751 	repo_path = realpath(path, NULL);
752 	if (repo_path == NULL) {
753 		err = got_error_from_errno2("realpath", path);
754 		goto done;
755 	}
756 
757 	for (;;) {
758 		char *parent_path;
759 
760 		err = open_repo(repo, repo_path);
761 		if (err == NULL)
762 			break;
763 		if (err->code != GOT_ERR_NOT_GIT_REPO)
764 			goto done;
765 		if (repo_path[0] == '/' && repo_path[1] == '\0') {
766 			err = got_error(GOT_ERR_NOT_GIT_REPO);
767 			goto done;
768 		}
769 		err = got_path_dirname(&parent_path, repo_path);
770 		if (err)
771 			goto done;
772 		free(repo_path);
773 		repo_path = parent_path;
774 	}
775 
776 	err = read_gotconfig(repo);
777 	if (err)
778 		goto done;
779 
780 	err = read_gitconfig(repo, global_gitconfig_path);
781 	if (err)
782 		goto done;
783 	if (repo->gitconfig_repository_format_version != 0) {
784 		err = got_error_path(path, GOT_ERR_GIT_REPO_FORMAT);
785 		goto done;
786 	}
787 	for (i = 0; i < repo->nextensions; i++) {
788 		char *ext = repo->extnames[i];
789 		char *val = repo->extvals[i];
790 		int j, supported = 0;
791 
792 		if (!is_boolean_val(val)) {
793 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
794 			goto done;
795 		}
796 
797 		if (!get_boolean_val(val))
798 			continue;
799 
800 		for (j = 0; j < nitems(repo_extensions); j++) {
801 			if (strcmp(ext, repo_extensions[j]) == 0) {
802 				supported = 1;
803 				break;
804 			}
805 		}
806 		if (!supported) {
807 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
808 			goto done;
809 		}
810 	}
811 
812 	err = got_repo_list_packidx(&repo->packidx_paths, repo);
813 done:
814 	if (err)
815 		got_repo_close(repo);
816 	else
817 		*repop = repo;
818 	free(repo_path);
819 	return err;
820 }
821 
822 const struct got_error *
823 got_repo_close(struct got_repository *repo)
824 {
825 	const struct got_error *err = NULL, *child_err;
826 	struct got_packidx_bloom_filter *bf;
827 	size_t i;
828 
829 	for (i = 0; i < repo->pack_cache_size; i++) {
830 		if (repo->packidx_cache[i] == NULL)
831 			break;
832 		got_packidx_close(repo->packidx_cache[i]);
833 	}
834 
835 	while ((bf = RB_MIN(got_packidx_bloom_filter_tree,
836 	    &repo->packidx_bloom_filters))) {
837 		RB_REMOVE(got_packidx_bloom_filter_tree,
838 		    &repo->packidx_bloom_filters, bf);
839 		bloom_free(bf->bloom);
840 		free(bf->bloom);
841 		free(bf);
842 	}
843 
844 	for (i = 0; i < repo->pack_cache_size; i++)
845 		if (repo->packs[i].path_packfile)
846 			if (repo->packs[i].path_packfile)
847 				got_pack_close(&repo->packs[i]);
848 
849 	free(repo->path);
850 	free(repo->path_git_dir);
851 
852 	got_object_cache_close(&repo->objcache);
853 	got_object_cache_close(&repo->treecache);
854 	got_object_cache_close(&repo->commitcache);
855 	got_object_cache_close(&repo->tagcache);
856 	got_object_cache_close(&repo->rawcache);
857 
858 	for (i = 0; i < nitems(repo->privsep_children); i++) {
859 		if (repo->privsep_children[i].imsg_fd == -1)
860 			continue;
861 		imsg_clear(repo->privsep_children[i].ibuf);
862 		free(repo->privsep_children[i].ibuf);
863 		err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd);
864 		child_err = got_privsep_wait_for_child(
865 		    repo->privsep_children[i].pid);
866 		if (child_err && err == NULL)
867 			err = child_err;
868 		if (close(repo->privsep_children[i].imsg_fd) == -1 &&
869 		    err == NULL)
870 			err = got_error_from_errno("close");
871 	}
872 
873 	if (repo->gitdir_fd != -1 && close(repo->gitdir_fd) == -1 &&
874 	    err == NULL)
875 		err = got_error_from_errno("close");
876 
877 	if (repo->gotconfig)
878 		got_gotconfig_free(repo->gotconfig);
879 	free(repo->gitconfig_author_name);
880 	free(repo->gitconfig_author_email);
881 	for (i = 0; i < repo->ngitconfig_remotes; i++)
882 		got_repo_free_remote_repo_data(&repo->gitconfig_remotes[i]);
883 	free(repo->gitconfig_remotes);
884 	for (i = 0; i < repo->nextensions; i++) {
885 		free(repo->extnames[i]);
886 		free(repo->extvals[i]);
887 	}
888 	free(repo->extnames);
889 	free(repo->extvals);
890 
891 	got_pathlist_free(&repo->packidx_paths, GOT_PATHLIST_FREE_PATH);
892 	free(repo);
893 
894 	return err;
895 }
896 
897 void
898 got_repo_free_remote_repo_data(struct got_remote_repo *repo)
899 {
900 	int i;
901 
902 	free(repo->name);
903 	repo->name = NULL;
904 	free(repo->fetch_url);
905 	repo->fetch_url = NULL;
906 	free(repo->send_url);
907 	repo->send_url = NULL;
908 	for (i = 0; i < repo->nfetch_branches; i++)
909 		free(repo->fetch_branches[i]);
910 	free(repo->fetch_branches);
911 	repo->fetch_branches = NULL;
912 	repo->nfetch_branches = 0;
913 	for (i = 0; i < repo->nsend_branches; i++)
914 		free(repo->send_branches[i]);
915 	free(repo->send_branches);
916 	repo->send_branches = NULL;
917 	repo->nsend_branches = 0;
918 }
919 
920 const struct got_error *
921 got_repo_map_path(char **in_repo_path, struct got_repository *repo,
922     const char *input_path)
923 {
924 	const struct got_error *err = NULL;
925 	const char *repo_abspath = NULL;
926 	size_t repolen, len;
927 	char *canonpath, *path = NULL;
928 
929 	*in_repo_path = NULL;
930 
931 	canonpath = strdup(input_path);
932 	if (canonpath == NULL) {
933 		err = got_error_from_errno("strdup");
934 		goto done;
935 	}
936 	err = got_canonpath(input_path, canonpath, strlen(canonpath) + 1);
937 	if (err)
938 		goto done;
939 
940 	repo_abspath = got_repo_get_path(repo);
941 
942 	if (canonpath[0] == '\0') {
943 		path = strdup(canonpath);
944 		if (path == NULL) {
945 			err = got_error_from_errno("strdup");
946 			goto done;
947 		}
948 	} else {
949 		path = realpath(canonpath, NULL);
950 		if (path == NULL) {
951 			if (errno != ENOENT) {
952 				err = got_error_from_errno2("realpath",
953 				    canonpath);
954 				goto done;
955 			}
956 			/*
957 			 * Path is not on disk.
958 			 * Assume it is already relative to repository root.
959 			 */
960 			path = strdup(canonpath);
961 			if (path == NULL) {
962 				err = got_error_from_errno("strdup");
963 				goto done;
964 			}
965 		}
966 
967 		repolen = strlen(repo_abspath);
968 		len = strlen(path);
969 
970 
971 		if (strcmp(path, repo_abspath) == 0) {
972 			free(path);
973 			path = strdup("");
974 			if (path == NULL) {
975 				err = got_error_from_errno("strdup");
976 				goto done;
977 			}
978 		} else if (len > repolen &&
979 		    got_path_is_child(path, repo_abspath, repolen)) {
980 			/* Matched an on-disk path inside repository. */
981 			if (got_repo_is_bare(repo)) {
982 				/*
983 				 * Matched an on-disk path inside repository
984 				 * database. Treat input as repository-relative.
985 				 */
986 				free(path);
987 				path = canonpath;
988 				canonpath = NULL;
989 			} else {
990 				char *child;
991 				/* Strip common prefix with repository path. */
992 				err = got_path_skip_common_ancestor(&child,
993 				    repo_abspath, path);
994 				if (err)
995 					goto done;
996 				free(path);
997 				path = child;
998 			}
999 		} else {
1000 			/*
1001 			 * Matched unrelated on-disk path.
1002 			 * Treat input as repository-relative.
1003 			 */
1004 			free(path);
1005 			path = canonpath;
1006 			canonpath = NULL;
1007 		}
1008 	}
1009 
1010 	/* Make in-repository path absolute */
1011 	if (path[0] != '/') {
1012 		char *abspath;
1013 		if (asprintf(&abspath, "/%s", path) == -1) {
1014 			err = got_error_from_errno("asprintf");
1015 			goto done;
1016 		}
1017 		free(path);
1018 		path = abspath;
1019 	}
1020 
1021 done:
1022 	free(canonpath);
1023 	if (err)
1024 		free(path);
1025 	else
1026 		*in_repo_path = path;
1027 	return err;
1028 }
1029 
1030 static const struct got_error *
1031 cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
1032     const char *path_packidx)
1033 {
1034 	const struct got_error *err = NULL;
1035 	size_t i;
1036 
1037 	for (i = 0; i < repo->pack_cache_size; i++) {
1038 		if (repo->packidx_cache[i] == NULL)
1039 			break;
1040 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1041 		    path_packidx) == 0) {
1042 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1043 		}
1044 	}
1045 	if (i == repo->pack_cache_size) {
1046 		do {
1047 			i--;
1048 		} while (i > 0 && repo->pinned_packidx >= 0 &&
1049 		    i == repo->pinned_packidx);
1050 		err = got_packidx_close(repo->packidx_cache[i]);
1051 		if (err)
1052 			return err;
1053 	}
1054 
1055 	repo->packidx_cache[i] = packidx;
1056 
1057 	return NULL;
1058 }
1059 
1060 int
1061 got_repo_is_packidx_filename(const char *name, size_t len)
1062 {
1063 	if (len != GOT_PACKIDX_NAMELEN)
1064 		return 0;
1065 
1066 	if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
1067 		return 0;
1068 
1069 	if (strcmp(name + strlen(GOT_PACK_PREFIX) +
1070 	    SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
1071 		return 0;
1072 
1073 	return 1;
1074 }
1075 
1076 static struct got_packidx_bloom_filter *
1077 get_packidx_bloom_filter(struct got_repository *repo,
1078     const char *path, size_t path_len)
1079 {
1080 	struct got_packidx_bloom_filter key;
1081 
1082 	if (strlcpy(key.path, path, sizeof(key.path)) >= sizeof(key.path))
1083 		return NULL; /* XXX */
1084 	key.path_len = path_len;
1085 
1086 	return RB_FIND(got_packidx_bloom_filter_tree,
1087 	    &repo->packidx_bloom_filters, &key);
1088 }
1089 
1090 int
1091 got_repo_check_packidx_bloom_filter(struct got_repository *repo,
1092     const char *path_packidx, struct got_object_id *id)
1093 {
1094 	struct got_packidx_bloom_filter *bf;
1095 
1096 	bf = get_packidx_bloom_filter(repo, path_packidx, strlen(path_packidx));
1097 	if (bf)
1098 		return bloom_check(bf->bloom, id->sha1, sizeof(id->sha1));
1099 
1100 	/* No bloom filter means this pack index must be searched. */
1101 	return 1;
1102 }
1103 
1104 static const struct got_error *
1105 add_packidx_bloom_filter(struct got_repository *repo,
1106     struct got_packidx *packidx, const char *path_packidx)
1107 {
1108 	int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1109 	struct got_packidx_bloom_filter *bf;
1110 	size_t len;
1111 
1112 	/*
1113 	 * Don't use bloom filters for very large pack index files.
1114 	 * Large pack files will contain a relatively large fraction
1115 	 * of our objects so we will likely need to visit them anyway.
1116 	 * The more objects a pack file contains the higher the probability
1117 	 * of a false-positive match from the bloom filter. And reading
1118 	 * all object IDs from a large pack index file can be expensive.
1119 	 */
1120 	if (nobjects > 100000) /* cut-off at about 2MB, at 20 bytes per ID */
1121 		return NULL;
1122 
1123 	/* Do we already have a filter for this pack index? */
1124 	if (get_packidx_bloom_filter(repo, path_packidx,
1125 	    strlen(path_packidx)) != NULL)
1126 		return NULL;
1127 
1128 	bf = calloc(1, sizeof(*bf));
1129 	if (bf == NULL)
1130 		return got_error_from_errno("calloc");
1131 	bf->bloom = calloc(1, sizeof(*bf->bloom));
1132 	if (bf->bloom == NULL) {
1133 		free(bf);
1134 		return got_error_from_errno("calloc");
1135 	}
1136 
1137 	len = strlcpy(bf->path, path_packidx, sizeof(bf->path));
1138 	if (len >= sizeof(bf->path)) {
1139 		free(bf->bloom);
1140 		free(bf);
1141 		return got_error(GOT_ERR_NO_SPACE);
1142 	}
1143 	bf->path_len = len;
1144 
1145 	/* Minimum size supported by our bloom filter is 1000 entries. */
1146 	bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
1147 	for (i = 0; i < nobjects; i++) {
1148 		struct got_packidx_object_id *id;
1149 		id = &packidx->hdr.sorted_ids[i];
1150 		bloom_add(bf->bloom, id->sha1, sizeof(id->sha1));
1151 	}
1152 
1153 	RB_INSERT(got_packidx_bloom_filter_tree,
1154 	    &repo->packidx_bloom_filters, bf);
1155 	return NULL;
1156 }
1157 
1158 static void
1159 purge_packidx_paths(struct got_pathlist_head *packidx_paths)
1160 {
1161 	struct got_pathlist_entry *pe;
1162 
1163 	while (!TAILQ_EMPTY(packidx_paths)) {
1164 		pe = TAILQ_FIRST(packidx_paths);
1165 		TAILQ_REMOVE(packidx_paths, pe, entry);
1166 		free((char *)pe->path);
1167 		free(pe);
1168 	}
1169 }
1170 
1171 static const struct got_error *
1172 refresh_packidx_paths(struct got_repository *repo)
1173 {
1174 	const struct got_error *err = NULL;
1175 	char *objects_pack_dir = NULL;
1176 	struct stat sb;
1177 
1178 	objects_pack_dir = got_repo_get_path_objects_pack(repo);
1179 	if (objects_pack_dir == NULL)
1180 		return got_error_from_errno("got_repo_get_path_objects_pack");
1181 
1182 	if (stat(objects_pack_dir, &sb) == -1) {
1183 		if (errno != ENOENT) {
1184 			err = got_error_from_errno2("stat", objects_pack_dir);
1185 			goto done;
1186 		}
1187 	} else if (TAILQ_EMPTY(&repo->packidx_paths) ||
1188 	    sb.st_mtim.tv_sec != repo->pack_path_mtime.tv_sec ||
1189 	    sb.st_mtim.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1190 		purge_packidx_paths(&repo->packidx_paths);
1191 		err = got_repo_list_packidx(&repo->packidx_paths, repo);
1192 		if (err)
1193 			goto done;
1194 	}
1195 done:
1196 	free(objects_pack_dir);
1197 	return err;
1198 }
1199 
1200 const struct got_error *
1201 got_repo_search_packidx(struct got_packidx **packidx, int *idx,
1202     struct got_repository *repo, struct got_object_id *id)
1203 {
1204 	const struct got_error *err;
1205 	struct got_pathlist_entry *pe;
1206 	size_t i;
1207 
1208 	/* Search pack index cache. */
1209 	for (i = 0; i < repo->pack_cache_size; i++) {
1210 		if (repo->packidx_cache[i] == NULL)
1211 			break;
1212 		if (!got_repo_check_packidx_bloom_filter(repo,
1213 		    repo->packidx_cache[i]->path_packidx, id))
1214 			continue; /* object will not be found in this index */
1215 		*idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
1216 		if (*idx != -1) {
1217 			*packidx = repo->packidx_cache[i];
1218 			/*
1219 			 * Move this cache entry to the front. Repeatedly
1220 			 * searching a wrong pack index can be expensive.
1221 			 */
1222 			if (i > 0) {
1223 				memmove(&repo->packidx_cache[1],
1224 				    &repo->packidx_cache[0],
1225 				    i * sizeof(repo->packidx_cache[0]));
1226 				repo->packidx_cache[0] = *packidx;
1227 				if (repo->pinned_packidx >= 0 &&
1228 				    repo->pinned_packidx < i)
1229 					repo->pinned_packidx++;
1230 				else if (repo->pinned_packidx == i)
1231 					repo->pinned_packidx = 0;
1232 			}
1233 			return NULL;
1234 		}
1235 	}
1236 	/* No luck. Search the filesystem. */
1237 
1238 	err = refresh_packidx_paths(repo);
1239 	if (err)
1240 		return err;
1241 
1242 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1243 		const char *path_packidx = pe->path;
1244 		int is_cached = 0;
1245 
1246 		if (!got_repo_check_packidx_bloom_filter(repo,
1247 		    pe->path, id))
1248 			continue; /* object will not be found in this index */
1249 
1250 		for (i = 0; i < repo->pack_cache_size; i++) {
1251 			if (repo->packidx_cache[i] == NULL)
1252 				break;
1253 			if (strcmp(repo->packidx_cache[i]->path_packidx,
1254 			    path_packidx) == 0) {
1255 				is_cached = 1;
1256 				break;
1257 			}
1258 		}
1259 		if (is_cached)
1260 			continue; /* already searched */
1261 
1262 		err = got_packidx_open(packidx, got_repo_get_fd(repo),
1263 		    path_packidx, 0);
1264 		if (err)
1265 			goto done;
1266 
1267 		err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1268 		if (err)
1269 			goto done;
1270 
1271 		err = cache_packidx(repo, *packidx, path_packidx);
1272 		if (err)
1273 			goto done;
1274 
1275 		*idx = got_packidx_get_object_idx(*packidx, id);
1276 		if (*idx != -1) {
1277 			err = NULL; /* found the object */
1278 			goto done;
1279 		}
1280 	}
1281 
1282 	err = got_error_no_obj(id);
1283 done:
1284 	return err;
1285 }
1286 
1287 const struct got_error *
1288 got_repo_list_packidx(struct got_pathlist_head *packidx_paths,
1289     struct got_repository *repo)
1290 {
1291 	const struct got_error *err = NULL;
1292 	DIR *packdir = NULL;
1293 	struct dirent *dent;
1294 	char *path_packidx = NULL;
1295 	int packdir_fd;
1296 	struct stat sb;
1297 
1298 	packdir_fd = openat(got_repo_get_fd(repo),
1299 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1300 	if (packdir_fd == -1) {
1301 		return got_error_from_errno_fmt("openat: %s/%s",
1302 		    got_repo_get_path_git_dir(repo),
1303 		    GOT_OBJECTS_PACK_DIR);
1304 	}
1305 
1306 	packdir = fdopendir(packdir_fd);
1307 	if (packdir == NULL) {
1308 		err = got_error_from_errno("fdopendir");
1309 		goto done;
1310 	}
1311 
1312 	if (fstat(packdir_fd, &sb) == -1) {
1313 		err = got_error_from_errno("fstat");
1314 		goto done;
1315 	}
1316 	repo->pack_path_mtime.tv_sec = sb.st_mtim.tv_sec;
1317 	repo->pack_path_mtime.tv_nsec = sb.st_mtim.tv_nsec;
1318 
1319 	while ((dent = readdir(packdir)) != NULL) {
1320 		if (!got_repo_is_packidx_filename(dent->d_name,
1321 		    strlen(dent->d_name)))
1322 			continue;
1323 
1324 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
1325 		    dent->d_name) == -1) {
1326 			err = got_error_from_errno("asprintf");
1327 			path_packidx = NULL;
1328 			break;
1329 		}
1330 
1331 		err = got_pathlist_append(packidx_paths, path_packidx, NULL);
1332 		if (err)
1333 			break;
1334 	}
1335 done:
1336 	if (err)
1337 		free(path_packidx);
1338 	if (packdir && closedir(packdir) != 0 && err == NULL)
1339 		err = got_error_from_errno("closedir");
1340 	return err;
1341 }
1342 
1343 const struct got_error *
1344 got_repo_get_packidx(struct got_packidx **packidx, const char *path_packidx,
1345     struct got_repository *repo)
1346 {
1347 	const struct got_error *err;
1348 	size_t i;
1349 
1350 	*packidx = NULL;
1351 
1352 	/* Search pack index cache. */
1353 	for (i = 0; i < repo->pack_cache_size; i++) {
1354 		if (repo->packidx_cache[i] == NULL)
1355 			break;
1356 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1357 		    path_packidx) == 0) {
1358 			*packidx = repo->packidx_cache[i];
1359 			return NULL;
1360 		}
1361 	}
1362 	/* No luck. Search the filesystem. */
1363 
1364 	err = got_packidx_open(packidx, got_repo_get_fd(repo),
1365 	    path_packidx, 0);
1366 	if (err)
1367 		return err;
1368 
1369 	err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1370 	if (err)
1371 		goto done;
1372 
1373 	err = cache_packidx(repo, *packidx, path_packidx);
1374 done:
1375 	if (err) {
1376 		got_packidx_close(*packidx);
1377 		*packidx = NULL;
1378 	}
1379 	return err;
1380 }
1381 
1382 static const struct got_error *
1383 read_packfile_hdr(int fd, struct got_packidx *packidx)
1384 {
1385 	const struct got_error *err = NULL;
1386 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
1387 	struct got_packfile_hdr hdr;
1388 	ssize_t n;
1389 
1390 	n = read(fd, &hdr, sizeof(hdr));
1391 	if (n < 0)
1392 		return got_error_from_errno("read");
1393 	if (n != sizeof(hdr))
1394 		return got_error(GOT_ERR_BAD_PACKFILE);
1395 
1396 	if (be32toh(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
1397 	    be32toh(hdr.version) != GOT_PACKFILE_VERSION ||
1398 	    be32toh(hdr.nobjects) != totobj)
1399 		err = got_error(GOT_ERR_BAD_PACKFILE);
1400 
1401 	return err;
1402 }
1403 
1404 static const struct got_error *
1405 open_packfile(int *fd, struct got_repository *repo,
1406     const char *relpath, struct got_packidx *packidx)
1407 {
1408 	const struct got_error *err = NULL;
1409 
1410 	*fd = openat(got_repo_get_fd(repo), relpath,
1411 	    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1412 	if (*fd == -1)
1413 		return got_error_from_errno_fmt("openat: %s/%s",
1414 		    got_repo_get_path_git_dir(repo), relpath);
1415 
1416 	if (packidx) {
1417 		err = read_packfile_hdr(*fd, packidx);
1418 		if (err) {
1419 			close(*fd);
1420 			*fd = -1;
1421 		}
1422 	}
1423 
1424 	return err;
1425 }
1426 
1427 const struct got_error *
1428 got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
1429     const char *path_packfile, struct got_packidx *packidx)
1430 {
1431 	const struct got_error *err = NULL;
1432 	struct got_pack *pack = NULL;
1433 	struct stat sb;
1434 	size_t i;
1435 
1436 	if (packp)
1437 		*packp = NULL;
1438 
1439 	for (i = 0; i < repo->pack_cache_size; i++) {
1440 		pack = &repo->packs[i];
1441 		if (pack->path_packfile == NULL)
1442 			break;
1443 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1444 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1445 	}
1446 
1447 	if (i == repo->pack_cache_size) {
1448 		struct got_pack tmp;
1449 		do {
1450 			i--;
1451 		} while (i > 0 && repo->pinned_pack >= 0 &&
1452 		    i == repo->pinned_pack);
1453 		err = got_pack_close(&repo->packs[i]);
1454 		if (err)
1455 			return err;
1456 		if (ftruncate(repo->packs[i].basefd, 0L) == -1)
1457 			return got_error_from_errno("ftruncate");
1458 		if (ftruncate(repo->packs[i].accumfd, 0L) == -1)
1459 			return got_error_from_errno("ftruncate");
1460 		memcpy(&tmp, &repo->packs[i], sizeof(tmp));
1461 		memcpy(&repo->packs[i], &repo->packs[0],
1462 		    sizeof(repo->packs[i]));
1463 		memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0]));
1464 		if (repo->pinned_pack == 0)
1465 			repo->pinned_pack = i;
1466 		else if (repo->pinned_pack == i)
1467 			repo->pinned_pack = 0;
1468 		i = 0;
1469 	}
1470 
1471 	pack = &repo->packs[i];
1472 
1473 	pack->path_packfile = strdup(path_packfile);
1474 	if (pack->path_packfile == NULL) {
1475 		err = got_error_from_errno("strdup");
1476 		goto done;
1477 	}
1478 
1479 	err = open_packfile(&pack->fd, repo, path_packfile, packidx);
1480 	if (err)
1481 		goto done;
1482 
1483 	if (fstat(pack->fd, &sb) != 0) {
1484 		err = got_error_from_errno("fstat");
1485 		goto done;
1486 	}
1487 	pack->filesize = sb.st_size;
1488 
1489 	pack->privsep_child = NULL;
1490 
1491 	err = got_delta_cache_alloc(&pack->delta_cache);
1492 	if (err)
1493 		goto done;
1494 
1495 #ifndef GOT_PACK_NO_MMAP
1496 	if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1497 		pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1498 		    pack->fd, 0);
1499 		if (pack->map == MAP_FAILED) {
1500 			if (errno != ENOMEM) {
1501 				err = got_error_from_errno("mmap");
1502 				goto done;
1503 			}
1504 			pack->map = NULL; /* fall back to read(2) */
1505 		}
1506 	}
1507 #endif
1508 done:
1509 	if (err) {
1510 		if (pack)
1511 			got_pack_close(pack);
1512 	} else if (packp)
1513 		*packp = pack;
1514 	return err;
1515 }
1516 
1517 struct got_pack *
1518 got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
1519 {
1520 	struct got_pack *pack = NULL;
1521 	size_t i;
1522 
1523 	for (i = 0; i < repo->pack_cache_size; i++) {
1524 		pack = &repo->packs[i];
1525 		if (pack->path_packfile == NULL)
1526 			break;
1527 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1528 			return pack;
1529 	}
1530 
1531 	return NULL;
1532 }
1533 
1534 const struct got_error *
1535 got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx,
1536     struct got_pack *pack)
1537 {
1538 	size_t i;
1539 	int pinned_pack = -1, pinned_packidx = -1;
1540 
1541 	for (i = 0; i < repo->pack_cache_size; i++) {
1542 		if (repo->packidx_cache[i] &&
1543 		    strcmp(repo->packidx_cache[i]->path_packidx,
1544 		    packidx->path_packidx) == 0)
1545 			pinned_packidx = i;
1546 		if (repo->packs[i].path_packfile &&
1547 		    strcmp(repo->packs[i].path_packfile,
1548 		    pack->path_packfile) == 0)
1549 			pinned_pack = i;
1550 	}
1551 
1552 	if (pinned_packidx == -1 || pinned_pack == -1)
1553 		return got_error(GOT_ERR_PIN_PACK);
1554 
1555 	repo->pinned_pack = pinned_pack;
1556 	repo->pinned_packidx = pinned_packidx;
1557 	if (repo->packs[pinned_pack].privsep_child)
1558 		repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid;
1559 	return NULL;
1560 }
1561 
1562 struct got_pack *
1563 got_repo_get_pinned_pack(struct got_repository *repo)
1564 {
1565 	if (repo->pinned_pack >= 0 &&
1566 	    repo->pinned_pack < repo->pack_cache_size)
1567 		return &repo->packs[repo->pinned_pack];
1568 
1569 	return NULL;
1570 }
1571 
1572 void
1573 got_repo_unpin_pack(struct got_repository *repo)
1574 {
1575 	repo->pinned_packidx = -1;
1576 	repo->pinned_pack = -1;
1577 	repo->pinned_pid = 0;
1578 }
1579 
1580 const struct got_error *
1581 got_repo_init(const char *repo_path, const char *head_name)
1582 {
1583 	const struct got_error *err = NULL;
1584 	const char *dirnames[] = {
1585 		GOT_OBJECTS_DIR,
1586 		GOT_OBJECTS_PACK_DIR,
1587 		GOT_REFS_DIR,
1588 	};
1589 	const char *description_str = "Unnamed repository; "
1590 	    "edit this file 'description' to name the repository.";
1591 	const char *headref = "ref: refs/heads/";
1592 	const char *gitconfig_str = "[core]\n"
1593 	    "\trepositoryformatversion = 0\n"
1594 	    "\tfilemode = true\n"
1595 	    "\tbare = true\n";
1596 	char *headref_str, *path;
1597 	size_t i;
1598 
1599 	if (!got_path_dir_is_empty(repo_path))
1600 		return got_error(GOT_ERR_DIR_NOT_EMPTY);
1601 
1602 	for (i = 0; i < nitems(dirnames); i++) {
1603 		if (asprintf(&path, "%s/%s", repo_path, dirnames[i]) == -1) {
1604 			return got_error_from_errno("asprintf");
1605 		}
1606 		err = got_path_mkdir(path);
1607 		free(path);
1608 		if (err)
1609 			return err;
1610 	}
1611 
1612 	if (asprintf(&path, "%s/%s", repo_path, "description") == -1)
1613 		return got_error_from_errno("asprintf");
1614 	err = got_path_create_file(path, description_str);
1615 	free(path);
1616 	if (err)
1617 		return err;
1618 
1619 	if (asprintf(&path, "%s/%s", repo_path, GOT_HEAD_FILE) == -1)
1620 		return got_error_from_errno("asprintf");
1621 	if (asprintf(&headref_str, "%s%s", headref,
1622 	    head_name ? head_name : "main") == -1) {
1623 		free(path);
1624 		return got_error_from_errno("asprintf");
1625 	}
1626 	err = got_path_create_file(path, headref_str);
1627 	free(headref_str);
1628 	free(path);
1629 	if (err)
1630 		return err;
1631 
1632 	if (asprintf(&path, "%s/%s", repo_path, "config") == -1)
1633 		return got_error_from_errno("asprintf");
1634 	err = got_path_create_file(path, gitconfig_str);
1635 	free(path);
1636 	if (err)
1637 		return err;
1638 
1639 	return NULL;
1640 }
1641 
1642 static const struct got_error *
1643 match_packed_object(struct got_object_id **unique_id,
1644     struct got_repository *repo, const char *id_str_prefix, int obj_type)
1645 {
1646 	const struct got_error *err = NULL;
1647 	struct got_object_id_queue matched_ids;
1648 	struct got_pathlist_entry *pe;
1649 
1650 	STAILQ_INIT(&matched_ids);
1651 
1652 	err = refresh_packidx_paths(repo);
1653 	if (err)
1654 		return err;
1655 
1656 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1657 		const char *path_packidx = pe->path;
1658 		struct got_packidx *packidx;
1659 		struct got_object_qid *qid;
1660 
1661 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
1662 		    path_packidx, 0);
1663 		if (err)
1664 			break;
1665 
1666 		err = got_packidx_match_id_str_prefix(&matched_ids,
1667 		    packidx, id_str_prefix);
1668 		if (err) {
1669 			got_packidx_close(packidx);
1670 			break;
1671 		}
1672 		err = got_packidx_close(packidx);
1673 		if (err)
1674 			break;
1675 
1676 		STAILQ_FOREACH(qid, &matched_ids, entry) {
1677 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1678 				int matched_type;
1679 				err = got_object_get_type(&matched_type, repo,
1680 				    &qid->id);
1681 				if (err)
1682 					goto done;
1683 				if (matched_type != obj_type)
1684 					continue;
1685 			}
1686 			if (*unique_id == NULL) {
1687 				*unique_id = got_object_id_dup(&qid->id);
1688 				if (*unique_id == NULL) {
1689 					err = got_error_from_errno("malloc");
1690 					goto done;
1691 				}
1692 			} else {
1693 				if (got_object_id_cmp(*unique_id,
1694 				    &qid->id) == 0)
1695 					continue; /* packed multiple times */
1696 				err = got_error(GOT_ERR_AMBIGUOUS_ID);
1697 				goto done;
1698 			}
1699 		}
1700 	}
1701 done:
1702 	got_object_id_queue_free(&matched_ids);
1703 	if (err) {
1704 		free(*unique_id);
1705 		*unique_id = NULL;
1706 	}
1707 	return err;
1708 }
1709 
1710 static const struct got_error *
1711 match_loose_object(struct got_object_id **unique_id, const char *path_objects,
1712     const char *object_dir, const char *id_str_prefix, int obj_type,
1713     struct got_repository *repo)
1714 {
1715 	const struct got_error *err = NULL;
1716 	char *path, *id_str = NULL;
1717 	DIR *dir = NULL;
1718 	struct dirent *dent;
1719 	struct got_object_id id;
1720 
1721 	if (asprintf(&path, "%s/%s", path_objects, object_dir) == -1) {
1722 		err = got_error_from_errno("asprintf");
1723 		goto done;
1724 	}
1725 
1726 	dir = opendir(path);
1727 	if (dir == NULL) {
1728 		if (errno == ENOENT) {
1729 			err = NULL;
1730 			goto done;
1731 		}
1732 		err = got_error_from_errno2("opendir", path);
1733 		goto done;
1734 	}
1735 	while ((dent = readdir(dir)) != NULL) {
1736 		int cmp;
1737 		enum got_hash_algorithm algo = GOT_HASH_SHA1;
1738 
1739 		free(id_str);
1740 		id_str = NULL;
1741 
1742 		if (strcmp(dent->d_name, ".") == 0 ||
1743 		    strcmp(dent->d_name, "..") == 0)
1744 			continue;
1745 
1746 		if (asprintf(&id_str, "%s%s", object_dir, dent->d_name) == -1) {
1747 			err = got_error_from_errno("asprintf");
1748 			goto done;
1749 		}
1750 
1751 		if (!got_parse_object_id(&id, id_str, algo))
1752 			continue;
1753 
1754 		/*
1755 		 * Directory entries do not necessarily appear in
1756 		 * sorted order, so we must iterate over all of them.
1757 		 */
1758 		cmp = strncmp(id_str, id_str_prefix, strlen(id_str_prefix));
1759 		if (cmp != 0)
1760 			continue;
1761 
1762 		if (*unique_id == NULL) {
1763 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1764 				int matched_type;
1765 				err = got_object_get_type(&matched_type, repo,
1766 				    &id);
1767 				if (err)
1768 					goto done;
1769 				if (matched_type != obj_type)
1770 					continue;
1771 			}
1772 			*unique_id = got_object_id_dup(&id);
1773 			if (*unique_id == NULL) {
1774 				err = got_error_from_errno("got_object_id_dup");
1775 				goto done;
1776 			}
1777 		} else {
1778 			if (got_object_id_cmp(*unique_id, &id) == 0)
1779 				continue; /* both packed and loose */
1780 			err = got_error(GOT_ERR_AMBIGUOUS_ID);
1781 			goto done;
1782 		}
1783 	}
1784 done:
1785 	if (dir && closedir(dir) != 0 && err == NULL)
1786 		err = got_error_from_errno("closedir");
1787 	if (err) {
1788 		free(*unique_id);
1789 		*unique_id = NULL;
1790 	}
1791 	free(id_str);
1792 	free(path);
1793 	return err;
1794 }
1795 
1796 const struct got_error *
1797 got_repo_match_object_id_prefix(struct got_object_id **id,
1798     const char *id_str_prefix, int obj_type, struct got_repository *repo)
1799 {
1800 	const struct got_error *err = NULL;
1801 	char *path_objects = NULL, *object_dir = NULL;
1802 	size_t len;
1803 	int i;
1804 
1805 	*id = NULL;
1806 
1807 	path_objects = got_repo_get_path_objects(repo);
1808 
1809 	len = strlen(id_str_prefix);
1810 	if (len > SHA1_DIGEST_STRING_LENGTH - 1) {
1811 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1812 		goto done;
1813 	}
1814 
1815 	for (i = 0; i < len; i++) {
1816 		if (isxdigit((unsigned char)id_str_prefix[i]))
1817 			continue;
1818 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1819 		goto done;
1820 	}
1821 
1822 	if (len >= 2) {
1823 		err = match_packed_object(id, repo, id_str_prefix, obj_type);
1824 		if (err)
1825 			goto done;
1826 		object_dir = strndup(id_str_prefix, 2);
1827 		if (object_dir == NULL) {
1828 			err = got_error_from_errno("strdup");
1829 			goto done;
1830 		}
1831 		err = match_loose_object(id, path_objects, object_dir,
1832 		    id_str_prefix, obj_type, repo);
1833 	} else if (len == 1) {
1834 		int i;
1835 		for (i = 0; i < 0xf; i++) {
1836 			if (asprintf(&object_dir, "%s%.1x", id_str_prefix, i)
1837 			    == -1) {
1838 				err = got_error_from_errno("asprintf");
1839 				goto done;
1840 			}
1841 			err = match_packed_object(id, repo, object_dir,
1842 			    obj_type);
1843 			if (err)
1844 				goto done;
1845 			err = match_loose_object(id, path_objects, object_dir,
1846 			    id_str_prefix, obj_type, repo);
1847 			if (err)
1848 				goto done;
1849 		}
1850 	} else {
1851 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1852 		goto done;
1853 	}
1854 done:
1855 	free(path_objects);
1856 	free(object_dir);
1857 	if (err) {
1858 		free(*id);
1859 		*id = NULL;
1860 	} else if (*id == NULL) {
1861 		switch (obj_type) {
1862 		case GOT_OBJ_TYPE_BLOB:
1863 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1864 			    GOT_OBJ_LABEL_BLOB, id_str_prefix);
1865 			break;
1866 		case GOT_OBJ_TYPE_TREE:
1867 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1868 			    GOT_OBJ_LABEL_TREE, id_str_prefix);
1869 			break;
1870 		case GOT_OBJ_TYPE_COMMIT:
1871 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1872 			    GOT_OBJ_LABEL_COMMIT, id_str_prefix);
1873 			break;
1874 		case GOT_OBJ_TYPE_TAG:
1875 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1876 			    GOT_OBJ_LABEL_TAG, id_str_prefix);
1877 			break;
1878 		default:
1879 			err = got_error_path(id_str_prefix, GOT_ERR_NO_OBJ);
1880 			break;
1881 		}
1882 	}
1883 
1884 	return err;
1885 }
1886 
1887 const struct got_error *
1888 got_repo_match_object_id(struct got_object_id **id, char **label,
1889     const char *id_str, int obj_type, struct got_reflist_head *refs,
1890     struct got_repository *repo)
1891 {
1892 	const struct got_error *err;
1893 	struct got_tag_object *tag;
1894 	struct got_reference *ref = NULL;
1895 
1896 	*id = NULL;
1897 	if (label)
1898 		*label = NULL;
1899 
1900 	if (refs) {
1901 		err = got_repo_object_match_tag(&tag, id_str, obj_type,
1902 		    refs, repo);
1903 		if (err == NULL) {
1904 			*id = got_object_id_dup(
1905 			    got_object_tag_get_object_id(tag));
1906 			if (*id == NULL)
1907 				err = got_error_from_errno("got_object_id_dup");
1908 			else if (label && asprintf(label, "refs/tags/%s",
1909 			    got_object_tag_get_name(tag)) == -1) {
1910 				err = got_error_from_errno("asprintf");
1911 				free(*id);
1912 				*id = NULL;
1913 			}
1914 			got_object_tag_close(tag);
1915 			return err;
1916 		} else if (err->code != GOT_ERR_OBJ_TYPE &&
1917 		    err->code != GOT_ERR_NO_OBJ)
1918 			return err;
1919 	}
1920 
1921 	err = got_ref_open(&ref, repo, id_str, 0);
1922 	if (err == NULL) {
1923 		err = got_ref_resolve(id, repo, ref);
1924 		if (err)
1925 			goto done;
1926 		if (label) {
1927 			*label = strdup(got_ref_get_name(ref));
1928 			if (*label == NULL) {
1929 				err = got_error_from_errno("strdup");
1930 				goto done;
1931 			}
1932 		}
1933 	} else {
1934 		if (err->code != GOT_ERR_NOT_REF &&
1935 		    err->code != GOT_ERR_BAD_REF_NAME)
1936 			goto done;
1937 		err = got_repo_match_object_id_prefix(id, id_str,
1938 		    obj_type, repo);
1939 		if (err) {
1940 			if (err->code == GOT_ERR_BAD_OBJ_ID_STR)
1941 				err = got_error_not_ref(id_str);
1942 			goto done;
1943 		}
1944 		if (label) {
1945 			err = got_object_id_str(label, *id);
1946 			if (*label == NULL) {
1947 				err = got_error_from_errno("strdup");
1948 				goto done;
1949 			}
1950 		}
1951 	}
1952 done:
1953 	if (ref)
1954 		got_ref_close(ref);
1955 	return err;
1956 }
1957 
1958 const struct got_error *
1959 got_repo_object_match_tag(struct got_tag_object **tag, const char *name,
1960     int obj_type, struct got_reflist_head *refs, struct got_repository *repo)
1961 {
1962 	const struct got_error *err = NULL;
1963 	struct got_reflist_entry *re;
1964 	struct got_object_id *tag_id;
1965 	int name_is_absolute = (strncmp(name, "refs/", 5) == 0);
1966 
1967 	*tag = NULL;
1968 
1969 	TAILQ_FOREACH(re, refs, entry) {
1970 		const char *refname;
1971 		refname = got_ref_get_name(re->ref);
1972 		if (got_ref_is_symbolic(re->ref))
1973 			continue;
1974 		if (strncmp(refname, "refs/tags/", 10) != 0)
1975 			continue;
1976 		if (!name_is_absolute)
1977 			refname += strlen("refs/tags/");
1978 		if (strcmp(refname, name) != 0)
1979 			continue;
1980 		err = got_ref_resolve(&tag_id, repo, re->ref);
1981 		if (err)
1982 			break;
1983 		err = got_object_open_as_tag(tag, repo, tag_id);
1984 		free(tag_id);
1985 		if (err)
1986 			break;
1987 		if (obj_type == GOT_OBJ_TYPE_ANY ||
1988 		    got_object_tag_get_object_type(*tag) == obj_type)
1989 			break;
1990 		got_object_tag_close(*tag);
1991 		*tag = NULL;
1992 	}
1993 
1994 	if (err == NULL && *tag == NULL)
1995 		err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1996 		    GOT_OBJ_LABEL_TAG, name);
1997 	return err;
1998 }
1999 
2000 static const struct got_error *
2001 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
2002     const char *name, mode_t mode, struct got_object_id *blob_id)
2003 {
2004 	const struct got_error *err = NULL;
2005 
2006 	 *new_te = NULL;
2007 
2008 	*new_te = calloc(1, sizeof(**new_te));
2009 	if (*new_te == NULL)
2010 		return got_error_from_errno("calloc");
2011 
2012 	if (strlcpy((*new_te)->name, name, sizeof((*new_te)->name)) >=
2013 	    sizeof((*new_te)->name)) {
2014 		err = got_error(GOT_ERR_NO_SPACE);
2015 		goto done;
2016 	}
2017 
2018 	if (S_ISLNK(mode)) {
2019 		(*new_te)->mode = S_IFLNK;
2020 	} else {
2021 		(*new_te)->mode = S_IFREG;
2022 		(*new_te)->mode |= (mode & (S_IRWXU | S_IRWXG | S_IRWXO));
2023 	}
2024 	memcpy(&(*new_te)->id, blob_id, sizeof((*new_te)->id));
2025 done:
2026 	if (err && *new_te) {
2027 		free(*new_te);
2028 		*new_te = NULL;
2029 	}
2030 	return err;
2031 }
2032 
2033 static const struct got_error *
2034 import_file(struct got_tree_entry **new_te, struct dirent *de,
2035     const char *path, struct got_repository *repo)
2036 {
2037 	const struct got_error *err;
2038 	struct got_object_id *blob_id = NULL;
2039 	char *filepath;
2040 	struct stat sb;
2041 
2042 	if (asprintf(&filepath, "%s%s%s", path,
2043 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2044 		return got_error_from_errno("asprintf");
2045 
2046 	if (lstat(filepath, &sb) != 0) {
2047 		err = got_error_from_errno2("lstat", path);
2048 		goto done;
2049 	}
2050 
2051 	err = got_object_blob_create(&blob_id, filepath, repo);
2052 	if (err)
2053 		goto done;
2054 
2055 	err = alloc_added_blob_tree_entry(new_te, de->d_name, sb.st_mode,
2056 	    blob_id);
2057 done:
2058 	free(filepath);
2059 	if (err)
2060 		free(blob_id);
2061 	return err;
2062 }
2063 
2064 static const struct got_error *
2065 insert_tree_entry(struct got_tree_entry *new_te,
2066     struct got_pathlist_head *paths)
2067 {
2068 	const struct got_error *err = NULL;
2069 	struct got_pathlist_entry *new_pe;
2070 
2071 	err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
2072 	if (err)
2073 		return err;
2074 	if (new_pe == NULL)
2075 		return got_error(GOT_ERR_TREE_DUP_ENTRY);
2076 	return NULL;
2077 }
2078 
2079 static const struct got_error *write_tree(struct got_object_id **,
2080     const char *, struct got_pathlist_head *, struct got_repository *,
2081     got_repo_import_cb progress_cb, void *progress_arg);
2082 
2083 static const struct got_error *
2084 import_subdir(struct got_tree_entry **new_te, struct dirent *de,
2085     const char *path, struct got_pathlist_head *ignores,
2086     struct got_repository *repo,
2087     got_repo_import_cb progress_cb, void *progress_arg)
2088 {
2089 	const struct got_error *err;
2090 	struct got_object_id *id = NULL;
2091 	char *subdirpath;
2092 
2093 	if (asprintf(&subdirpath, "%s%s%s", path,
2094 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2095 		return got_error_from_errno("asprintf");
2096 
2097 	(*new_te) = calloc(1, sizeof(**new_te));
2098 	if (*new_te == NULL)
2099 		return got_error_from_errno("calloc");
2100 	(*new_te)->mode = S_IFDIR;
2101 	if (strlcpy((*new_te)->name, de->d_name, sizeof((*new_te)->name)) >=
2102 	    sizeof((*new_te)->name)) {
2103 		err = got_error(GOT_ERR_NO_SPACE);
2104 		goto done;
2105 	}
2106 	err = write_tree(&id, subdirpath, ignores,  repo,
2107 	    progress_cb, progress_arg);
2108 	if (err)
2109 		goto done;
2110 	memcpy(&(*new_te)->id, id, sizeof((*new_te)->id));
2111 
2112 done:
2113 	free(id);
2114 	free(subdirpath);
2115 	if (err) {
2116 		free(*new_te);
2117 		*new_te = NULL;
2118 	}
2119 	return err;
2120 }
2121 
2122 static const struct got_error *
2123 write_tree(struct got_object_id **new_tree_id, const char *path_dir,
2124     struct got_pathlist_head *ignores, struct got_repository *repo,
2125     got_repo_import_cb progress_cb, void *progress_arg)
2126 {
2127 	const struct got_error *err = NULL;
2128 	DIR *dir;
2129 	struct dirent *de;
2130 	int nentries;
2131 	struct got_tree_entry *new_te = NULL;
2132 	struct got_pathlist_head paths;
2133 	struct got_pathlist_entry *pe;
2134 
2135 	*new_tree_id = NULL;
2136 
2137 	TAILQ_INIT(&paths);
2138 
2139 	dir = opendir(path_dir);
2140 	if (dir == NULL) {
2141 		err = got_error_from_errno2("opendir", path_dir);
2142 		goto done;
2143 	}
2144 
2145 	nentries = 0;
2146 	while ((de = readdir(dir)) != NULL) {
2147 		int ignore = 0;
2148 		int type;
2149 
2150 		if (strcmp(de->d_name, ".") == 0 ||
2151 		    strcmp(de->d_name, "..") == 0)
2152 			continue;
2153 
2154 		err = got_path_dirent_type(&type, path_dir, de);
2155 		if (err)
2156 			goto done;
2157 
2158 		TAILQ_FOREACH(pe, ignores, entry) {
2159 			if (type == DT_DIR && pe->path_len > 0 &&
2160 			    pe->path[pe->path_len - 1] == '/') {
2161 				char stripped[PATH_MAX];
2162 
2163 				if (strlcpy(stripped, pe->path,
2164 				    sizeof(stripped)) >= sizeof(stripped)) {
2165 					err = got_error(GOT_ERR_NO_SPACE);
2166 					goto done;
2167 				}
2168 				got_path_strip_trailing_slashes(stripped);
2169 				if (fnmatch(stripped, de->d_name, 0) == 0) {
2170 					ignore = 1;
2171 					break;
2172 				}
2173 			} else if (fnmatch(pe->path, de->d_name, 0) == 0) {
2174 				ignore = 1;
2175 				break;
2176 			}
2177 		}
2178 		if (ignore)
2179 			continue;
2180 
2181 		if (type == DT_DIR) {
2182 			err = import_subdir(&new_te, de, path_dir,
2183 			    ignores, repo, progress_cb, progress_arg);
2184 			if (err) {
2185 				if (err->code != GOT_ERR_NO_TREE_ENTRY)
2186 					goto done;
2187 				err = NULL;
2188 				continue;
2189 			}
2190 		} else if (type == DT_REG || type == DT_LNK) {
2191 			err = import_file(&new_te, de, path_dir, repo);
2192 			if (err)
2193 				goto done;
2194 		} else
2195 			continue;
2196 
2197 		err = insert_tree_entry(new_te, &paths);
2198 		if (err)
2199 			goto done;
2200 		nentries++;
2201 	}
2202 
2203 	if (TAILQ_EMPTY(&paths)) {
2204 		err = got_error_msg(GOT_ERR_NO_TREE_ENTRY,
2205 		    "cannot create tree without any entries");
2206 		goto done;
2207 	}
2208 
2209 	TAILQ_FOREACH(pe, &paths, entry) {
2210 		struct got_tree_entry *te = pe->data;
2211 		char *path;
2212 		if (!S_ISREG(te->mode) && !S_ISLNK(te->mode))
2213 			continue;
2214 		if (asprintf(&path, "%s/%s", path_dir, pe->path) == -1) {
2215 			err = got_error_from_errno("asprintf");
2216 			goto done;
2217 		}
2218 		err = (*progress_cb)(progress_arg, path);
2219 		free(path);
2220 		if (err)
2221 			goto done;
2222 	}
2223 
2224 	err = got_object_tree_create(new_tree_id, &paths, nentries, repo);
2225 done:
2226 	if (dir)
2227 		closedir(dir);
2228 	got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2229 	return err;
2230 }
2231 
2232 const struct got_error *
2233 got_repo_import(struct got_object_id **new_commit_id, const char *path_dir,
2234     const char *logmsg, const char *author, struct got_pathlist_head *ignores,
2235     struct got_repository *repo, got_repo_import_cb progress_cb,
2236     void *progress_arg)
2237 {
2238 	const struct got_error *err;
2239 	struct got_object_id *new_tree_id;
2240 
2241 	err = write_tree(&new_tree_id, path_dir, ignores, repo,
2242 	    progress_cb, progress_arg);
2243 	if (err)
2244 		return err;
2245 
2246 	err = got_object_commit_create(new_commit_id, new_tree_id, NULL, 0,
2247 	    author, time(NULL), author, time(NULL), logmsg, repo);
2248 	free(new_tree_id);
2249 	return err;
2250 }
2251 
2252 const struct got_error *
2253 got_repo_get_loose_object_info(int *nobjects, off_t *ondisk_size,
2254     struct got_repository *repo)
2255 {
2256 	const struct got_error *err = NULL;
2257 	char *path_objects = NULL, *path = NULL;
2258 	DIR *dir = NULL;
2259 	struct got_object_id id;
2260 	int i;
2261 
2262 	*nobjects = 0;
2263 	*ondisk_size = 0;
2264 
2265 	path_objects = got_repo_get_path_objects(repo);
2266 	if (path_objects == NULL)
2267 		return got_error_from_errno("got_repo_get_path_objects");
2268 
2269 	for (i = 0; i <= 0xff; i++) {
2270 		struct dirent *dent;
2271 
2272 		if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
2273 			err = got_error_from_errno("asprintf");
2274 			break;
2275 		}
2276 
2277 		dir = opendir(path);
2278 		if (dir == NULL) {
2279 			if (errno == ENOENT) {
2280 				err = NULL;
2281 				continue;
2282 			}
2283 			err = got_error_from_errno2("opendir", path);
2284 			break;
2285 		}
2286 
2287 		while ((dent = readdir(dir)) != NULL) {
2288 			char *id_str;
2289 			int fd;
2290 			struct stat sb;
2291 			enum got_hash_algorithm algo = GOT_HASH_SHA1;
2292 
2293 			if (strcmp(dent->d_name, ".") == 0 ||
2294 			    strcmp(dent->d_name, "..") == 0)
2295 				continue;
2296 
2297 			if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
2298 				err = got_error_from_errno("asprintf");
2299 				goto done;
2300 			}
2301 
2302 			if (!got_parse_object_id(&id, id_str, algo)) {
2303 				free(id_str);
2304 				continue;
2305 			}
2306 			free(id_str);
2307 
2308 			err = got_object_open_loose_fd(&fd, &id, repo);
2309 			if (err)
2310 				goto done;
2311 
2312 			if (fstat(fd, &sb) == -1) {
2313 				err = got_error_from_errno("fstat");
2314 				close(fd);
2315 				goto done;
2316 			}
2317 			(*nobjects)++;
2318 			(*ondisk_size) += sb.st_size;
2319 
2320 			if (close(fd) == -1) {
2321 				err = got_error_from_errno("close");
2322 				goto done;
2323 			}
2324 		}
2325 
2326 		if (closedir(dir) != 0) {
2327 			err = got_error_from_errno("closedir");
2328 			goto done;
2329 		}
2330 		dir = NULL;
2331 
2332 		free(path);
2333 		path = NULL;
2334 	}
2335 done:
2336 	if (dir && closedir(dir) != 0 && err == NULL)
2337 		err = got_error_from_errno("closedir");
2338 
2339 	if (err) {
2340 		*nobjects = 0;
2341 		*ondisk_size = 0;
2342 	}
2343 	free(path_objects);
2344 	free(path);
2345 	return err;
2346 }
2347 
2348 const struct got_error *
2349 got_repo_get_packfile_info(int *npackfiles, int *nobjects,
2350     off_t *total_packsize, struct got_repository *repo)
2351 {
2352 	const struct got_error *err = NULL;
2353 	DIR *packdir = NULL;
2354 	struct dirent *dent;
2355 	struct got_packidx *packidx = NULL;
2356 	char *path_packidx;
2357 	char *path_packfile;
2358 	int packdir_fd;
2359 	struct stat sb;
2360 
2361 	*npackfiles = 0;
2362 	*nobjects = 0;
2363 	*total_packsize = 0;
2364 
2365 	packdir_fd = openat(got_repo_get_fd(repo),
2366 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
2367 	if (packdir_fd == -1) {
2368 		return got_error_from_errno_fmt("openat: %s/%s",
2369 		    got_repo_get_path_git_dir(repo),
2370 		    GOT_OBJECTS_PACK_DIR);
2371 	}
2372 
2373 	packdir = fdopendir(packdir_fd);
2374 	if (packdir == NULL) {
2375 		err = got_error_from_errno("fdopendir");
2376 		goto done;
2377 	}
2378 
2379 	while ((dent = readdir(packdir)) != NULL) {
2380 		if (!got_repo_is_packidx_filename(dent->d_name,
2381 		    strlen(dent->d_name)))
2382 			continue;
2383 
2384 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
2385 		    dent->d_name) == -1) {
2386 			err = got_error_from_errno("asprintf");
2387 			goto done;
2388 		}
2389 
2390 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
2391 		    path_packidx, 0);
2392 		free(path_packidx);
2393 		if (err)
2394 			goto done;
2395 
2396 		if (fstat(packidx->fd, &sb) == -1)
2397 			goto done;
2398 		*total_packsize += sb.st_size;
2399 
2400 		err = got_packidx_get_packfile_path(&path_packfile,
2401 		    packidx->path_packidx);
2402 		if (err)
2403 			goto done;
2404 
2405 		if (fstatat(got_repo_get_fd(repo), path_packfile, &sb,
2406 		    0) == -1) {
2407 			free(path_packfile);
2408 			goto done;
2409 		}
2410 		free(path_packfile);
2411 		*total_packsize += sb.st_size;
2412 
2413 		*nobjects += be32toh(packidx->hdr.fanout_table[0xff]);
2414 
2415 		(*npackfiles)++;
2416 
2417 		got_packidx_close(packidx);
2418 		packidx = NULL;
2419 	}
2420 done:
2421 	if (packidx)
2422 		got_packidx_close(packidx);
2423 	if (packdir && closedir(packdir) != 0 && err == NULL)
2424 		err = got_error_from_errno("closedir");
2425 	if (err) {
2426 		*npackfiles = 0;
2427 		*nobjects = 0;
2428 		*total_packsize = 0;
2429 	}
2430 	return err;
2431 }
2432 
2433 RB_GENERATE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
2434     got_packidx_bloom_filter_cmp);