1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include "got_compat.h"
18 
19 #include <sys/types.h>
20 #include <sys/queue.h>
21 #include <sys/uio.h>
22 #include <sys/socket.h>
23 #include <sys/stat.h>
24 #include <sys/mman.h>
25 #include <sys/resource.h>
26 
27 #include <ctype.h>
28 #include <fcntl.h>
29 #include <fnmatch.h>
30 #include <limits.h>
31 #include <dirent.h>
32 #include <stdlib.h>
33 #include <stdio.h>
34 #include <string.h>
35 #include <time.h>
36 #include <unistd.h>
37 #include <zlib.h>
38 #include <errno.h>
39 #include <libgen.h>
40 #include <stdint.h>
41 
42 #include "bloom.h"
43 
44 #include "got_error.h"
45 #include "got_reference.h"
46 #include "got_repository.h"
47 #include "got_path.h"
48 #include "got_cancel.h"
49 #include "got_object.h"
50 #include "got_opentemp.h"
51 
52 #include "got_lib_delta.h"
53 #include "got_lib_delta_cache.h"
54 #include "got_lib_inflate.h"
55 #include "got_lib_object.h"
56 #include "got_lib_object_parse.h"
57 #include "got_lib_object_create.h"
58 #include "got_lib_pack.h"
59 #include "got_lib_privsep.h"
60 #include "got_lib_hash.h"
61 #include "got_lib_object_cache.h"
62 #include "got_lib_repository.h"
63 #include "got_lib_gotconfig.h"
64 
65 #ifndef nitems
66 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
67 #endif
68 
69 #define GOT_PACK_NUM_TEMPFILES		GOT_PACK_CACHE_SIZE * 2
70 
71 RB_PROTOTYPE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
72     got_packidx_bloom_filter_cmp);
73 
74 static inline int
75 is_boolean_val(const char *val)
76 {
77 	return (strcasecmp(val, "true") == 0 ||
78 	    strcasecmp(val, "false") == 0 ||
79 	    strcasecmp(val, "on") == 0 ||
80     	    strcasecmp(val, "off") == 0 ||
81 	    strcasecmp(val, "yes") == 0 ||
82     	    strcasecmp(val, "no") == 0 ||
83 	    strcasecmp(val, "1") == 0 ||
84 	    strcasecmp(val, "0") == 0);
85 }
86 
87 static inline int
88 get_boolean_val(const char *val)
89 {
90 	return (strcasecmp(val, "true") == 0 ||
91 	    strcasecmp(val, "on") == 0 ||
92 	    strcasecmp(val, "yes") == 0 ||
93 	    strcasecmp(val, "1") == 0);
94 }
95 
96 const char *
97 got_repo_get_path(struct got_repository *repo)
98 {
99 	return repo->path;
100 }
101 
102 const char *
103 got_repo_get_path_git_dir(struct got_repository *repo)
104 {
105 	return repo->path_git_dir;
106 }
107 
108 int
109 got_repo_get_fd(struct got_repository *repo)
110 {
111 	return repo->gitdir_fd;
112 }
113 
114 enum got_hash_algorithm
115 got_repo_get_object_format(struct got_repository *repo)
116 {
117 	return repo->algo;
118 }
119 
120 const char *
121 got_repo_get_gitconfig_author_name(struct got_repository *repo)
122 {
123 	return repo->gitconfig_author_name;
124 }
125 
126 const char *
127 got_repo_get_gitconfig_author_email(struct got_repository *repo)
128 {
129 	return repo->gitconfig_author_email;
130 }
131 
132 const char *
133 got_repo_get_global_gitconfig_author_name(struct got_repository *repo)
134 {
135 	return repo->global_gitconfig_author_name;
136 }
137 
138 const char *
139 got_repo_get_global_gitconfig_author_email(struct got_repository *repo)
140 {
141 	return repo->global_gitconfig_author_email;
142 }
143 
144 const char *
145 got_repo_get_gitconfig_owner(struct got_repository *repo)
146 {
147 	return repo->gitconfig_owner;
148 }
149 
150 int
151 got_repo_has_extension(struct got_repository *repo, const char *ext)
152 {
153 	int i;
154 
155 	for (i = 0; i < repo->nextensions; ++i) {
156 		if (!strcasecmp(ext, repo->extnames[i]))
157 			return get_boolean_val(repo->extvals[i]);
158 	}
159 
160 	return 0;
161 }
162 
163 int
164 got_repo_is_bare(struct got_repository *repo)
165 {
166 	return (strcmp(repo->path, repo->path_git_dir) == 0);
167 }
168 
169 static char *
170 get_path_git_child(struct got_repository *repo, const char *basename)
171 {
172 	char *path_child;
173 
174 	if (asprintf(&path_child, "%s/%s", repo->path_git_dir,
175 	    basename) == -1)
176 		return NULL;
177 
178 	return path_child;
179 }
180 
181 char *
182 got_repo_get_path_objects(struct got_repository *repo)
183 {
184 	return get_path_git_child(repo, GOT_OBJECTS_DIR);
185 }
186 
187 char *
188 got_repo_get_path_objects_pack(struct got_repository *repo)
189 {
190 	return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR);
191 }
192 
193 char *
194 got_repo_get_path_refs(struct got_repository *repo)
195 {
196 	return get_path_git_child(repo, GOT_REFS_DIR);
197 }
198 
199 char *
200 got_repo_get_path_packed_refs(struct got_repository *repo)
201 {
202 	return get_path_git_child(repo, GOT_PACKED_REFS_FILE);
203 }
204 
205 static char *
206 get_path_head(struct got_repository *repo)
207 {
208 	return get_path_git_child(repo, GOT_HEAD_FILE);
209 }
210 
211 char *
212 got_repo_get_path_gitconfig(struct got_repository *repo)
213 {
214 	return get_path_git_child(repo, GOT_GITCONFIG);
215 }
216 
217 char *
218 got_repo_get_path_gotconfig(struct got_repository *repo)
219 {
220 	return get_path_git_child(repo, GOT_GOTCONFIG_FILENAME);
221 }
222 
223 const struct got_gotconfig *
224 got_repo_get_gotconfig(struct got_repository *repo)
225 {
226 	return repo->gotconfig;
227 }
228 
229 void
230 got_repo_get_gitconfig_remotes(int *nremotes,
231     const struct got_remote_repo **remotes, struct got_repository *repo)
232 {
233 	*nremotes = repo->ngitconfig_remotes;
234 	*remotes = repo->gitconfig_remotes;
235 }
236 
237 static int
238 is_git_repo(struct got_repository *repo)
239 {
240 	const char *path_git = got_repo_get_path_git_dir(repo);
241 	char *path_objects = got_repo_get_path_objects(repo);
242 	char *path_refs = got_repo_get_path_refs(repo);
243 	char *path_head = get_path_head(repo);
244 	int ret = 0;
245 	struct stat sb;
246 	struct got_reference *head_ref;
247 
248 	if (lstat(path_git, &sb) == -1)
249 		goto done;
250 	if (!S_ISDIR(sb.st_mode))
251 		goto done;
252 
253 	if (lstat(path_objects, &sb) == -1)
254 		goto done;
255 	if (!S_ISDIR(sb.st_mode))
256 		goto done;
257 
258 	if (lstat(path_refs, &sb) == -1)
259 		goto done;
260 	if (!S_ISDIR(sb.st_mode))
261 		goto done;
262 
263 	if (lstat(path_head, &sb) == -1)
264 		goto done;
265 	if (!S_ISREG(sb.st_mode))
266 		goto done;
267 
268 	/* Check if the HEAD reference can be opened. */
269 	if (got_ref_open(&head_ref, repo, GOT_REF_HEAD, 0) != NULL)
270 		goto done;
271 	got_ref_close(head_ref);
272 
273 	ret = 1;
274 done:
275 	free(path_objects);
276 	free(path_refs);
277 	free(path_head);
278 	return ret;
279 
280 }
281 
282 static const struct got_error *
283 close_tempfiles(int *fds, size_t nfds)
284 {
285 	const struct got_error *err = NULL;
286 	int i;
287 
288 	for (i = 0; i < nfds; i++) {
289 		if (fds[i] == -1)
290 			continue;
291 		if (close(fds[i]) == -1) {
292 			err = got_error_from_errno("close");
293 			break;
294 		}
295 	}
296 	free(fds);
297 	return err;
298 }
299 
300 static const struct got_error *
301 open_tempfiles(int **fds, size_t array_size, size_t nfds)
302 {
303 	const struct got_error *err = NULL;
304 	int i;
305 
306 	*fds = calloc(array_size, sizeof(**fds));
307 	if (*fds == NULL)
308 		return got_error_from_errno("calloc");
309 
310 	for (i = 0; i < array_size; i++)
311 		(*fds)[i] = -1;
312 
313 	for (i = 0; i < nfds; i++) {
314 		(*fds)[i] = got_opentempfd();
315 		if ((*fds)[i] == -1) {
316 			err = got_error_from_errno("got_opentempfd");
317 			close_tempfiles(*fds, nfds);
318 			*fds = NULL;
319 			return err;
320 		}
321 	}
322 
323 	return NULL;
324 }
325 
326 static const struct got_error *
327 get_pack_cache_size(int *pack_cache_size)
328 {
329 	struct rlimit rl;
330 
331 	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
332 		return got_error_from_errno("getrlimit");
333 
334 	*pack_cache_size = GOT_PACK_CACHE_SIZE;
335 	if (*pack_cache_size > rl.rlim_cur / 8)
336 		*pack_cache_size = rl.rlim_cur / 8;
337 
338 	return NULL;
339 }
340 
341 const struct got_error *
342 got_repo_pack_fds_open(int **pack_fds)
343 {
344 	const struct got_error *err;
345 	int nfds;
346 
347 	err = get_pack_cache_size(&nfds);
348 	if (err)
349 		return err;
350 
351 	/*
352 	 * We need one basefd and one accumfd per cached pack.
353 	 * Our constants should be set up in a way such that
354 	 * this error never triggers.
355 	 */
356 	if (nfds * 2 > GOT_PACK_NUM_TEMPFILES)
357 		return got_error(GOT_ERR_NO_SPACE);
358 
359 	return open_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES, nfds * 2);
360 }
361 
362 const struct got_error *
363 got_repo_pack_fds_close(int *pack_fds)
364 {
365 	return close_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES);
366 }
367 
368 const struct got_error *
369 got_repo_temp_fds_open(int **temp_fds)
370 {
371 	return open_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES,
372 	    GOT_REPO_NUM_TEMPFILES);
373 }
374 
375 void
376 got_repo_temp_fds_set(struct got_repository *repo, int *temp_fds)
377 {
378 	int i;
379 
380 	for (i = 0; i < GOT_REPO_NUM_TEMPFILES; i++)
381 		repo->tempfiles[i] = temp_fds[i];
382 }
383 
384 const struct got_error *
385 got_repo_temp_fds_get(int *fd, int *idx, struct got_repository *repo)
386 {
387 	int i;
388 
389 	*fd = -1;
390 	*idx = -1;
391 
392 	for (i = 0; i < nitems(repo->tempfiles); i++) {
393 		if (repo->tempfile_use_mask & (1 << i))
394 			continue;
395 		if (repo->tempfiles[i] != -1) {
396 			if (ftruncate(repo->tempfiles[i], 0L) == -1)
397 				return got_error_from_errno("ftruncate");
398 			*fd = repo->tempfiles[i];
399 			*idx = i;
400 			repo->tempfile_use_mask |= (1 << i);
401 			return NULL;
402 		}
403 	}
404 
405 	return got_error(GOT_ERR_REPO_TEMPFILE);
406 }
407 
408 void
409 got_repo_temp_fds_put(int idx, struct got_repository *repo)
410 {
411 	repo->tempfile_use_mask &= ~(1 << idx);
412 }
413 
414 const struct got_error *
415 got_repo_temp_fds_close(int *temp_fds)
416 {
417 	return close_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES);
418 }
419 
420 const struct got_error *
421 got_repo_cache_object(struct got_repository *repo, struct got_object_id *id,
422     struct got_object *obj)
423 {
424 #ifndef GOT_NO_OBJ_CACHE
425 	const struct got_error *err = NULL;
426 	err = got_object_cache_add(&repo->objcache, id, obj);
427 	if (err) {
428 		if (err->code == GOT_ERR_OBJ_EXISTS ||
429 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
430 			err = NULL;
431 		return err;
432 	}
433 	obj->refcnt++;
434 #endif
435 	return NULL;
436 }
437 
438 struct got_object *
439 got_repo_get_cached_object(struct got_repository *repo,
440     struct got_object_id *id)
441 {
442 	return (struct got_object *)got_object_cache_get(&repo->objcache, id);
443 }
444 
445 const struct got_error *
446 got_repo_cache_tree(struct got_repository *repo, struct got_object_id *id,
447     struct got_tree_object *tree)
448 {
449 #ifndef GOT_NO_OBJ_CACHE
450 	const struct got_error *err = NULL;
451 	err = got_object_cache_add(&repo->treecache, id, tree);
452 	if (err) {
453 		if (err->code == GOT_ERR_OBJ_EXISTS ||
454 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
455 			err = NULL;
456 		return err;
457 	}
458 	tree->refcnt++;
459 #endif
460 	return NULL;
461 }
462 
463 struct got_tree_object *
464 got_repo_get_cached_tree(struct got_repository *repo,
465     struct got_object_id *id)
466 {
467 	return (struct got_tree_object *)got_object_cache_get(
468 	    &repo->treecache, id);
469 }
470 
471 const struct got_error *
472 got_repo_cache_commit(struct got_repository *repo, struct got_object_id *id,
473     struct got_commit_object *commit)
474 {
475 #ifndef GOT_NO_OBJ_CACHE
476 	const struct got_error *err = NULL;
477 	err = got_object_cache_add(&repo->commitcache, id, commit);
478 	if (err) {
479 		if (err->code == GOT_ERR_OBJ_EXISTS ||
480 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
481 			err = NULL;
482 		return err;
483 	}
484 	commit->refcnt++;
485 #endif
486 	return NULL;
487 }
488 
489 struct got_commit_object *
490 got_repo_get_cached_commit(struct got_repository *repo,
491     struct got_object_id *id)
492 {
493 	return (struct got_commit_object *)got_object_cache_get(
494 	    &repo->commitcache, id);
495 }
496 
497 const struct got_error *
498 got_repo_cache_tag(struct got_repository *repo, struct got_object_id *id,
499     struct got_tag_object *tag)
500 {
501 #ifndef GOT_NO_OBJ_CACHE
502 	const struct got_error *err = NULL;
503 	err = got_object_cache_add(&repo->tagcache, id, tag);
504 	if (err) {
505 		if (err->code == GOT_ERR_OBJ_EXISTS ||
506 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
507 			err = NULL;
508 		return err;
509 	}
510 	tag->refcnt++;
511 #endif
512 	return NULL;
513 }
514 
515 struct got_tag_object *
516 got_repo_get_cached_tag(struct got_repository *repo, struct got_object_id *id)
517 {
518 	return (struct got_tag_object *)got_object_cache_get(
519 	    &repo->tagcache, id);
520 }
521 
522 const struct got_error *
523 got_repo_cache_raw_object(struct got_repository *repo, struct got_object_id *id,
524     struct got_raw_object *raw)
525 {
526 #ifndef GOT_NO_OBJ_CACHE
527 	const struct got_error *err = NULL;
528 	err = got_object_cache_add(&repo->rawcache, id, raw);
529 	if (err) {
530 		if (err->code == GOT_ERR_OBJ_EXISTS ||
531 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
532 			err = NULL;
533 		return err;
534 	}
535 	raw->refcnt++;
536 #endif
537 	return NULL;
538 }
539 
540 
541 struct got_raw_object *
542 got_repo_get_cached_raw_object(struct got_repository *repo,
543     struct got_object_id *id)
544 {
545 	return (struct got_raw_object *)got_object_cache_get(&repo->rawcache, id);
546 }
547 
548 
549 static const struct got_error *
550 open_repo(struct got_repository *repo, const char *path)
551 {
552 	const struct got_error *err = NULL;
553 
554 	repo->gitdir_fd = -1;
555 
556 	/* bare git repository? */
557 	repo->path_git_dir = strdup(path);
558 	if (repo->path_git_dir == NULL)
559 		return got_error_from_errno("strdup");
560 	if (is_git_repo(repo)) {
561 		repo->path = strdup(repo->path_git_dir);
562 		if (repo->path == NULL) {
563 			err = got_error_from_errno("strdup");
564 			goto done;
565 		}
566 		repo->gitdir_fd = open(repo->path_git_dir,
567 		    O_DIRECTORY | O_CLOEXEC);
568 		if (repo->gitdir_fd == -1) {
569 			err = got_error_from_errno2("open",
570 			    repo->path_git_dir);
571 			goto done;
572 		}
573 		return NULL;
574 	}
575 
576 	/* git repository with working tree? */
577 	free(repo->path_git_dir);
578 	repo->path_git_dir = NULL;
579 	if (asprintf(&repo->path_git_dir, "%s/%s", path, GOT_GIT_DIR) == -1) {
580 		err = got_error_from_errno("asprintf");
581 		goto done;
582 	}
583 	if (is_git_repo(repo)) {
584 		repo->path = strdup(path);
585 		if (repo->path == NULL) {
586 			err = got_error_from_errno("strdup");
587 			goto done;
588 		}
589 		repo->gitdir_fd = open(repo->path_git_dir,
590 		    O_DIRECTORY | O_CLOEXEC);
591 		if (repo->gitdir_fd == -1) {
592 			err = got_error_from_errno2("open",
593 			    repo->path_git_dir);
594 			goto done;
595 		}
596 		return NULL;
597 	}
598 
599 	err = got_error(GOT_ERR_NOT_GIT_REPO);
600 done:
601 	if (err) {
602 		free(repo->path);
603 		repo->path = NULL;
604 		free(repo->path_git_dir);
605 		repo->path_git_dir = NULL;
606 		if (repo->gitdir_fd != -1)
607 			close(repo->gitdir_fd);
608 		repo->gitdir_fd = -1;
609 
610 	}
611 	return err;
612 }
613 
614 static const struct got_error *
615 read_gitconfig(struct got_repository *repo, const char *global_gitconfig_path)
616 {
617 	const struct got_error *err = NULL;
618 	char *repo_gitconfig_path = NULL;
619 
620 	if (global_gitconfig_path) {
621 		/* Read settings from ~/.gitconfig. */
622 		int dummy_repo_version;
623 		err = got_repo_read_gitconfig(&dummy_repo_version,
624 		    &repo->global_gitconfig_author_name,
625 		    &repo->global_gitconfig_author_email,
626 		    NULL, NULL, NULL, NULL, NULL, NULL,
627 		    global_gitconfig_path);
628 		if (err)
629 			return err;
630 	}
631 
632 	/* Read repository's .git/config file. */
633 	repo_gitconfig_path = got_repo_get_path_gitconfig(repo);
634 	if (repo_gitconfig_path == NULL)
635 		return got_error_from_errno("got_repo_get_path_gitconfig");
636 
637 	err = got_repo_read_gitconfig(
638 	    &repo->gitconfig_repository_format_version,
639 	    &repo->gitconfig_author_name, &repo->gitconfig_author_email,
640 	    &repo->gitconfig_remotes, &repo->ngitconfig_remotes,
641 	    &repo->gitconfig_owner, &repo->extnames, &repo->extvals,
642 	    &repo->nextensions, repo_gitconfig_path);
643 	if (err)
644 		goto done;
645 
646 	if (getenv("GOT_IGNORE_GITCONFIG") != NULL) {
647 		int i;
648 
649 		for (i = 0; i < repo->ngitconfig_remotes; i++) {
650 			got_repo_free_remote_repo_data(
651 			    &repo->gitconfig_remotes[i]);
652 		}
653 		free(repo->gitconfig_remotes);
654 		repo->gitconfig_remotes = NULL;
655 		repo->ngitconfig_remotes = 0;
656 
657 		free(repo->gitconfig_author_name);
658 		repo->gitconfig_author_name = NULL;
659 		free(repo->gitconfig_author_email);
660 		repo->gitconfig_author_email = NULL;
661 
662 		free(repo->global_gitconfig_author_name);
663 		repo->global_gitconfig_author_name = NULL;
664 		free(repo->global_gitconfig_author_email);
665 		repo->global_gitconfig_author_email = NULL;
666 	}
667 
668 done:
669 	free(repo_gitconfig_path);
670 	return err;
671 }
672 
673 static const struct got_error *
674 read_gotconfig(struct got_repository *repo)
675 {
676 	const struct got_error *err = NULL;
677 	char *gotconfig_path;
678 
679 	gotconfig_path = got_repo_get_path_gotconfig(repo);
680 	if (gotconfig_path == NULL)
681 		return got_error_from_errno("got_repo_get_path_gotconfig");
682 
683 	err = got_gotconfig_read(&repo->gotconfig, gotconfig_path);
684 	free(gotconfig_path);
685 	return err;
686 }
687 
688 /* Supported repository format extensions. */
689 static const char *const repo_extensions[] = {
690 	"noop",			/* Got supports repository format version 1. */
691 	"preciousObjects",	/* Supported by gotadmin cleanup. */
692 	"worktreeConfig",	/* Got does not care about Git work trees. */
693 };
694 
695 const struct got_error *
696 got_repo_open(struct got_repository **repop, const char *path,
697     const char *global_gitconfig_path, int *pack_fds)
698 {
699 	struct got_repository *repo = NULL;
700 	const struct got_error *err = NULL;
701 	char *repo_path = NULL;
702 	size_t i, j = 0;
703 
704 	*repop = NULL;
705 
706 	repo = calloc(1, sizeof(*repo));
707 	if (repo == NULL)
708 		return got_error_from_errno("calloc");
709 
710 	RB_INIT(&repo->packidx_bloom_filters);
711 	TAILQ_INIT(&repo->packidx_paths);
712 
713 	for (i = 0; i < nitems(repo->privsep_children); i++) {
714 		memset(&repo->privsep_children[i], 0,
715 		    sizeof(repo->privsep_children[0]));
716 		repo->privsep_children[i].imsg_fd = -1;
717 	}
718 
719 	err = got_object_cache_init(&repo->objcache,
720 	    GOT_OBJECT_CACHE_TYPE_OBJ);
721 	if (err)
722 		goto done;
723 	err = got_object_cache_init(&repo->treecache,
724 	    GOT_OBJECT_CACHE_TYPE_TREE);
725 	if (err)
726 		goto done;
727 	err = got_object_cache_init(&repo->commitcache,
728 	    GOT_OBJECT_CACHE_TYPE_COMMIT);
729 	if (err)
730 		goto done;
731 	err = got_object_cache_init(&repo->tagcache,
732 	    GOT_OBJECT_CACHE_TYPE_TAG);
733 	if (err)
734 		goto done;
735 	err = got_object_cache_init(&repo->rawcache,
736 	    GOT_OBJECT_CACHE_TYPE_RAW);
737 	if (err)
738 		goto done;
739 
740 	err = get_pack_cache_size(&repo->pack_cache_size);
741 	if (err)
742 		goto done;
743 	for (i = 0; i < nitems(repo->packs); i++) {
744 		if (pack_fds != NULL && i < repo->pack_cache_size) {
745 			repo->packs[i].basefd = pack_fds[j++];
746 			repo->packs[i].accumfd = pack_fds[j++];
747 		} else {
748 			repo->packs[i].basefd = -1;
749 			repo->packs[i].accumfd = -1;
750 		}
751 	}
752 	for (i = 0; i < nitems(repo->tempfiles); i++)
753 		repo->tempfiles[i] = -1;
754 	repo->pinned_pack = -1;
755 	repo->pinned_packidx = -1;
756 	repo->pinned_pid = 0;
757 
758 	repo_path = realpath(path, NULL);
759 	if (repo_path == NULL) {
760 		err = got_error_from_errno2("realpath", path);
761 		goto done;
762 	}
763 
764 	for (;;) {
765 		char *parent_path;
766 
767 		err = open_repo(repo, repo_path);
768 		if (err == NULL)
769 			break;
770 		if (err->code != GOT_ERR_NOT_GIT_REPO)
771 			goto done;
772 		if (repo_path[0] == '/' && repo_path[1] == '\0') {
773 			err = got_error(GOT_ERR_NOT_GIT_REPO);
774 			goto done;
775 		}
776 		err = got_path_dirname(&parent_path, repo_path);
777 		if (err)
778 			goto done;
779 		free(repo_path);
780 		repo_path = parent_path;
781 	}
782 
783 	err = read_gotconfig(repo);
784 	if (err)
785 		goto done;
786 
787 	err = read_gitconfig(repo, global_gitconfig_path);
788 	if (err)
789 		goto done;
790 	if (repo->gitconfig_repository_format_version != 0) {
791 		err = got_error_path(path, GOT_ERR_GIT_REPO_FORMAT);
792 		goto done;
793 	}
794 	for (i = 0; i < repo->nextensions; i++) {
795 		char *ext = repo->extnames[i];
796 		char *val = repo->extvals[i];
797 		int j, supported = 0;
798 
799 		if (!is_boolean_val(val)) {
800 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
801 			goto done;
802 		}
803 
804 		if (!get_boolean_val(val))
805 			continue;
806 
807 		for (j = 0; j < nitems(repo_extensions); j++) {
808 			if (strcmp(ext, repo_extensions[j]) == 0) {
809 				supported = 1;
810 				break;
811 			}
812 		}
813 		if (!supported) {
814 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
815 			goto done;
816 		}
817 	}
818 
819 	err = got_repo_list_packidx(&repo->packidx_paths, repo);
820 done:
821 	if (err)
822 		got_repo_close(repo);
823 	else
824 		*repop = repo;
825 	free(repo_path);
826 	return err;
827 }
828 
829 const struct got_error *
830 got_repo_close(struct got_repository *repo)
831 {
832 	const struct got_error *err = NULL, *child_err;
833 	struct got_packidx_bloom_filter *bf;
834 	size_t i;
835 
836 	for (i = 0; i < repo->pack_cache_size; i++) {
837 		if (repo->packidx_cache[i] == NULL)
838 			break;
839 		got_packidx_close(repo->packidx_cache[i]);
840 	}
841 
842 	while ((bf = RB_MIN(got_packidx_bloom_filter_tree,
843 	    &repo->packidx_bloom_filters))) {
844 		RB_REMOVE(got_packidx_bloom_filter_tree,
845 		    &repo->packidx_bloom_filters, bf);
846 		bloom_free(bf->bloom);
847 		free(bf->bloom);
848 		free(bf);
849 	}
850 
851 	for (i = 0; i < repo->pack_cache_size; i++)
852 		if (repo->packs[i].path_packfile)
853 			if (repo->packs[i].path_packfile)
854 				got_pack_close(&repo->packs[i]);
855 
856 	free(repo->path);
857 	free(repo->path_git_dir);
858 
859 	got_object_cache_close(&repo->objcache);
860 	got_object_cache_close(&repo->treecache);
861 	got_object_cache_close(&repo->commitcache);
862 	got_object_cache_close(&repo->tagcache);
863 	got_object_cache_close(&repo->rawcache);
864 
865 	for (i = 0; i < nitems(repo->privsep_children); i++) {
866 		if (repo->privsep_children[i].imsg_fd == -1)
867 			continue;
868 		imsg_clear(repo->privsep_children[i].ibuf);
869 		free(repo->privsep_children[i].ibuf);
870 		err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd);
871 		child_err = got_privsep_wait_for_child(
872 		    repo->privsep_children[i].pid);
873 		if (child_err && err == NULL)
874 			err = child_err;
875 		if (close(repo->privsep_children[i].imsg_fd) == -1 &&
876 		    err == NULL)
877 			err = got_error_from_errno("close");
878 	}
879 
880 	if (repo->gitdir_fd != -1 && close(repo->gitdir_fd) == -1 &&
881 	    err == NULL)
882 		err = got_error_from_errno("close");
883 
884 	if (repo->gotconfig)
885 		got_gotconfig_free(repo->gotconfig);
886 	free(repo->gitconfig_author_name);
887 	free(repo->gitconfig_author_email);
888 	for (i = 0; i < repo->ngitconfig_remotes; i++)
889 		got_repo_free_remote_repo_data(&repo->gitconfig_remotes[i]);
890 	free(repo->gitconfig_remotes);
891 	for (i = 0; i < repo->nextensions; i++) {
892 		free(repo->extnames[i]);
893 		free(repo->extvals[i]);
894 	}
895 	free(repo->extnames);
896 	free(repo->extvals);
897 
898 	got_pathlist_free(&repo->packidx_paths, GOT_PATHLIST_FREE_PATH);
899 	free(repo);
900 
901 	return err;
902 }
903 
904 void
905 got_repo_free_remote_repo_data(struct got_remote_repo *repo)
906 {
907 	int i;
908 
909 	free(repo->name);
910 	repo->name = NULL;
911 	free(repo->fetch_url);
912 	repo->fetch_url = NULL;
913 	free(repo->send_url);
914 	repo->send_url = NULL;
915 	for (i = 0; i < repo->nfetch_branches; i++)
916 		free(repo->fetch_branches[i]);
917 	free(repo->fetch_branches);
918 	repo->fetch_branches = NULL;
919 	repo->nfetch_branches = 0;
920 	for (i = 0; i < repo->nsend_branches; i++)
921 		free(repo->send_branches[i]);
922 	free(repo->send_branches);
923 	repo->send_branches = NULL;
924 	repo->nsend_branches = 0;
925 }
926 
927 const struct got_error *
928 got_repo_map_path(char **in_repo_path, struct got_repository *repo,
929     const char *input_path)
930 {
931 	const struct got_error *err = NULL;
932 	const char *repo_abspath = NULL;
933 	size_t repolen, len;
934 	char *canonpath, *path = NULL;
935 
936 	*in_repo_path = NULL;
937 
938 	canonpath = strdup(input_path);
939 	if (canonpath == NULL) {
940 		err = got_error_from_errno("strdup");
941 		goto done;
942 	}
943 	err = got_canonpath(input_path, canonpath, strlen(canonpath) + 1);
944 	if (err)
945 		goto done;
946 
947 	repo_abspath = got_repo_get_path(repo);
948 
949 	if (canonpath[0] == '\0') {
950 		path = strdup(canonpath);
951 		if (path == NULL) {
952 			err = got_error_from_errno("strdup");
953 			goto done;
954 		}
955 	} else {
956 		path = realpath(canonpath, NULL);
957 		if (path == NULL) {
958 			if (errno != ENOENT) {
959 				err = got_error_from_errno2("realpath",
960 				    canonpath);
961 				goto done;
962 			}
963 			/*
964 			 * Path is not on disk.
965 			 * Assume it is already relative to repository root.
966 			 */
967 			path = strdup(canonpath);
968 			if (path == NULL) {
969 				err = got_error_from_errno("strdup");
970 				goto done;
971 			}
972 		}
973 
974 		repolen = strlen(repo_abspath);
975 		len = strlen(path);
976 
977 
978 		if (strcmp(path, repo_abspath) == 0) {
979 			free(path);
980 			path = strdup("");
981 			if (path == NULL) {
982 				err = got_error_from_errno("strdup");
983 				goto done;
984 			}
985 		} else if (len > repolen &&
986 		    got_path_is_child(path, repo_abspath, repolen)) {
987 			/* Matched an on-disk path inside repository. */
988 			if (got_repo_is_bare(repo)) {
989 				/*
990 				 * Matched an on-disk path inside repository
991 				 * database. Treat input as repository-relative.
992 				 */
993 				free(path);
994 				path = canonpath;
995 				canonpath = NULL;
996 			} else {
997 				char *child;
998 				/* Strip common prefix with repository path. */
999 				err = got_path_skip_common_ancestor(&child,
1000 				    repo_abspath, path);
1001 				if (err)
1002 					goto done;
1003 				free(path);
1004 				path = child;
1005 			}
1006 		} else {
1007 			/*
1008 			 * Matched unrelated on-disk path.
1009 			 * Treat input as repository-relative.
1010 			 */
1011 			free(path);
1012 			path = canonpath;
1013 			canonpath = NULL;
1014 		}
1015 	}
1016 
1017 	/* Make in-repository path absolute */
1018 	if (path[0] != '/') {
1019 		char *abspath;
1020 		if (asprintf(&abspath, "/%s", path) == -1) {
1021 			err = got_error_from_errno("asprintf");
1022 			goto done;
1023 		}
1024 		free(path);
1025 		path = abspath;
1026 	}
1027 
1028 done:
1029 	free(canonpath);
1030 	if (err)
1031 		free(path);
1032 	else
1033 		*in_repo_path = path;
1034 	return err;
1035 }
1036 
1037 static const struct got_error *
1038 cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
1039     const char *path_packidx)
1040 {
1041 	const struct got_error *err = NULL;
1042 	size_t i;
1043 
1044 	for (i = 0; i < repo->pack_cache_size; i++) {
1045 		if (repo->packidx_cache[i] == NULL)
1046 			break;
1047 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1048 		    path_packidx) == 0) {
1049 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1050 		}
1051 	}
1052 	if (i == repo->pack_cache_size) {
1053 		do {
1054 			i--;
1055 		} while (i > 0 && repo->pinned_packidx >= 0 &&
1056 		    i == repo->pinned_packidx);
1057 		err = got_packidx_close(repo->packidx_cache[i]);
1058 		if (err)
1059 			return err;
1060 	}
1061 
1062 	repo->packidx_cache[i] = packidx;
1063 
1064 	return NULL;
1065 }
1066 
1067 int
1068 got_repo_is_packidx_filename(const char *name, size_t len)
1069 {
1070 	if (len != GOT_PACKIDX_NAMELEN)
1071 		return 0;
1072 
1073 	if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
1074 		return 0;
1075 
1076 	if (strcmp(name + strlen(GOT_PACK_PREFIX) +
1077 	    SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
1078 		return 0;
1079 
1080 	return 1;
1081 }
1082 
1083 static struct got_packidx_bloom_filter *
1084 get_packidx_bloom_filter(struct got_repository *repo,
1085     const char *path, size_t path_len)
1086 {
1087 	struct got_packidx_bloom_filter key;
1088 
1089 	if (strlcpy(key.path, path, sizeof(key.path)) >= sizeof(key.path))
1090 		return NULL; /* XXX */
1091 	key.path_len = path_len;
1092 
1093 	return RB_FIND(got_packidx_bloom_filter_tree,
1094 	    &repo->packidx_bloom_filters, &key);
1095 }
1096 
1097 int
1098 got_repo_check_packidx_bloom_filter(struct got_repository *repo,
1099     const char *path_packidx, struct got_object_id *id)
1100 {
1101 	struct got_packidx_bloom_filter *bf;
1102 
1103 	bf = get_packidx_bloom_filter(repo, path_packidx, strlen(path_packidx));
1104 	if (bf)
1105 		return bloom_check(bf->bloom, id->sha1, sizeof(id->sha1));
1106 
1107 	/* No bloom filter means this pack index must be searched. */
1108 	return 1;
1109 }
1110 
1111 static const struct got_error *
1112 add_packidx_bloom_filter(struct got_repository *repo,
1113     struct got_packidx *packidx, const char *path_packidx)
1114 {
1115 	int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1116 	struct got_packidx_bloom_filter *bf;
1117 	size_t len;
1118 
1119 	/*
1120 	 * Don't use bloom filters for very large pack index files.
1121 	 * Large pack files will contain a relatively large fraction
1122 	 * of our objects so we will likely need to visit them anyway.
1123 	 * The more objects a pack file contains the higher the probability
1124 	 * of a false-positive match from the bloom filter. And reading
1125 	 * all object IDs from a large pack index file can be expensive.
1126 	 */
1127 	if (nobjects > 100000) /* cut-off at about 2MB, at 20 bytes per ID */
1128 		return NULL;
1129 
1130 	/* Do we already have a filter for this pack index? */
1131 	if (get_packidx_bloom_filter(repo, path_packidx,
1132 	    strlen(path_packidx)) != NULL)
1133 		return NULL;
1134 
1135 	bf = calloc(1, sizeof(*bf));
1136 	if (bf == NULL)
1137 		return got_error_from_errno("calloc");
1138 	bf->bloom = calloc(1, sizeof(*bf->bloom));
1139 	if (bf->bloom == NULL) {
1140 		free(bf);
1141 		return got_error_from_errno("calloc");
1142 	}
1143 
1144 	len = strlcpy(bf->path, path_packidx, sizeof(bf->path));
1145 	if (len >= sizeof(bf->path)) {
1146 		free(bf->bloom);
1147 		free(bf);
1148 		return got_error(GOT_ERR_NO_SPACE);
1149 	}
1150 	bf->path_len = len;
1151 
1152 	/* Minimum size supported by our bloom filter is 1000 entries. */
1153 	bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
1154 	for (i = 0; i < nobjects; i++) {
1155 		struct got_packidx_object_id *id;
1156 		id = &packidx->hdr.sorted_ids[i];
1157 		bloom_add(bf->bloom, id->sha1, sizeof(id->sha1));
1158 	}
1159 
1160 	RB_INSERT(got_packidx_bloom_filter_tree,
1161 	    &repo->packidx_bloom_filters, bf);
1162 	return NULL;
1163 }
1164 
1165 static void
1166 purge_packidx_paths(struct got_pathlist_head *packidx_paths)
1167 {
1168 	struct got_pathlist_entry *pe;
1169 
1170 	while (!TAILQ_EMPTY(packidx_paths)) {
1171 		pe = TAILQ_FIRST(packidx_paths);
1172 		TAILQ_REMOVE(packidx_paths, pe, entry);
1173 		free((char *)pe->path);
1174 		free(pe);
1175 	}
1176 }
1177 
1178 static const struct got_error *
1179 refresh_packidx_paths(struct got_repository *repo)
1180 {
1181 	const struct got_error *err = NULL;
1182 	char *objects_pack_dir = NULL;
1183 	struct stat sb;
1184 
1185 	objects_pack_dir = got_repo_get_path_objects_pack(repo);
1186 	if (objects_pack_dir == NULL)
1187 		return got_error_from_errno("got_repo_get_path_objects_pack");
1188 
1189 	if (stat(objects_pack_dir, &sb) == -1) {
1190 		if (errno != ENOENT) {
1191 			err = got_error_from_errno2("stat", objects_pack_dir);
1192 			goto done;
1193 		}
1194 	} else if (TAILQ_EMPTY(&repo->packidx_paths) ||
1195 	    sb.st_mtim.tv_sec != repo->pack_path_mtime.tv_sec ||
1196 	    sb.st_mtim.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1197 		purge_packidx_paths(&repo->packidx_paths);
1198 		err = got_repo_list_packidx(&repo->packidx_paths, repo);
1199 		if (err)
1200 			goto done;
1201 	}
1202 done:
1203 	free(objects_pack_dir);
1204 	return err;
1205 }
1206 
1207 const struct got_error *
1208 got_repo_search_packidx(struct got_packidx **packidx, int *idx,
1209     struct got_repository *repo, struct got_object_id *id)
1210 {
1211 	const struct got_error *err;
1212 	struct got_pathlist_entry *pe;
1213 	size_t i;
1214 
1215 	/* Search pack index cache. */
1216 	for (i = 0; i < repo->pack_cache_size; i++) {
1217 		if (repo->packidx_cache[i] == NULL)
1218 			break;
1219 		if (!got_repo_check_packidx_bloom_filter(repo,
1220 		    repo->packidx_cache[i]->path_packidx, id))
1221 			continue; /* object will not be found in this index */
1222 		*idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
1223 		if (*idx != -1) {
1224 			*packidx = repo->packidx_cache[i];
1225 			/*
1226 			 * Move this cache entry to the front. Repeatedly
1227 			 * searching a wrong pack index can be expensive.
1228 			 */
1229 			if (i > 0) {
1230 				memmove(&repo->packidx_cache[1],
1231 				    &repo->packidx_cache[0],
1232 				    i * sizeof(repo->packidx_cache[0]));
1233 				repo->packidx_cache[0] = *packidx;
1234 				if (repo->pinned_packidx >= 0 &&
1235 				    repo->pinned_packidx < i)
1236 					repo->pinned_packidx++;
1237 				else if (repo->pinned_packidx == i)
1238 					repo->pinned_packidx = 0;
1239 			}
1240 			return NULL;
1241 		}
1242 	}
1243 	/* No luck. Search the filesystem. */
1244 
1245 	err = refresh_packidx_paths(repo);
1246 	if (err)
1247 		return err;
1248 
1249 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1250 		const char *path_packidx = pe->path;
1251 		int is_cached = 0;
1252 
1253 		if (!got_repo_check_packidx_bloom_filter(repo,
1254 		    pe->path, id))
1255 			continue; /* object will not be found in this index */
1256 
1257 		for (i = 0; i < repo->pack_cache_size; i++) {
1258 			if (repo->packidx_cache[i] == NULL)
1259 				break;
1260 			if (strcmp(repo->packidx_cache[i]->path_packidx,
1261 			    path_packidx) == 0) {
1262 				is_cached = 1;
1263 				break;
1264 			}
1265 		}
1266 		if (is_cached)
1267 			continue; /* already searched */
1268 
1269 		err = got_packidx_open(packidx, got_repo_get_fd(repo),
1270 		    path_packidx, 0);
1271 		if (err)
1272 			goto done;
1273 
1274 		err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1275 		if (err)
1276 			goto done;
1277 
1278 		err = cache_packidx(repo, *packidx, path_packidx);
1279 		if (err)
1280 			goto done;
1281 
1282 		*idx = got_packidx_get_object_idx(*packidx, id);
1283 		if (*idx != -1) {
1284 			err = NULL; /* found the object */
1285 			goto done;
1286 		}
1287 	}
1288 
1289 	err = got_error_no_obj(id);
1290 done:
1291 	return err;
1292 }
1293 
1294 const struct got_error *
1295 got_repo_list_packidx(struct got_pathlist_head *packidx_paths,
1296     struct got_repository *repo)
1297 {
1298 	const struct got_error *err = NULL;
1299 	DIR *packdir = NULL;
1300 	struct dirent *dent;
1301 	char *path_packidx = NULL;
1302 	int packdir_fd;
1303 	struct stat sb;
1304 
1305 	packdir_fd = openat(got_repo_get_fd(repo),
1306 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1307 	if (packdir_fd == -1) {
1308 		return got_error_from_errno_fmt("openat: %s/%s",
1309 		    got_repo_get_path_git_dir(repo),
1310 		    GOT_OBJECTS_PACK_DIR);
1311 	}
1312 
1313 	packdir = fdopendir(packdir_fd);
1314 	if (packdir == NULL) {
1315 		err = got_error_from_errno("fdopendir");
1316 		goto done;
1317 	}
1318 
1319 	if (fstat(packdir_fd, &sb) == -1) {
1320 		err = got_error_from_errno("fstat");
1321 		goto done;
1322 	}
1323 	repo->pack_path_mtime.tv_sec = sb.st_mtim.tv_sec;
1324 	repo->pack_path_mtime.tv_nsec = sb.st_mtim.tv_nsec;
1325 
1326 	while ((dent = readdir(packdir)) != NULL) {
1327 		if (!got_repo_is_packidx_filename(dent->d_name,
1328 		    strlen(dent->d_name)))
1329 			continue;
1330 
1331 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
1332 		    dent->d_name) == -1) {
1333 			err = got_error_from_errno("asprintf");
1334 			path_packidx = NULL;
1335 			break;
1336 		}
1337 
1338 		err = got_pathlist_append(packidx_paths, path_packidx, NULL);
1339 		if (err)
1340 			break;
1341 	}
1342 done:
1343 	if (err)
1344 		free(path_packidx);
1345 	if (packdir && closedir(packdir) != 0 && err == NULL)
1346 		err = got_error_from_errno("closedir");
1347 	return err;
1348 }
1349 
1350 const struct got_error *
1351 got_repo_get_packidx(struct got_packidx **packidx, const char *path_packidx,
1352     struct got_repository *repo)
1353 {
1354 	const struct got_error *err;
1355 	size_t i;
1356 
1357 	*packidx = NULL;
1358 
1359 	/* Search pack index cache. */
1360 	for (i = 0; i < repo->pack_cache_size; i++) {
1361 		if (repo->packidx_cache[i] == NULL)
1362 			break;
1363 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1364 		    path_packidx) == 0) {
1365 			*packidx = repo->packidx_cache[i];
1366 			return NULL;
1367 		}
1368 	}
1369 	/* No luck. Search the filesystem. */
1370 
1371 	err = got_packidx_open(packidx, got_repo_get_fd(repo),
1372 	    path_packidx, 0);
1373 	if (err)
1374 		return err;
1375 
1376 	err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1377 	if (err)
1378 		goto done;
1379 
1380 	err = cache_packidx(repo, *packidx, path_packidx);
1381 done:
1382 	if (err) {
1383 		got_packidx_close(*packidx);
1384 		*packidx = NULL;
1385 	}
1386 	return err;
1387 }
1388 
1389 static const struct got_error *
1390 read_packfile_hdr(int fd, struct got_packidx *packidx)
1391 {
1392 	const struct got_error *err = NULL;
1393 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
1394 	struct got_packfile_hdr hdr;
1395 	ssize_t n;
1396 
1397 	n = read(fd, &hdr, sizeof(hdr));
1398 	if (n < 0)
1399 		return got_error_from_errno("read");
1400 	if (n != sizeof(hdr))
1401 		return got_error(GOT_ERR_BAD_PACKFILE);
1402 
1403 	if (be32toh(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
1404 	    be32toh(hdr.version) != GOT_PACKFILE_VERSION ||
1405 	    be32toh(hdr.nobjects) != totobj)
1406 		err = got_error(GOT_ERR_BAD_PACKFILE);
1407 
1408 	return err;
1409 }
1410 
1411 static const struct got_error *
1412 open_packfile(int *fd, struct got_repository *repo,
1413     const char *relpath, struct got_packidx *packidx)
1414 {
1415 	const struct got_error *err = NULL;
1416 
1417 	*fd = openat(got_repo_get_fd(repo), relpath,
1418 	    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1419 	if (*fd == -1)
1420 		return got_error_from_errno_fmt("openat: %s/%s",
1421 		    got_repo_get_path_git_dir(repo), relpath);
1422 
1423 	if (packidx) {
1424 		err = read_packfile_hdr(*fd, packidx);
1425 		if (err) {
1426 			close(*fd);
1427 			*fd = -1;
1428 		}
1429 	}
1430 
1431 	return err;
1432 }
1433 
1434 const struct got_error *
1435 got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
1436     const char *path_packfile, struct got_packidx *packidx)
1437 {
1438 	const struct got_error *err = NULL;
1439 	struct got_pack *pack = NULL;
1440 	struct stat sb;
1441 	size_t i;
1442 
1443 	if (packp)
1444 		*packp = NULL;
1445 
1446 	for (i = 0; i < repo->pack_cache_size; i++) {
1447 		pack = &repo->packs[i];
1448 		if (pack->path_packfile == NULL)
1449 			break;
1450 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1451 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1452 	}
1453 
1454 	if (i == repo->pack_cache_size) {
1455 		struct got_pack tmp;
1456 		do {
1457 			i--;
1458 		} while (i > 0 && repo->pinned_pack >= 0 &&
1459 		    i == repo->pinned_pack);
1460 		err = got_pack_close(&repo->packs[i]);
1461 		if (err)
1462 			return err;
1463 		if (ftruncate(repo->packs[i].basefd, 0L) == -1)
1464 			return got_error_from_errno("ftruncate");
1465 		if (ftruncate(repo->packs[i].accumfd, 0L) == -1)
1466 			return got_error_from_errno("ftruncate");
1467 		memcpy(&tmp, &repo->packs[i], sizeof(tmp));
1468 		memcpy(&repo->packs[i], &repo->packs[0],
1469 		    sizeof(repo->packs[i]));
1470 		memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0]));
1471 		if (repo->pinned_pack == 0)
1472 			repo->pinned_pack = i;
1473 		else if (repo->pinned_pack == i)
1474 			repo->pinned_pack = 0;
1475 		i = 0;
1476 	}
1477 
1478 	pack = &repo->packs[i];
1479 
1480 	pack->path_packfile = strdup(path_packfile);
1481 	if (pack->path_packfile == NULL) {
1482 		err = got_error_from_errno("strdup");
1483 		goto done;
1484 	}
1485 
1486 	err = open_packfile(&pack->fd, repo, path_packfile, packidx);
1487 	if (err)
1488 		goto done;
1489 
1490 	if (fstat(pack->fd, &sb) != 0) {
1491 		err = got_error_from_errno("fstat");
1492 		goto done;
1493 	}
1494 	pack->filesize = sb.st_size;
1495 
1496 	pack->privsep_child = NULL;
1497 
1498 	err = got_delta_cache_alloc(&pack->delta_cache);
1499 	if (err)
1500 		goto done;
1501 
1502 #ifndef GOT_PACK_NO_MMAP
1503 	if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1504 		pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1505 		    pack->fd, 0);
1506 		if (pack->map == MAP_FAILED) {
1507 			if (errno != ENOMEM) {
1508 				err = got_error_from_errno("mmap");
1509 				goto done;
1510 			}
1511 			pack->map = NULL; /* fall back to read(2) */
1512 		}
1513 	}
1514 #endif
1515 done:
1516 	if (err) {
1517 		if (pack)
1518 			got_pack_close(pack);
1519 	} else if (packp)
1520 		*packp = pack;
1521 	return err;
1522 }
1523 
1524 struct got_pack *
1525 got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
1526 {
1527 	struct got_pack *pack = NULL;
1528 	size_t i;
1529 
1530 	for (i = 0; i < repo->pack_cache_size; i++) {
1531 		pack = &repo->packs[i];
1532 		if (pack->path_packfile == NULL)
1533 			break;
1534 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1535 			return pack;
1536 	}
1537 
1538 	return NULL;
1539 }
1540 
1541 const struct got_error *
1542 got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx,
1543     struct got_pack *pack)
1544 {
1545 	size_t i;
1546 	int pinned_pack = -1, pinned_packidx = -1;
1547 
1548 	for (i = 0; i < repo->pack_cache_size; i++) {
1549 		if (repo->packidx_cache[i] &&
1550 		    strcmp(repo->packidx_cache[i]->path_packidx,
1551 		    packidx->path_packidx) == 0)
1552 			pinned_packidx = i;
1553 		if (repo->packs[i].path_packfile &&
1554 		    strcmp(repo->packs[i].path_packfile,
1555 		    pack->path_packfile) == 0)
1556 			pinned_pack = i;
1557 	}
1558 
1559 	if (pinned_packidx == -1 || pinned_pack == -1)
1560 		return got_error(GOT_ERR_PIN_PACK);
1561 
1562 	repo->pinned_pack = pinned_pack;
1563 	repo->pinned_packidx = pinned_packidx;
1564 	if (repo->packs[pinned_pack].privsep_child)
1565 		repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid;
1566 	return NULL;
1567 }
1568 
1569 struct got_pack *
1570 got_repo_get_pinned_pack(struct got_repository *repo)
1571 {
1572 	if (repo->pinned_pack >= 0 &&
1573 	    repo->pinned_pack < repo->pack_cache_size)
1574 		return &repo->packs[repo->pinned_pack];
1575 
1576 	return NULL;
1577 }
1578 
1579 void
1580 got_repo_unpin_pack(struct got_repository *repo)
1581 {
1582 	repo->pinned_packidx = -1;
1583 	repo->pinned_pack = -1;
1584 	repo->pinned_pid = 0;
1585 }
1586 
1587 const struct got_error *
1588 got_repo_init(const char *repo_path, const char *head_name)
1589 {
1590 	const struct got_error *err = NULL;
1591 	const char *dirnames[] = {
1592 		GOT_OBJECTS_DIR,
1593 		GOT_OBJECTS_PACK_DIR,
1594 		GOT_REFS_DIR,
1595 	};
1596 	const char *description_str = "Unnamed repository; "
1597 	    "edit this file 'description' to name the repository.";
1598 	const char *headref = "ref: refs/heads/";
1599 	const char *gitconfig_str = "[core]\n"
1600 	    "\trepositoryformatversion = 0\n"
1601 	    "\tfilemode = true\n"
1602 	    "\tbare = true\n";
1603 	char *headref_str, *path;
1604 	size_t i;
1605 
1606 	if (!got_path_dir_is_empty(repo_path))
1607 		return got_error(GOT_ERR_DIR_NOT_EMPTY);
1608 
1609 	for (i = 0; i < nitems(dirnames); i++) {
1610 		if (asprintf(&path, "%s/%s", repo_path, dirnames[i]) == -1) {
1611 			return got_error_from_errno("asprintf");
1612 		}
1613 		err = got_path_mkdir(path);
1614 		free(path);
1615 		if (err)
1616 			return err;
1617 	}
1618 
1619 	if (asprintf(&path, "%s/%s", repo_path, "description") == -1)
1620 		return got_error_from_errno("asprintf");
1621 	err = got_path_create_file(path, description_str);
1622 	free(path);
1623 	if (err)
1624 		return err;
1625 
1626 	if (asprintf(&path, "%s/%s", repo_path, GOT_HEAD_FILE) == -1)
1627 		return got_error_from_errno("asprintf");
1628 	if (asprintf(&headref_str, "%s%s", headref,
1629 	    head_name ? head_name : "main") == -1) {
1630 		free(path);
1631 		return got_error_from_errno("asprintf");
1632 	}
1633 	err = got_path_create_file(path, headref_str);
1634 	free(headref_str);
1635 	free(path);
1636 	if (err)
1637 		return err;
1638 
1639 	if (asprintf(&path, "%s/%s", repo_path, "config") == -1)
1640 		return got_error_from_errno("asprintf");
1641 	err = got_path_create_file(path, gitconfig_str);
1642 	free(path);
1643 	if (err)
1644 		return err;
1645 
1646 	return NULL;
1647 }
1648 
1649 static const struct got_error *
1650 match_packed_object(struct got_object_id **unique_id,
1651     struct got_repository *repo, const char *id_str_prefix, int obj_type)
1652 {
1653 	const struct got_error *err = NULL;
1654 	struct got_object_id_queue matched_ids;
1655 	struct got_pathlist_entry *pe;
1656 
1657 	STAILQ_INIT(&matched_ids);
1658 
1659 	err = refresh_packidx_paths(repo);
1660 	if (err)
1661 		return err;
1662 
1663 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1664 		const char *path_packidx = pe->path;
1665 		struct got_packidx *packidx;
1666 		struct got_object_qid *qid;
1667 
1668 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
1669 		    path_packidx, 0);
1670 		if (err)
1671 			break;
1672 
1673 		err = got_packidx_match_id_str_prefix(&matched_ids,
1674 		    packidx, id_str_prefix);
1675 		if (err) {
1676 			got_packidx_close(packidx);
1677 			break;
1678 		}
1679 		err = got_packidx_close(packidx);
1680 		if (err)
1681 			break;
1682 
1683 		STAILQ_FOREACH(qid, &matched_ids, entry) {
1684 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1685 				int matched_type;
1686 				err = got_object_get_type(&matched_type, repo,
1687 				    &qid->id);
1688 				if (err)
1689 					goto done;
1690 				if (matched_type != obj_type)
1691 					continue;
1692 			}
1693 			if (*unique_id == NULL) {
1694 				*unique_id = got_object_id_dup(&qid->id);
1695 				if (*unique_id == NULL) {
1696 					err = got_error_from_errno("malloc");
1697 					goto done;
1698 				}
1699 			} else {
1700 				if (got_object_id_cmp(*unique_id,
1701 				    &qid->id) == 0)
1702 					continue; /* packed multiple times */
1703 				err = got_error(GOT_ERR_AMBIGUOUS_ID);
1704 				goto done;
1705 			}
1706 		}
1707 	}
1708 done:
1709 	got_object_id_queue_free(&matched_ids);
1710 	if (err) {
1711 		free(*unique_id);
1712 		*unique_id = NULL;
1713 	}
1714 	return err;
1715 }
1716 
1717 static const struct got_error *
1718 match_loose_object(struct got_object_id **unique_id, const char *path_objects,
1719     const char *object_dir, const char *id_str_prefix, int obj_type,
1720     struct got_repository *repo)
1721 {
1722 	const struct got_error *err = NULL;
1723 	char *path, *id_str = NULL;
1724 	DIR *dir = NULL;
1725 	struct dirent *dent;
1726 	struct got_object_id id;
1727 
1728 	if (asprintf(&path, "%s/%s", path_objects, object_dir) == -1) {
1729 		err = got_error_from_errno("asprintf");
1730 		goto done;
1731 	}
1732 
1733 	dir = opendir(path);
1734 	if (dir == NULL) {
1735 		if (errno == ENOENT) {
1736 			err = NULL;
1737 			goto done;
1738 		}
1739 		err = got_error_from_errno2("opendir", path);
1740 		goto done;
1741 	}
1742 	while ((dent = readdir(dir)) != NULL) {
1743 		int cmp;
1744 
1745 		free(id_str);
1746 		id_str = NULL;
1747 
1748 		if (strcmp(dent->d_name, ".") == 0 ||
1749 		    strcmp(dent->d_name, "..") == 0)
1750 			continue;
1751 
1752 		if (asprintf(&id_str, "%s%s", object_dir, dent->d_name) == -1) {
1753 			err = got_error_from_errno("asprintf");
1754 			goto done;
1755 		}
1756 
1757 		if (!got_parse_object_id(&id, id_str, repo->algo))
1758 			continue;
1759 
1760 		/*
1761 		 * Directory entries do not necessarily appear in
1762 		 * sorted order, so we must iterate over all of them.
1763 		 */
1764 		cmp = strncmp(id_str, id_str_prefix, strlen(id_str_prefix));
1765 		if (cmp != 0)
1766 			continue;
1767 
1768 		if (*unique_id == NULL) {
1769 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1770 				int matched_type;
1771 				err = got_object_get_type(&matched_type, repo,
1772 				    &id);
1773 				if (err)
1774 					goto done;
1775 				if (matched_type != obj_type)
1776 					continue;
1777 			}
1778 			*unique_id = got_object_id_dup(&id);
1779 			if (*unique_id == NULL) {
1780 				err = got_error_from_errno("got_object_id_dup");
1781 				goto done;
1782 			}
1783 		} else {
1784 			if (got_object_id_cmp(*unique_id, &id) == 0)
1785 				continue; /* both packed and loose */
1786 			err = got_error(GOT_ERR_AMBIGUOUS_ID);
1787 			goto done;
1788 		}
1789 	}
1790 done:
1791 	if (dir && closedir(dir) != 0 && err == NULL)
1792 		err = got_error_from_errno("closedir");
1793 	if (err) {
1794 		free(*unique_id);
1795 		*unique_id = NULL;
1796 	}
1797 	free(id_str);
1798 	free(path);
1799 	return err;
1800 }
1801 
1802 const struct got_error *
1803 got_repo_match_object_id_prefix(struct got_object_id **id,
1804     const char *id_str_prefix, int obj_type, struct got_repository *repo)
1805 {
1806 	const struct got_error *err = NULL;
1807 	char *path_objects = NULL, *object_dir = NULL;
1808 	size_t len;
1809 	int i;
1810 
1811 	*id = NULL;
1812 
1813 	path_objects = got_repo_get_path_objects(repo);
1814 
1815 	len = strlen(id_str_prefix);
1816 	if (len > SHA1_DIGEST_STRING_LENGTH - 1) {
1817 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1818 		goto done;
1819 	}
1820 
1821 	for (i = 0; i < len; i++) {
1822 		if (isxdigit((unsigned char)id_str_prefix[i]))
1823 			continue;
1824 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1825 		goto done;
1826 	}
1827 
1828 	if (len >= 2) {
1829 		err = match_packed_object(id, repo, id_str_prefix, obj_type);
1830 		if (err)
1831 			goto done;
1832 		object_dir = strndup(id_str_prefix, 2);
1833 		if (object_dir == NULL) {
1834 			err = got_error_from_errno("strdup");
1835 			goto done;
1836 		}
1837 		err = match_loose_object(id, path_objects, object_dir,
1838 		    id_str_prefix, obj_type, repo);
1839 	} else if (len == 1) {
1840 		int i;
1841 		for (i = 0; i < 0xf; i++) {
1842 			if (asprintf(&object_dir, "%s%.1x", id_str_prefix, i)
1843 			    == -1) {
1844 				err = got_error_from_errno("asprintf");
1845 				goto done;
1846 			}
1847 			err = match_packed_object(id, repo, object_dir,
1848 			    obj_type);
1849 			if (err)
1850 				goto done;
1851 			err = match_loose_object(id, path_objects, object_dir,
1852 			    id_str_prefix, obj_type, repo);
1853 			if (err)
1854 				goto done;
1855 		}
1856 	} else {
1857 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1858 		goto done;
1859 	}
1860 done:
1861 	free(path_objects);
1862 	free(object_dir);
1863 	if (err) {
1864 		free(*id);
1865 		*id = NULL;
1866 	} else if (*id == NULL) {
1867 		switch (obj_type) {
1868 		case GOT_OBJ_TYPE_BLOB:
1869 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1870 			    GOT_OBJ_LABEL_BLOB, id_str_prefix);
1871 			break;
1872 		case GOT_OBJ_TYPE_TREE:
1873 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1874 			    GOT_OBJ_LABEL_TREE, id_str_prefix);
1875 			break;
1876 		case GOT_OBJ_TYPE_COMMIT:
1877 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1878 			    GOT_OBJ_LABEL_COMMIT, id_str_prefix);
1879 			break;
1880 		case GOT_OBJ_TYPE_TAG:
1881 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1882 			    GOT_OBJ_LABEL_TAG, id_str_prefix);
1883 			break;
1884 		default:
1885 			err = got_error_path(id_str_prefix, GOT_ERR_NO_OBJ);
1886 			break;
1887 		}
1888 	}
1889 
1890 	return err;
1891 }
1892 
1893 const struct got_error *
1894 got_repo_match_object_id(struct got_object_id **id, char **label,
1895     const char *id_str, int obj_type, struct got_reflist_head *refs,
1896     struct got_repository *repo)
1897 {
1898 	const struct got_error *err;
1899 	struct got_tag_object *tag;
1900 	struct got_reference *ref = NULL;
1901 
1902 	*id = NULL;
1903 	if (label)
1904 		*label = NULL;
1905 
1906 	if (refs) {
1907 		err = got_repo_object_match_tag(&tag, id_str, obj_type,
1908 		    refs, repo);
1909 		if (err == NULL) {
1910 			*id = got_object_id_dup(
1911 			    got_object_tag_get_object_id(tag));
1912 			if (*id == NULL)
1913 				err = got_error_from_errno("got_object_id_dup");
1914 			else if (label && asprintf(label, "refs/tags/%s",
1915 			    got_object_tag_get_name(tag)) == -1) {
1916 				err = got_error_from_errno("asprintf");
1917 				free(*id);
1918 				*id = NULL;
1919 			}
1920 			got_object_tag_close(tag);
1921 			return err;
1922 		} else if (err->code != GOT_ERR_OBJ_TYPE &&
1923 		    err->code != GOT_ERR_NO_OBJ)
1924 			return err;
1925 	}
1926 
1927 	err = got_ref_open(&ref, repo, id_str, 0);
1928 	if (err == NULL) {
1929 		err = got_ref_resolve(id, repo, ref);
1930 		if (err)
1931 			goto done;
1932 		if (label) {
1933 			*label = strdup(got_ref_get_name(ref));
1934 			if (*label == NULL) {
1935 				err = got_error_from_errno("strdup");
1936 				goto done;
1937 			}
1938 		}
1939 	} else {
1940 		if (err->code != GOT_ERR_NOT_REF &&
1941 		    err->code != GOT_ERR_BAD_REF_NAME)
1942 			goto done;
1943 		err = got_repo_match_object_id_prefix(id, id_str,
1944 		    obj_type, repo);
1945 		if (err) {
1946 			if (err->code == GOT_ERR_BAD_OBJ_ID_STR)
1947 				err = got_error_not_ref(id_str);
1948 			goto done;
1949 		}
1950 		if (label) {
1951 			err = got_object_id_str(label, *id);
1952 			if (*label == NULL) {
1953 				err = got_error_from_errno("strdup");
1954 				goto done;
1955 			}
1956 		}
1957 	}
1958 done:
1959 	if (ref)
1960 		got_ref_close(ref);
1961 	return err;
1962 }
1963 
1964 const struct got_error *
1965 got_repo_object_match_tag(struct got_tag_object **tag, const char *name,
1966     int obj_type, struct got_reflist_head *refs, struct got_repository *repo)
1967 {
1968 	const struct got_error *err = NULL;
1969 	struct got_reflist_entry *re;
1970 	struct got_object_id *tag_id;
1971 	int name_is_absolute = (strncmp(name, "refs/", 5) == 0);
1972 
1973 	*tag = NULL;
1974 
1975 	TAILQ_FOREACH(re, refs, entry) {
1976 		const char *refname;
1977 		refname = got_ref_get_name(re->ref);
1978 		if (got_ref_is_symbolic(re->ref))
1979 			continue;
1980 		if (strncmp(refname, "refs/tags/", 10) != 0)
1981 			continue;
1982 		if (!name_is_absolute)
1983 			refname += strlen("refs/tags/");
1984 		if (strcmp(refname, name) != 0)
1985 			continue;
1986 		err = got_ref_resolve(&tag_id, repo, re->ref);
1987 		if (err)
1988 			break;
1989 		err = got_object_open_as_tag(tag, repo, tag_id);
1990 		free(tag_id);
1991 		if (err)
1992 			break;
1993 		if (obj_type == GOT_OBJ_TYPE_ANY ||
1994 		    got_object_tag_get_object_type(*tag) == obj_type)
1995 			break;
1996 		got_object_tag_close(*tag);
1997 		*tag = NULL;
1998 	}
1999 
2000 	if (err == NULL && *tag == NULL)
2001 		err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2002 		    GOT_OBJ_LABEL_TAG, name);
2003 	return err;
2004 }
2005 
2006 static const struct got_error *
2007 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
2008     const char *name, mode_t mode, struct got_object_id *blob_id)
2009 {
2010 	const struct got_error *err = NULL;
2011 
2012 	 *new_te = NULL;
2013 
2014 	*new_te = calloc(1, sizeof(**new_te));
2015 	if (*new_te == NULL)
2016 		return got_error_from_errno("calloc");
2017 
2018 	if (strlcpy((*new_te)->name, name, sizeof((*new_te)->name)) >=
2019 	    sizeof((*new_te)->name)) {
2020 		err = got_error(GOT_ERR_NO_SPACE);
2021 		goto done;
2022 	}
2023 
2024 	if (S_ISLNK(mode)) {
2025 		(*new_te)->mode = S_IFLNK;
2026 	} else {
2027 		(*new_te)->mode = S_IFREG;
2028 		(*new_te)->mode |= (mode & (S_IRWXU | S_IRWXG | S_IRWXO));
2029 	}
2030 	memcpy(&(*new_te)->id, blob_id, sizeof((*new_te)->id));
2031 done:
2032 	if (err && *new_te) {
2033 		free(*new_te);
2034 		*new_te = NULL;
2035 	}
2036 	return err;
2037 }
2038 
2039 static const struct got_error *
2040 import_file(struct got_tree_entry **new_te, struct dirent *de,
2041     const char *path, struct got_repository *repo)
2042 {
2043 	const struct got_error *err;
2044 	struct got_object_id *blob_id = NULL;
2045 	char *filepath;
2046 	struct stat sb;
2047 
2048 	if (asprintf(&filepath, "%s%s%s", path,
2049 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2050 		return got_error_from_errno("asprintf");
2051 
2052 	if (lstat(filepath, &sb) != 0) {
2053 		err = got_error_from_errno2("lstat", path);
2054 		goto done;
2055 	}
2056 
2057 	err = got_object_blob_create(&blob_id, filepath, repo);
2058 	if (err)
2059 		goto done;
2060 
2061 	err = alloc_added_blob_tree_entry(new_te, de->d_name, sb.st_mode,
2062 	    blob_id);
2063 done:
2064 	free(filepath);
2065 	if (err)
2066 		free(blob_id);
2067 	return err;
2068 }
2069 
2070 static const struct got_error *
2071 insert_tree_entry(struct got_tree_entry *new_te,
2072     struct got_pathlist_head *paths)
2073 {
2074 	const struct got_error *err = NULL;
2075 	struct got_pathlist_entry *new_pe;
2076 
2077 	err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
2078 	if (err)
2079 		return err;
2080 	if (new_pe == NULL)
2081 		return got_error(GOT_ERR_TREE_DUP_ENTRY);
2082 	return NULL;
2083 }
2084 
2085 static const struct got_error *write_tree(struct got_object_id **,
2086     const char *, struct got_pathlist_head *, struct got_repository *,
2087     got_repo_import_cb progress_cb, void *progress_arg);
2088 
2089 static const struct got_error *
2090 import_subdir(struct got_tree_entry **new_te, struct dirent *de,
2091     const char *path, struct got_pathlist_head *ignores,
2092     struct got_repository *repo,
2093     got_repo_import_cb progress_cb, void *progress_arg)
2094 {
2095 	const struct got_error *err;
2096 	struct got_object_id *id = NULL;
2097 	char *subdirpath;
2098 
2099 	if (asprintf(&subdirpath, "%s%s%s", path,
2100 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2101 		return got_error_from_errno("asprintf");
2102 
2103 	(*new_te) = calloc(1, sizeof(**new_te));
2104 	if (*new_te == NULL)
2105 		return got_error_from_errno("calloc");
2106 	(*new_te)->mode = S_IFDIR;
2107 	if (strlcpy((*new_te)->name, de->d_name, sizeof((*new_te)->name)) >=
2108 	    sizeof((*new_te)->name)) {
2109 		err = got_error(GOT_ERR_NO_SPACE);
2110 		goto done;
2111 	}
2112 	err = write_tree(&id, subdirpath, ignores,  repo,
2113 	    progress_cb, progress_arg);
2114 	if (err)
2115 		goto done;
2116 	memcpy(&(*new_te)->id, id, sizeof((*new_te)->id));
2117 
2118 done:
2119 	free(id);
2120 	free(subdirpath);
2121 	if (err) {
2122 		free(*new_te);
2123 		*new_te = NULL;
2124 	}
2125 	return err;
2126 }
2127 
2128 static const struct got_error *
2129 write_tree(struct got_object_id **new_tree_id, const char *path_dir,
2130     struct got_pathlist_head *ignores, struct got_repository *repo,
2131     got_repo_import_cb progress_cb, void *progress_arg)
2132 {
2133 	const struct got_error *err = NULL;
2134 	DIR *dir;
2135 	struct dirent *de;
2136 	int nentries;
2137 	struct got_tree_entry *new_te = NULL;
2138 	struct got_pathlist_head paths;
2139 	struct got_pathlist_entry *pe;
2140 
2141 	*new_tree_id = NULL;
2142 
2143 	TAILQ_INIT(&paths);
2144 
2145 	dir = opendir(path_dir);
2146 	if (dir == NULL) {
2147 		err = got_error_from_errno2("opendir", path_dir);
2148 		goto done;
2149 	}
2150 
2151 	nentries = 0;
2152 	while ((de = readdir(dir)) != NULL) {
2153 		int ignore = 0;
2154 		int type;
2155 
2156 		if (strcmp(de->d_name, ".") == 0 ||
2157 		    strcmp(de->d_name, "..") == 0)
2158 			continue;
2159 
2160 		err = got_path_dirent_type(&type, path_dir, de);
2161 		if (err)
2162 			goto done;
2163 
2164 		TAILQ_FOREACH(pe, ignores, entry) {
2165 			if (type == DT_DIR && pe->path_len > 0 &&
2166 			    pe->path[pe->path_len - 1] == '/') {
2167 				char stripped[PATH_MAX];
2168 
2169 				if (strlcpy(stripped, pe->path,
2170 				    sizeof(stripped)) >= sizeof(stripped)) {
2171 					err = got_error(GOT_ERR_NO_SPACE);
2172 					goto done;
2173 				}
2174 				got_path_strip_trailing_slashes(stripped);
2175 				if (fnmatch(stripped, de->d_name, 0) == 0) {
2176 					ignore = 1;
2177 					break;
2178 				}
2179 			} else if (fnmatch(pe->path, de->d_name, 0) == 0) {
2180 				ignore = 1;
2181 				break;
2182 			}
2183 		}
2184 		if (ignore)
2185 			continue;
2186 
2187 		if (type == DT_DIR) {
2188 			err = import_subdir(&new_te, de, path_dir,
2189 			    ignores, repo, progress_cb, progress_arg);
2190 			if (err) {
2191 				if (err->code != GOT_ERR_NO_TREE_ENTRY)
2192 					goto done;
2193 				err = NULL;
2194 				continue;
2195 			}
2196 		} else if (type == DT_REG || type == DT_LNK) {
2197 			err = import_file(&new_te, de, path_dir, repo);
2198 			if (err)
2199 				goto done;
2200 		} else
2201 			continue;
2202 
2203 		err = insert_tree_entry(new_te, &paths);
2204 		if (err)
2205 			goto done;
2206 		nentries++;
2207 	}
2208 
2209 	if (TAILQ_EMPTY(&paths)) {
2210 		err = got_error_msg(GOT_ERR_NO_TREE_ENTRY,
2211 		    "cannot create tree without any entries");
2212 		goto done;
2213 	}
2214 
2215 	TAILQ_FOREACH(pe, &paths, entry) {
2216 		struct got_tree_entry *te = pe->data;
2217 		char *path;
2218 		if (!S_ISREG(te->mode) && !S_ISLNK(te->mode))
2219 			continue;
2220 		if (asprintf(&path, "%s/%s", path_dir, pe->path) == -1) {
2221 			err = got_error_from_errno("asprintf");
2222 			goto done;
2223 		}
2224 		err = (*progress_cb)(progress_arg, path);
2225 		free(path);
2226 		if (err)
2227 			goto done;
2228 	}
2229 
2230 	err = got_object_tree_create(new_tree_id, &paths, nentries, repo);
2231 done:
2232 	if (dir)
2233 		closedir(dir);
2234 	got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2235 	return err;
2236 }
2237 
2238 const struct got_error *
2239 got_repo_import(struct got_object_id **new_commit_id, const char *path_dir,
2240     const char *logmsg, const char *author, struct got_pathlist_head *ignores,
2241     struct got_repository *repo, got_repo_import_cb progress_cb,
2242     void *progress_arg)
2243 {
2244 	const struct got_error *err;
2245 	struct got_object_id *new_tree_id;
2246 
2247 	err = write_tree(&new_tree_id, path_dir, ignores, repo,
2248 	    progress_cb, progress_arg);
2249 	if (err)
2250 		return err;
2251 
2252 	err = got_object_commit_create(new_commit_id, new_tree_id, NULL, 0,
2253 	    author, time(NULL), author, time(NULL), logmsg, repo);
2254 	free(new_tree_id);
2255 	return err;
2256 }
2257 
2258 const struct got_error *
2259 got_repo_get_loose_object_info(int *nobjects, off_t *ondisk_size,
2260     struct got_repository *repo)
2261 {
2262 	const struct got_error *err = NULL;
2263 	char *path_objects = NULL, *path = NULL;
2264 	DIR *dir = NULL;
2265 	struct got_object_id id;
2266 	int i;
2267 
2268 	*nobjects = 0;
2269 	*ondisk_size = 0;
2270 
2271 	path_objects = got_repo_get_path_objects(repo);
2272 	if (path_objects == NULL)
2273 		return got_error_from_errno("got_repo_get_path_objects");
2274 
2275 	for (i = 0; i <= 0xff; i++) {
2276 		struct dirent *dent;
2277 
2278 		if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
2279 			err = got_error_from_errno("asprintf");
2280 			break;
2281 		}
2282 
2283 		dir = opendir(path);
2284 		if (dir == NULL) {
2285 			if (errno == ENOENT) {
2286 				err = NULL;
2287 				continue;
2288 			}
2289 			err = got_error_from_errno2("opendir", path);
2290 			break;
2291 		}
2292 
2293 		while ((dent = readdir(dir)) != NULL) {
2294 			char *id_str;
2295 			int fd;
2296 			struct stat sb;
2297 
2298 			if (strcmp(dent->d_name, ".") == 0 ||
2299 			    strcmp(dent->d_name, "..") == 0)
2300 				continue;
2301 
2302 			if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
2303 				err = got_error_from_errno("asprintf");
2304 				goto done;
2305 			}
2306 
2307 			if (!got_parse_object_id(&id, id_str, repo->algo)) {
2308 				free(id_str);
2309 				continue;
2310 			}
2311 			free(id_str);
2312 
2313 			err = got_object_open_loose_fd(&fd, &id, repo);
2314 			if (err)
2315 				goto done;
2316 
2317 			if (fstat(fd, &sb) == -1) {
2318 				err = got_error_from_errno("fstat");
2319 				close(fd);
2320 				goto done;
2321 			}
2322 			(*nobjects)++;
2323 			(*ondisk_size) += sb.st_size;
2324 
2325 			if (close(fd) == -1) {
2326 				err = got_error_from_errno("close");
2327 				goto done;
2328 			}
2329 		}
2330 
2331 		if (closedir(dir) != 0) {
2332 			err = got_error_from_errno("closedir");
2333 			goto done;
2334 		}
2335 		dir = NULL;
2336 
2337 		free(path);
2338 		path = NULL;
2339 	}
2340 done:
2341 	if (dir && closedir(dir) != 0 && err == NULL)
2342 		err = got_error_from_errno("closedir");
2343 
2344 	if (err) {
2345 		*nobjects = 0;
2346 		*ondisk_size = 0;
2347 	}
2348 	free(path_objects);
2349 	free(path);
2350 	return err;
2351 }
2352 
2353 const struct got_error *
2354 got_repo_get_packfile_info(int *npackfiles, int *nobjects,
2355     off_t *total_packsize, struct got_repository *repo)
2356 {
2357 	const struct got_error *err = NULL;
2358 	DIR *packdir = NULL;
2359 	struct dirent *dent;
2360 	struct got_packidx *packidx = NULL;
2361 	char *path_packidx;
2362 	char *path_packfile;
2363 	int packdir_fd;
2364 	struct stat sb;
2365 
2366 	*npackfiles = 0;
2367 	*nobjects = 0;
2368 	*total_packsize = 0;
2369 
2370 	packdir_fd = openat(got_repo_get_fd(repo),
2371 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
2372 	if (packdir_fd == -1) {
2373 		return got_error_from_errno_fmt("openat: %s/%s",
2374 		    got_repo_get_path_git_dir(repo),
2375 		    GOT_OBJECTS_PACK_DIR);
2376 	}
2377 
2378 	packdir = fdopendir(packdir_fd);
2379 	if (packdir == NULL) {
2380 		err = got_error_from_errno("fdopendir");
2381 		goto done;
2382 	}
2383 
2384 	while ((dent = readdir(packdir)) != NULL) {
2385 		if (!got_repo_is_packidx_filename(dent->d_name,
2386 		    strlen(dent->d_name)))
2387 			continue;
2388 
2389 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
2390 		    dent->d_name) == -1) {
2391 			err = got_error_from_errno("asprintf");
2392 			goto done;
2393 		}
2394 
2395 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
2396 		    path_packidx, 0);
2397 		free(path_packidx);
2398 		if (err)
2399 			goto done;
2400 
2401 		if (fstat(packidx->fd, &sb) == -1)
2402 			goto done;
2403 		*total_packsize += sb.st_size;
2404 
2405 		err = got_packidx_get_packfile_path(&path_packfile,
2406 		    packidx->path_packidx);
2407 		if (err)
2408 			goto done;
2409 
2410 		if (fstatat(got_repo_get_fd(repo), path_packfile, &sb,
2411 		    0) == -1) {
2412 			free(path_packfile);
2413 			goto done;
2414 		}
2415 		free(path_packfile);
2416 		*total_packsize += sb.st_size;
2417 
2418 		*nobjects += be32toh(packidx->hdr.fanout_table[0xff]);
2419 
2420 		(*npackfiles)++;
2421 
2422 		got_packidx_close(packidx);
2423 		packidx = NULL;
2424 	}
2425 done:
2426 	if (packidx)
2427 		got_packidx_close(packidx);
2428 	if (packdir && closedir(packdir) != 0 && err == NULL)
2429 		err = got_error_from_errno("closedir");
2430 	if (err) {
2431 		*npackfiles = 0;
2432 		*nobjects = 0;
2433 		*total_packsize = 0;
2434 	}
2435 	return err;
2436 }
2437 
2438 RB_GENERATE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
2439     got_packidx_bloom_filter_cmp);