1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/mman.h>
23 #include <sys/resource.h>
24 
25 #include <ctype.h>
26 #include <fcntl.h>
27 #include <fnmatch.h>
28 #include <limits.h>
29 #include <dirent.h>
30 #include <stdlib.h>
31 #include <stdio.h>
32 #include <sha1.h>
33 #include <sha2.h>
34 #include <string.h>
35 #include <time.h>
36 #include <unistd.h>
37 #include <zlib.h>
38 #include <errno.h>
39 #include <libgen.h>
40 #include <stdint.h>
41 
42 #include "bloom.h"
43 
44 #include "got_error.h"
45 #include "got_reference.h"
46 #include "got_repository.h"
47 #include "got_path.h"
48 #include "got_cancel.h"
49 #include "got_object.h"
50 #include "got_opentemp.h"
51 
52 #include "got_lib_delta.h"
53 #include "got_lib_delta_cache.h"
54 #include "got_lib_inflate.h"
55 #include "got_lib_object.h"
56 #include "got_lib_object_parse.h"
57 #include "got_lib_object_create.h"
58 #include "got_lib_pack.h"
59 #include "got_lib_privsep.h"
60 #include "got_lib_sha1.h"
61 #include "got_lib_object_cache.h"
62 #include "got_lib_repository.h"
63 #include "got_lib_gotconfig.h"
64 
65 #ifndef nitems
66 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
67 #endif
68 
69 #define GOT_PACK_NUM_TEMPFILES		GOT_PACK_CACHE_SIZE * 2
70 
71 RB_PROTOTYPE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
72     got_packidx_bloom_filter_cmp);
73 
74 static inline int
75 is_boolean_val(const char *val)
76 {
77 	return (strcasecmp(val, "true") == 0 ||
78 	    strcasecmp(val, "false") == 0 ||
79 	    strcasecmp(val, "on") == 0 ||
80     	    strcasecmp(val, "off") == 0 ||
81 	    strcasecmp(val, "yes") == 0 ||
82     	    strcasecmp(val, "no") == 0 ||
83 	    strcasecmp(val, "1") == 0 ||
84 	    strcasecmp(val, "0") == 0);
85 }
86 
87 static inline int
88 get_boolean_val(const char *val)
89 {
90 	return (strcasecmp(val, "true") == 0 ||
91 	    strcasecmp(val, "on") == 0 ||
92 	    strcasecmp(val, "yes") == 0 ||
93 	    strcasecmp(val, "1") == 0);
94 }
95 
96 const char *
97 got_repo_get_path(struct got_repository *repo)
98 {
99 	return repo->path;
100 }
101 
102 const char *
103 got_repo_get_path_git_dir(struct got_repository *repo)
104 {
105 	return repo->path_git_dir;
106 }
107 
108 int
109 got_repo_get_fd(struct got_repository *repo)
110 {
111 	return repo->gitdir_fd;
112 }
113 
114 const char *
115 got_repo_get_gitconfig_author_name(struct got_repository *repo)
116 {
117 	return repo->gitconfig_author_name;
118 }
119 
120 const char *
121 got_repo_get_gitconfig_author_email(struct got_repository *repo)
122 {
123 	return repo->gitconfig_author_email;
124 }
125 
126 const char *
127 got_repo_get_global_gitconfig_author_name(struct got_repository *repo)
128 {
129 	return repo->global_gitconfig_author_name;
130 }
131 
132 const char *
133 got_repo_get_global_gitconfig_author_email(struct got_repository *repo)
134 {
135 	return repo->global_gitconfig_author_email;
136 }
137 
138 const char *
139 got_repo_get_gitconfig_owner(struct got_repository *repo)
140 {
141 	return repo->gitconfig_owner;
142 }
143 
144 int
145 got_repo_has_extension(struct got_repository *repo, const char *ext)
146 {
147 	int i;
148 
149 	for (i = 0; i < repo->nextensions; ++i) {
150 		if (!strcasecmp(ext, repo->extnames[i]))
151 			return get_boolean_val(repo->extvals[i]);
152 	}
153 
154 	return 0;
155 }
156 
157 int
158 got_repo_is_bare(struct got_repository *repo)
159 {
160 	return (strcmp(repo->path, repo->path_git_dir) == 0);
161 }
162 
163 static char *
164 get_path_git_child(struct got_repository *repo, const char *basename)
165 {
166 	char *path_child;
167 
168 	if (asprintf(&path_child, "%s/%s", repo->path_git_dir,
169 	    basename) == -1)
170 		return NULL;
171 
172 	return path_child;
173 }
174 
175 char *
176 got_repo_get_path_objects(struct got_repository *repo)
177 {
178 	return get_path_git_child(repo, GOT_OBJECTS_DIR);
179 }
180 
181 char *
182 got_repo_get_path_objects_pack(struct got_repository *repo)
183 {
184 	return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR);
185 }
186 
187 char *
188 got_repo_get_path_refs(struct got_repository *repo)
189 {
190 	return get_path_git_child(repo, GOT_REFS_DIR);
191 }
192 
193 char *
194 got_repo_get_path_packed_refs(struct got_repository *repo)
195 {
196 	return get_path_git_child(repo, GOT_PACKED_REFS_FILE);
197 }
198 
199 static char *
200 get_path_head(struct got_repository *repo)
201 {
202 	return get_path_git_child(repo, GOT_HEAD_FILE);
203 }
204 
205 char *
206 got_repo_get_path_gitconfig(struct got_repository *repo)
207 {
208 	return get_path_git_child(repo, GOT_GITCONFIG);
209 }
210 
211 char *
212 got_repo_get_path_gotconfig(struct got_repository *repo)
213 {
214 	return get_path_git_child(repo, GOT_GOTCONFIG_FILENAME);
215 }
216 
217 const struct got_gotconfig *
218 got_repo_get_gotconfig(struct got_repository *repo)
219 {
220 	return repo->gotconfig;
221 }
222 
223 void
224 got_repo_get_gitconfig_remotes(int *nremotes,
225     const struct got_remote_repo **remotes, struct got_repository *repo)
226 {
227 	*nremotes = repo->ngitconfig_remotes;
228 	*remotes = repo->gitconfig_remotes;
229 }
230 
231 static int
232 is_git_repo(struct got_repository *repo)
233 {
234 	const char *path_git = got_repo_get_path_git_dir(repo);
235 	char *path_objects = got_repo_get_path_objects(repo);
236 	char *path_refs = got_repo_get_path_refs(repo);
237 	char *path_head = get_path_head(repo);
238 	int ret = 0;
239 	struct stat sb;
240 	struct got_reference *head_ref;
241 
242 	if (lstat(path_git, &sb) == -1)
243 		goto done;
244 	if (!S_ISDIR(sb.st_mode))
245 		goto done;
246 
247 	if (lstat(path_objects, &sb) == -1)
248 		goto done;
249 	if (!S_ISDIR(sb.st_mode))
250 		goto done;
251 
252 	if (lstat(path_refs, &sb) == -1)
253 		goto done;
254 	if (!S_ISDIR(sb.st_mode))
255 		goto done;
256 
257 	if (lstat(path_head, &sb) == -1)
258 		goto done;
259 	if (!S_ISREG(sb.st_mode))
260 		goto done;
261 
262 	/* Check if the HEAD reference can be opened. */
263 	if (got_ref_open(&head_ref, repo, GOT_REF_HEAD, 0) != NULL)
264 		goto done;
265 	got_ref_close(head_ref);
266 
267 	ret = 1;
268 done:
269 	free(path_objects);
270 	free(path_refs);
271 	free(path_head);
272 	return ret;
273 
274 }
275 
276 static const struct got_error *
277 close_tempfiles(int *fds, size_t nfds)
278 {
279 	const struct got_error *err = NULL;
280 	int i;
281 
282 	for (i = 0; i < nfds; i++) {
283 		if (fds[i] == -1)
284 			continue;
285 		if (close(fds[i]) == -1) {
286 			err = got_error_from_errno("close");
287 			break;
288 		}
289 	}
290 	free(fds);
291 	return err;
292 }
293 
294 static const struct got_error *
295 open_tempfiles(int **fds, size_t array_size, size_t nfds)
296 {
297 	const struct got_error *err = NULL;
298 	int i;
299 
300 	*fds = calloc(array_size, sizeof(**fds));
301 	if (*fds == NULL)
302 		return got_error_from_errno("calloc");
303 
304 	for (i = 0; i < array_size; i++)
305 		(*fds)[i] = -1;
306 
307 	for (i = 0; i < nfds; i++) {
308 		(*fds)[i] = got_opentempfd();
309 		if ((*fds)[i] == -1) {
310 			err = got_error_from_errno("got_opentempfd");
311 			close_tempfiles(*fds, nfds);
312 			*fds = NULL;
313 			return err;
314 		}
315 	}
316 
317 	return NULL;
318 }
319 
320 static const struct got_error *
321 get_pack_cache_size(int *pack_cache_size)
322 {
323 	struct rlimit rl;
324 
325 	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
326 		return got_error_from_errno("getrlimit");
327 
328 	*pack_cache_size = GOT_PACK_CACHE_SIZE;
329 	if (*pack_cache_size > rl.rlim_cur / 8)
330 		*pack_cache_size = rl.rlim_cur / 8;
331 
332 	return NULL;
333 }
334 
335 const struct got_error *
336 got_repo_pack_fds_open(int **pack_fds)
337 {
338 	const struct got_error *err;
339 	int nfds;
340 
341 	err = get_pack_cache_size(&nfds);
342 	if (err)
343 		return err;
344 
345 	/*
346 	 * We need one basefd and one accumfd per cached pack.
347 	 * Our constants should be set up in a way such that
348 	 * this error never triggers.
349 	 */
350 	if (nfds * 2 > GOT_PACK_NUM_TEMPFILES)
351 		return got_error(GOT_ERR_NO_SPACE);
352 
353 	return open_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES, nfds * 2);
354 }
355 
356 const struct got_error *
357 got_repo_pack_fds_close(int *pack_fds)
358 {
359 	return close_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES);
360 }
361 
362 const struct got_error *
363 got_repo_temp_fds_open(int **temp_fds)
364 {
365 	return open_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES,
366 	    GOT_REPO_NUM_TEMPFILES);
367 }
368 
369 void
370 got_repo_temp_fds_set(struct got_repository *repo, int *temp_fds)
371 {
372 	int i;
373 
374 	for (i = 0; i < GOT_REPO_NUM_TEMPFILES; i++)
375 		repo->tempfiles[i] = temp_fds[i];
376 }
377 
378 const struct got_error *
379 got_repo_temp_fds_get(int *fd, int *idx, struct got_repository *repo)
380 {
381 	int i;
382 
383 	*fd = -1;
384 	*idx = -1;
385 
386 	for (i = 0; i < nitems(repo->tempfiles); i++) {
387 		if (repo->tempfile_use_mask & (1 << i))
388 			continue;
389 		if (repo->tempfiles[i] != -1) {
390 			if (ftruncate(repo->tempfiles[i], 0L) == -1)
391 				return got_error_from_errno("ftruncate");
392 			*fd = repo->tempfiles[i];
393 			*idx = i;
394 			repo->tempfile_use_mask |= (1 << i);
395 			return NULL;
396 		}
397 	}
398 
399 	return got_error(GOT_ERR_REPO_TEMPFILE);
400 }
401 
402 void
403 got_repo_temp_fds_put(int idx, struct got_repository *repo)
404 {
405 	repo->tempfile_use_mask &= ~(1 << idx);
406 }
407 
408 const struct got_error *
409 got_repo_temp_fds_close(int *temp_fds)
410 {
411 	return close_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES);
412 }
413 
414 const struct got_error *
415 got_repo_cache_object(struct got_repository *repo, struct got_object_id *id,
416     struct got_object *obj)
417 {
418 #ifndef GOT_NO_OBJ_CACHE
419 	const struct got_error *err = NULL;
420 	err = got_object_cache_add(&repo->objcache, id, obj);
421 	if (err) {
422 		if (err->code == GOT_ERR_OBJ_EXISTS ||
423 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
424 			err = NULL;
425 		return err;
426 	}
427 	obj->refcnt++;
428 #endif
429 	return NULL;
430 }
431 
432 struct got_object *
433 got_repo_get_cached_object(struct got_repository *repo,
434     struct got_object_id *id)
435 {
436 	return (struct got_object *)got_object_cache_get(&repo->objcache, id);
437 }
438 
439 const struct got_error *
440 got_repo_cache_tree(struct got_repository *repo, struct got_object_id *id,
441     struct got_tree_object *tree)
442 {
443 #ifndef GOT_NO_OBJ_CACHE
444 	const struct got_error *err = NULL;
445 	err = got_object_cache_add(&repo->treecache, id, tree);
446 	if (err) {
447 		if (err->code == GOT_ERR_OBJ_EXISTS ||
448 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
449 			err = NULL;
450 		return err;
451 	}
452 	tree->refcnt++;
453 #endif
454 	return NULL;
455 }
456 
457 struct got_tree_object *
458 got_repo_get_cached_tree(struct got_repository *repo,
459     struct got_object_id *id)
460 {
461 	return (struct got_tree_object *)got_object_cache_get(
462 	    &repo->treecache, id);
463 }
464 
465 const struct got_error *
466 got_repo_cache_commit(struct got_repository *repo, struct got_object_id *id,
467     struct got_commit_object *commit)
468 {
469 #ifndef GOT_NO_OBJ_CACHE
470 	const struct got_error *err = NULL;
471 	err = got_object_cache_add(&repo->commitcache, id, commit);
472 	if (err) {
473 		if (err->code == GOT_ERR_OBJ_EXISTS ||
474 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
475 			err = NULL;
476 		return err;
477 	}
478 	commit->refcnt++;
479 #endif
480 	return NULL;
481 }
482 
483 struct got_commit_object *
484 got_repo_get_cached_commit(struct got_repository *repo,
485     struct got_object_id *id)
486 {
487 	return (struct got_commit_object *)got_object_cache_get(
488 	    &repo->commitcache, id);
489 }
490 
491 const struct got_error *
492 got_repo_cache_tag(struct got_repository *repo, struct got_object_id *id,
493     struct got_tag_object *tag)
494 {
495 #ifndef GOT_NO_OBJ_CACHE
496 	const struct got_error *err = NULL;
497 	err = got_object_cache_add(&repo->tagcache, id, tag);
498 	if (err) {
499 		if (err->code == GOT_ERR_OBJ_EXISTS ||
500 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
501 			err = NULL;
502 		return err;
503 	}
504 	tag->refcnt++;
505 #endif
506 	return NULL;
507 }
508 
509 struct got_tag_object *
510 got_repo_get_cached_tag(struct got_repository *repo, struct got_object_id *id)
511 {
512 	return (struct got_tag_object *)got_object_cache_get(
513 	    &repo->tagcache, id);
514 }
515 
516 const struct got_error *
517 got_repo_cache_raw_object(struct got_repository *repo, struct got_object_id *id,
518     struct got_raw_object *raw)
519 {
520 #ifndef GOT_NO_OBJ_CACHE
521 	const struct got_error *err = NULL;
522 	err = got_object_cache_add(&repo->rawcache, id, raw);
523 	if (err) {
524 		if (err->code == GOT_ERR_OBJ_EXISTS ||
525 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
526 			err = NULL;
527 		return err;
528 	}
529 	raw->refcnt++;
530 #endif
531 	return NULL;
532 }
533 
534 
535 struct got_raw_object *
536 got_repo_get_cached_raw_object(struct got_repository *repo,
537     struct got_object_id *id)
538 {
539 	return (struct got_raw_object *)got_object_cache_get(&repo->rawcache, id);
540 }
541 
542 
543 static const struct got_error *
544 open_repo(struct got_repository *repo, const char *path)
545 {
546 	const struct got_error *err = NULL;
547 
548 	repo->gitdir_fd = -1;
549 
550 	/* bare git repository? */
551 	repo->path_git_dir = strdup(path);
552 	if (repo->path_git_dir == NULL)
553 		return got_error_from_errno("strdup");
554 	if (is_git_repo(repo)) {
555 		repo->path = strdup(repo->path_git_dir);
556 		if (repo->path == NULL) {
557 			err = got_error_from_errno("strdup");
558 			goto done;
559 		}
560 		repo->gitdir_fd = open(repo->path_git_dir,
561 		    O_DIRECTORY | O_CLOEXEC);
562 		if (repo->gitdir_fd == -1) {
563 			err = got_error_from_errno2("open",
564 			    repo->path_git_dir);
565 			goto done;
566 		}
567 		return NULL;
568 	}
569 
570 	/* git repository with working tree? */
571 	free(repo->path_git_dir);
572 	repo->path_git_dir = NULL;
573 	if (asprintf(&repo->path_git_dir, "%s/%s", path, GOT_GIT_DIR) == -1) {
574 		err = got_error_from_errno("asprintf");
575 		goto done;
576 	}
577 	if (is_git_repo(repo)) {
578 		repo->path = strdup(path);
579 		if (repo->path == NULL) {
580 			err = got_error_from_errno("strdup");
581 			goto done;
582 		}
583 		repo->gitdir_fd = open(repo->path_git_dir,
584 		    O_DIRECTORY | O_CLOEXEC);
585 		if (repo->gitdir_fd == -1) {
586 			err = got_error_from_errno2("open",
587 			    repo->path_git_dir);
588 			goto done;
589 		}
590 		return NULL;
591 	}
592 
593 	err = got_error(GOT_ERR_NOT_GIT_REPO);
594 done:
595 	if (err) {
596 		free(repo->path);
597 		repo->path = NULL;
598 		free(repo->path_git_dir);
599 		repo->path_git_dir = NULL;
600 		if (repo->gitdir_fd != -1)
601 			close(repo->gitdir_fd);
602 		repo->gitdir_fd = -1;
603 
604 	}
605 	return err;
606 }
607 
608 static const struct got_error *
609 read_gitconfig(struct got_repository *repo, const char *global_gitconfig_path)
610 {
611 	const struct got_error *err = NULL;
612 	char *repo_gitconfig_path = NULL;
613 
614 	if (global_gitconfig_path) {
615 		/* Read settings from ~/.gitconfig. */
616 		int dummy_repo_version;
617 		err = got_repo_read_gitconfig(&dummy_repo_version,
618 		    &repo->global_gitconfig_author_name,
619 		    &repo->global_gitconfig_author_email,
620 		    NULL, NULL, NULL, NULL, NULL, NULL,
621 		    global_gitconfig_path);
622 		if (err)
623 			return err;
624 	}
625 
626 	/* Read repository's .git/config file. */
627 	repo_gitconfig_path = got_repo_get_path_gitconfig(repo);
628 	if (repo_gitconfig_path == NULL)
629 		return got_error_from_errno("got_repo_get_path_gitconfig");
630 
631 	err = got_repo_read_gitconfig(
632 	    &repo->gitconfig_repository_format_version,
633 	    &repo->gitconfig_author_name, &repo->gitconfig_author_email,
634 	    &repo->gitconfig_remotes, &repo->ngitconfig_remotes,
635 	    &repo->gitconfig_owner, &repo->extnames, &repo->extvals,
636 	    &repo->nextensions, repo_gitconfig_path);
637 	if (err)
638 		goto done;
639 
640 	if (getenv("GOT_IGNORE_GITCONFIG") != NULL) {
641 		int i;
642 
643 		for (i = 0; i < repo->ngitconfig_remotes; i++) {
644 			got_repo_free_remote_repo_data(
645 			    &repo->gitconfig_remotes[i]);
646 		}
647 		free(repo->gitconfig_remotes);
648 		repo->gitconfig_remotes = NULL;
649 		repo->ngitconfig_remotes = 0;
650 
651 		free(repo->gitconfig_author_name);
652 		repo->gitconfig_author_name = NULL;
653 		free(repo->gitconfig_author_email);
654 		repo->gitconfig_author_email = NULL;
655 
656 		free(repo->global_gitconfig_author_name);
657 		repo->global_gitconfig_author_name = NULL;
658 		free(repo->global_gitconfig_author_email);
659 		repo->global_gitconfig_author_email = NULL;
660 	}
661 
662 done:
663 	free(repo_gitconfig_path);
664 	return err;
665 }
666 
667 static const struct got_error *
668 read_gotconfig(struct got_repository *repo)
669 {
670 	const struct got_error *err = NULL;
671 	char *gotconfig_path;
672 
673 	gotconfig_path = got_repo_get_path_gotconfig(repo);
674 	if (gotconfig_path == NULL)
675 		return got_error_from_errno("got_repo_get_path_gotconfig");
676 
677 	err = got_gotconfig_read(&repo->gotconfig, gotconfig_path);
678 	free(gotconfig_path);
679 	return err;
680 }
681 
682 /* Supported repository format extensions. */
683 static const char *const repo_extensions[] = {
684 	"noop",			/* Got supports repository format version 1. */
685 	"preciousObjects",	/* Supported by gotadmin cleanup. */
686 	"worktreeConfig",	/* Got does not care about Git work trees. */
687 };
688 
689 const struct got_error *
690 got_repo_open(struct got_repository **repop, const char *path,
691     const char *global_gitconfig_path, int *pack_fds)
692 {
693 	struct got_repository *repo = NULL;
694 	const struct got_error *err = NULL;
695 	char *repo_path = NULL;
696 	size_t i, j = 0;
697 
698 	*repop = NULL;
699 
700 	repo = calloc(1, sizeof(*repo));
701 	if (repo == NULL)
702 		return got_error_from_errno("calloc");
703 
704 	RB_INIT(&repo->packidx_bloom_filters);
705 	TAILQ_INIT(&repo->packidx_paths);
706 
707 	for (i = 0; i < nitems(repo->privsep_children); i++) {
708 		memset(&repo->privsep_children[i], 0,
709 		    sizeof(repo->privsep_children[0]));
710 		repo->privsep_children[i].imsg_fd = -1;
711 	}
712 
713 	err = got_object_cache_init(&repo->objcache,
714 	    GOT_OBJECT_CACHE_TYPE_OBJ);
715 	if (err)
716 		goto done;
717 	err = got_object_cache_init(&repo->treecache,
718 	    GOT_OBJECT_CACHE_TYPE_TREE);
719 	if (err)
720 		goto done;
721 	err = got_object_cache_init(&repo->commitcache,
722 	    GOT_OBJECT_CACHE_TYPE_COMMIT);
723 	if (err)
724 		goto done;
725 	err = got_object_cache_init(&repo->tagcache,
726 	    GOT_OBJECT_CACHE_TYPE_TAG);
727 	if (err)
728 		goto done;
729 	err = got_object_cache_init(&repo->rawcache,
730 	    GOT_OBJECT_CACHE_TYPE_RAW);
731 	if (err)
732 		goto done;
733 
734 	err = get_pack_cache_size(&repo->pack_cache_size);
735 	if (err)
736 		goto done;
737 	for (i = 0; i < nitems(repo->packs); i++) {
738 		if (pack_fds != NULL && i < repo->pack_cache_size) {
739 			repo->packs[i].basefd = pack_fds[j++];
740 			repo->packs[i].accumfd = pack_fds[j++];
741 		} else {
742 			repo->packs[i].basefd = -1;
743 			repo->packs[i].accumfd = -1;
744 		}
745 	}
746 	for (i = 0; i < nitems(repo->tempfiles); i++)
747 		repo->tempfiles[i] = -1;
748 	repo->pinned_pack = -1;
749 	repo->pinned_packidx = -1;
750 	repo->pinned_pid = 0;
751 
752 	repo_path = realpath(path, NULL);
753 	if (repo_path == NULL) {
754 		err = got_error_from_errno2("realpath", path);
755 		goto done;
756 	}
757 
758 	for (;;) {
759 		char *parent_path;
760 
761 		err = open_repo(repo, repo_path);
762 		if (err == NULL)
763 			break;
764 		if (err->code != GOT_ERR_NOT_GIT_REPO)
765 			goto done;
766 		if (repo_path[0] == '/' && repo_path[1] == '\0') {
767 			err = got_error(GOT_ERR_NOT_GIT_REPO);
768 			goto done;
769 		}
770 		err = got_path_dirname(&parent_path, repo_path);
771 		if (err)
772 			goto done;
773 		free(repo_path);
774 		repo_path = parent_path;
775 	}
776 
777 	err = read_gotconfig(repo);
778 	if (err)
779 		goto done;
780 
781 	err = read_gitconfig(repo, global_gitconfig_path);
782 	if (err)
783 		goto done;
784 	if (repo->gitconfig_repository_format_version != 0) {
785 		err = got_error_path(path, GOT_ERR_GIT_REPO_FORMAT);
786 		goto done;
787 	}
788 	for (i = 0; i < repo->nextensions; i++) {
789 		char *ext = repo->extnames[i];
790 		char *val = repo->extvals[i];
791 		int j, supported = 0;
792 
793 		if (!is_boolean_val(val)) {
794 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
795 			goto done;
796 		}
797 
798 		if (!get_boolean_val(val))
799 			continue;
800 
801 		for (j = 0; j < nitems(repo_extensions); j++) {
802 			if (strcmp(ext, repo_extensions[j]) == 0) {
803 				supported = 1;
804 				break;
805 			}
806 		}
807 		if (!supported) {
808 			err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
809 			goto done;
810 		}
811 	}
812 
813 	err = got_repo_list_packidx(&repo->packidx_paths, repo);
814 done:
815 	if (err)
816 		got_repo_close(repo);
817 	else
818 		*repop = repo;
819 	free(repo_path);
820 	return err;
821 }
822 
823 const struct got_error *
824 got_repo_close(struct got_repository *repo)
825 {
826 	const struct got_error *err = NULL, *child_err;
827 	struct got_packidx_bloom_filter *bf;
828 	size_t i;
829 
830 	for (i = 0; i < repo->pack_cache_size; i++) {
831 		if (repo->packidx_cache[i] == NULL)
832 			break;
833 		got_packidx_close(repo->packidx_cache[i]);
834 	}
835 
836 	while ((bf = RB_MIN(got_packidx_bloom_filter_tree,
837 	    &repo->packidx_bloom_filters))) {
838 		RB_REMOVE(got_packidx_bloom_filter_tree,
839 		    &repo->packidx_bloom_filters, bf);
840 		bloom_free(bf->bloom);
841 		free(bf->bloom);
842 		free(bf);
843 	}
844 
845 	for (i = 0; i < repo->pack_cache_size; i++)
846 		if (repo->packs[i].path_packfile)
847 			if (repo->packs[i].path_packfile)
848 				got_pack_close(&repo->packs[i]);
849 
850 	free(repo->path);
851 	free(repo->path_git_dir);
852 
853 	got_object_cache_close(&repo->objcache);
854 	got_object_cache_close(&repo->treecache);
855 	got_object_cache_close(&repo->commitcache);
856 	got_object_cache_close(&repo->tagcache);
857 	got_object_cache_close(&repo->rawcache);
858 
859 	for (i = 0; i < nitems(repo->privsep_children); i++) {
860 		if (repo->privsep_children[i].imsg_fd == -1)
861 			continue;
862 		imsg_clear(repo->privsep_children[i].ibuf);
863 		free(repo->privsep_children[i].ibuf);
864 		err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd);
865 		child_err = got_privsep_wait_for_child(
866 		    repo->privsep_children[i].pid);
867 		if (child_err && err == NULL)
868 			err = child_err;
869 		if (close(repo->privsep_children[i].imsg_fd) == -1 &&
870 		    err == NULL)
871 			err = got_error_from_errno("close");
872 	}
873 
874 	if (repo->gitdir_fd != -1 && close(repo->gitdir_fd) == -1 &&
875 	    err == NULL)
876 		err = got_error_from_errno("close");
877 
878 	if (repo->gotconfig)
879 		got_gotconfig_free(repo->gotconfig);
880 	free(repo->gitconfig_author_name);
881 	free(repo->gitconfig_author_email);
882 	for (i = 0; i < repo->ngitconfig_remotes; i++)
883 		got_repo_free_remote_repo_data(&repo->gitconfig_remotes[i]);
884 	free(repo->gitconfig_remotes);
885 	for (i = 0; i < repo->nextensions; i++) {
886 		free(repo->extnames[i]);
887 		free(repo->extvals[i]);
888 	}
889 	free(repo->extnames);
890 	free(repo->extvals);
891 
892 	got_pathlist_free(&repo->packidx_paths, GOT_PATHLIST_FREE_PATH);
893 	free(repo);
894 
895 	return err;
896 }
897 
898 void
899 got_repo_free_remote_repo_data(struct got_remote_repo *repo)
900 {
901 	int i;
902 
903 	free(repo->name);
904 	repo->name = NULL;
905 	free(repo->fetch_url);
906 	repo->fetch_url = NULL;
907 	free(repo->send_url);
908 	repo->send_url = NULL;
909 	for (i = 0; i < repo->nfetch_branches; i++)
910 		free(repo->fetch_branches[i]);
911 	free(repo->fetch_branches);
912 	repo->fetch_branches = NULL;
913 	repo->nfetch_branches = 0;
914 	for (i = 0; i < repo->nsend_branches; i++)
915 		free(repo->send_branches[i]);
916 	free(repo->send_branches);
917 	repo->send_branches = NULL;
918 	repo->nsend_branches = 0;
919 }
920 
921 const struct got_error *
922 got_repo_map_path(char **in_repo_path, struct got_repository *repo,
923     const char *input_path)
924 {
925 	const struct got_error *err = NULL;
926 	const char *repo_abspath = NULL;
927 	size_t repolen, len;
928 	char *canonpath, *path = NULL;
929 
930 	*in_repo_path = NULL;
931 
932 	canonpath = strdup(input_path);
933 	if (canonpath == NULL) {
934 		err = got_error_from_errno("strdup");
935 		goto done;
936 	}
937 	err = got_canonpath(input_path, canonpath, strlen(canonpath) + 1);
938 	if (err)
939 		goto done;
940 
941 	repo_abspath = got_repo_get_path(repo);
942 
943 	if (canonpath[0] == '\0') {
944 		path = strdup(canonpath);
945 		if (path == NULL) {
946 			err = got_error_from_errno("strdup");
947 			goto done;
948 		}
949 	} else {
950 		path = realpath(canonpath, NULL);
951 		if (path == NULL) {
952 			if (errno != ENOENT) {
953 				err = got_error_from_errno2("realpath",
954 				    canonpath);
955 				goto done;
956 			}
957 			/*
958 			 * Path is not on disk.
959 			 * Assume it is already relative to repository root.
960 			 */
961 			path = strdup(canonpath);
962 			if (path == NULL) {
963 				err = got_error_from_errno("strdup");
964 				goto done;
965 			}
966 		}
967 
968 		repolen = strlen(repo_abspath);
969 		len = strlen(path);
970 
971 
972 		if (strcmp(path, repo_abspath) == 0) {
973 			free(path);
974 			path = strdup("");
975 			if (path == NULL) {
976 				err = got_error_from_errno("strdup");
977 				goto done;
978 			}
979 		} else if (len > repolen &&
980 		    got_path_is_child(path, repo_abspath, repolen)) {
981 			/* Matched an on-disk path inside repository. */
982 			if (got_repo_is_bare(repo)) {
983 				/*
984 				 * Matched an on-disk path inside repository
985 				 * database. Treat input as repository-relative.
986 				 */
987 				free(path);
988 				path = canonpath;
989 				canonpath = NULL;
990 			} else {
991 				char *child;
992 				/* Strip common prefix with repository path. */
993 				err = got_path_skip_common_ancestor(&child,
994 				    repo_abspath, path);
995 				if (err)
996 					goto done;
997 				free(path);
998 				path = child;
999 			}
1000 		} else {
1001 			/*
1002 			 * Matched unrelated on-disk path.
1003 			 * Treat input as repository-relative.
1004 			 */
1005 			free(path);
1006 			path = canonpath;
1007 			canonpath = NULL;
1008 		}
1009 	}
1010 
1011 	/* Make in-repository path absolute */
1012 	if (path[0] != '/') {
1013 		char *abspath;
1014 		if (asprintf(&abspath, "/%s", path) == -1) {
1015 			err = got_error_from_errno("asprintf");
1016 			goto done;
1017 		}
1018 		free(path);
1019 		path = abspath;
1020 	}
1021 
1022 done:
1023 	free(canonpath);
1024 	if (err)
1025 		free(path);
1026 	else
1027 		*in_repo_path = path;
1028 	return err;
1029 }
1030 
1031 static const struct got_error *
1032 cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
1033     const char *path_packidx)
1034 {
1035 	const struct got_error *err = NULL;
1036 	size_t i;
1037 
1038 	for (i = 0; i < repo->pack_cache_size; i++) {
1039 		if (repo->packidx_cache[i] == NULL)
1040 			break;
1041 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1042 		    path_packidx) == 0) {
1043 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1044 		}
1045 	}
1046 	if (i == repo->pack_cache_size) {
1047 		do {
1048 			i--;
1049 		} while (i > 0 && repo->pinned_packidx >= 0 &&
1050 		    i == repo->pinned_packidx);
1051 		err = got_packidx_close(repo->packidx_cache[i]);
1052 		if (err)
1053 			return err;
1054 	}
1055 
1056 	repo->packidx_cache[i] = packidx;
1057 
1058 	return NULL;
1059 }
1060 
1061 int
1062 got_repo_is_packidx_filename(const char *name, size_t len)
1063 {
1064 	if (len != GOT_PACKIDX_NAMELEN)
1065 		return 0;
1066 
1067 	if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
1068 		return 0;
1069 
1070 	if (strcmp(name + strlen(GOT_PACK_PREFIX) +
1071 	    SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
1072 		return 0;
1073 
1074 	return 1;
1075 }
1076 
1077 static struct got_packidx_bloom_filter *
1078 get_packidx_bloom_filter(struct got_repository *repo,
1079     const char *path, size_t path_len)
1080 {
1081 	struct got_packidx_bloom_filter key;
1082 
1083 	if (strlcpy(key.path, path, sizeof(key.path)) >= sizeof(key.path))
1084 		return NULL; /* XXX */
1085 	key.path_len = path_len;
1086 
1087 	return RB_FIND(got_packidx_bloom_filter_tree,
1088 	    &repo->packidx_bloom_filters, &key);
1089 }
1090 
1091 int
1092 got_repo_check_packidx_bloom_filter(struct got_repository *repo,
1093     const char *path_packidx, struct got_object_id *id)
1094 {
1095 	struct got_packidx_bloom_filter *bf;
1096 
1097 	bf = get_packidx_bloom_filter(repo, path_packidx, strlen(path_packidx));
1098 	if (bf)
1099 		return bloom_check(bf->bloom, id->sha1, sizeof(id->sha1));
1100 
1101 	/* No bloom filter means this pack index must be searched. */
1102 	return 1;
1103 }
1104 
1105 static const struct got_error *
1106 add_packidx_bloom_filter(struct got_repository *repo,
1107     struct got_packidx *packidx, const char *path_packidx)
1108 {
1109 	int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1110 	struct got_packidx_bloom_filter *bf;
1111 	size_t len;
1112 
1113 	/*
1114 	 * Don't use bloom filters for very large pack index files.
1115 	 * Large pack files will contain a relatively large fraction
1116 	 * of our objects so we will likely need to visit them anyway.
1117 	 * The more objects a pack file contains the higher the probability
1118 	 * of a false-positive match from the bloom filter. And reading
1119 	 * all object IDs from a large pack index file can be expensive.
1120 	 */
1121 	if (nobjects > 100000) /* cut-off at about 2MB, at 20 bytes per ID */
1122 		return NULL;
1123 
1124 	/* Do we already have a filter for this pack index? */
1125 	if (get_packidx_bloom_filter(repo, path_packidx,
1126 	    strlen(path_packidx)) != NULL)
1127 		return NULL;
1128 
1129 	bf = calloc(1, sizeof(*bf));
1130 	if (bf == NULL)
1131 		return got_error_from_errno("calloc");
1132 	bf->bloom = calloc(1, sizeof(*bf->bloom));
1133 	if (bf->bloom == NULL) {
1134 		free(bf);
1135 		return got_error_from_errno("calloc");
1136 	}
1137 
1138 	len = strlcpy(bf->path, path_packidx, sizeof(bf->path));
1139 	if (len >= sizeof(bf->path)) {
1140 		free(bf->bloom);
1141 		free(bf);
1142 		return got_error(GOT_ERR_NO_SPACE);
1143 	}
1144 	bf->path_len = len;
1145 
1146 	/* Minimum size supported by our bloom filter is 1000 entries. */
1147 	bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
1148 	for (i = 0; i < nobjects; i++) {
1149 		struct got_packidx_object_id *id;
1150 		id = &packidx->hdr.sorted_ids[i];
1151 		bloom_add(bf->bloom, id->sha1, sizeof(id->sha1));
1152 	}
1153 
1154 	RB_INSERT(got_packidx_bloom_filter_tree,
1155 	    &repo->packidx_bloom_filters, bf);
1156 	return NULL;
1157 }
1158 
1159 static void
1160 purge_packidx_paths(struct got_pathlist_head *packidx_paths)
1161 {
1162 	struct got_pathlist_entry *pe;
1163 
1164 	while (!TAILQ_EMPTY(packidx_paths)) {
1165 		pe = TAILQ_FIRST(packidx_paths);
1166 		TAILQ_REMOVE(packidx_paths, pe, entry);
1167 		free((char *)pe->path);
1168 		free(pe);
1169 	}
1170 }
1171 
1172 static const struct got_error *
1173 refresh_packidx_paths(struct got_repository *repo)
1174 {
1175 	const struct got_error *err = NULL;
1176 	char *objects_pack_dir = NULL;
1177 	struct stat sb;
1178 
1179 	objects_pack_dir = got_repo_get_path_objects_pack(repo);
1180 	if (objects_pack_dir == NULL)
1181 		return got_error_from_errno("got_repo_get_path_objects_pack");
1182 
1183 	if (stat(objects_pack_dir, &sb) == -1) {
1184 		if (errno != ENOENT) {
1185 			err = got_error_from_errno2("stat", objects_pack_dir);
1186 			goto done;
1187 		}
1188 	} else if (TAILQ_EMPTY(&repo->packidx_paths) ||
1189 	    sb.st_mtim.tv_sec != repo->pack_path_mtime.tv_sec ||
1190 	    sb.st_mtim.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1191 		purge_packidx_paths(&repo->packidx_paths);
1192 		err = got_repo_list_packidx(&repo->packidx_paths, repo);
1193 		if (err)
1194 			goto done;
1195 	}
1196 done:
1197 	free(objects_pack_dir);
1198 	return err;
1199 }
1200 
1201 const struct got_error *
1202 got_repo_search_packidx(struct got_packidx **packidx, int *idx,
1203     struct got_repository *repo, struct got_object_id *id)
1204 {
1205 	const struct got_error *err;
1206 	struct got_pathlist_entry *pe;
1207 	size_t i;
1208 
1209 	/* Search pack index cache. */
1210 	for (i = 0; i < repo->pack_cache_size; i++) {
1211 		if (repo->packidx_cache[i] == NULL)
1212 			break;
1213 		if (!got_repo_check_packidx_bloom_filter(repo,
1214 		    repo->packidx_cache[i]->path_packidx, id))
1215 			continue; /* object will not be found in this index */
1216 		*idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
1217 		if (*idx != -1) {
1218 			*packidx = repo->packidx_cache[i];
1219 			/*
1220 			 * Move this cache entry to the front. Repeatedly
1221 			 * searching a wrong pack index can be expensive.
1222 			 */
1223 			if (i > 0) {
1224 				memmove(&repo->packidx_cache[1],
1225 				    &repo->packidx_cache[0],
1226 				    i * sizeof(repo->packidx_cache[0]));
1227 				repo->packidx_cache[0] = *packidx;
1228 				if (repo->pinned_packidx >= 0 &&
1229 				    repo->pinned_packidx < i)
1230 					repo->pinned_packidx++;
1231 				else if (repo->pinned_packidx == i)
1232 					repo->pinned_packidx = 0;
1233 			}
1234 			return NULL;
1235 		}
1236 	}
1237 	/* No luck. Search the filesystem. */
1238 
1239 	err = refresh_packidx_paths(repo);
1240 	if (err)
1241 		return err;
1242 
1243 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1244 		const char *path_packidx = pe->path;
1245 		int is_cached = 0;
1246 
1247 		if (!got_repo_check_packidx_bloom_filter(repo,
1248 		    pe->path, id))
1249 			continue; /* object will not be found in this index */
1250 
1251 		for (i = 0; i < repo->pack_cache_size; i++) {
1252 			if (repo->packidx_cache[i] == NULL)
1253 				break;
1254 			if (strcmp(repo->packidx_cache[i]->path_packidx,
1255 			    path_packidx) == 0) {
1256 				is_cached = 1;
1257 				break;
1258 			}
1259 		}
1260 		if (is_cached)
1261 			continue; /* already searched */
1262 
1263 		err = got_packidx_open(packidx, got_repo_get_fd(repo),
1264 		    path_packidx, 0);
1265 		if (err)
1266 			goto done;
1267 
1268 		err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1269 		if (err)
1270 			goto done;
1271 
1272 		err = cache_packidx(repo, *packidx, path_packidx);
1273 		if (err)
1274 			goto done;
1275 
1276 		*idx = got_packidx_get_object_idx(*packidx, id);
1277 		if (*idx != -1) {
1278 			err = NULL; /* found the object */
1279 			goto done;
1280 		}
1281 	}
1282 
1283 	err = got_error_no_obj(id);
1284 done:
1285 	return err;
1286 }
1287 
1288 const struct got_error *
1289 got_repo_list_packidx(struct got_pathlist_head *packidx_paths,
1290     struct got_repository *repo)
1291 {
1292 	const struct got_error *err = NULL;
1293 	DIR *packdir = NULL;
1294 	struct dirent *dent;
1295 	char *path_packidx = NULL;
1296 	int packdir_fd;
1297 	struct stat sb;
1298 
1299 	packdir_fd = openat(got_repo_get_fd(repo),
1300 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1301 	if (packdir_fd == -1) {
1302 		return got_error_from_errno_fmt("openat: %s/%s",
1303 		    got_repo_get_path_git_dir(repo),
1304 		    GOT_OBJECTS_PACK_DIR);
1305 	}
1306 
1307 	packdir = fdopendir(packdir_fd);
1308 	if (packdir == NULL) {
1309 		err = got_error_from_errno("fdopendir");
1310 		goto done;
1311 	}
1312 
1313 	if (fstat(packdir_fd, &sb) == -1) {
1314 		err = got_error_from_errno("fstat");
1315 		goto done;
1316 	}
1317 	repo->pack_path_mtime.tv_sec = sb.st_mtim.tv_sec;
1318 	repo->pack_path_mtime.tv_nsec = sb.st_mtim.tv_nsec;
1319 
1320 	while ((dent = readdir(packdir)) != NULL) {
1321 		if (!got_repo_is_packidx_filename(dent->d_name,
1322 		    strlen(dent->d_name)))
1323 			continue;
1324 
1325 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
1326 		    dent->d_name) == -1) {
1327 			err = got_error_from_errno("asprintf");
1328 			path_packidx = NULL;
1329 			break;
1330 		}
1331 
1332 		err = got_pathlist_append(packidx_paths, path_packidx, NULL);
1333 		if (err)
1334 			break;
1335 	}
1336 done:
1337 	if (err)
1338 		free(path_packidx);
1339 	if (packdir && closedir(packdir) != 0 && err == NULL)
1340 		err = got_error_from_errno("closedir");
1341 	return err;
1342 }
1343 
1344 const struct got_error *
1345 got_repo_get_packidx(struct got_packidx **packidx, const char *path_packidx,
1346     struct got_repository *repo)
1347 {
1348 	const struct got_error *err;
1349 	size_t i;
1350 
1351 	*packidx = NULL;
1352 
1353 	/* Search pack index cache. */
1354 	for (i = 0; i < repo->pack_cache_size; i++) {
1355 		if (repo->packidx_cache[i] == NULL)
1356 			break;
1357 		if (strcmp(repo->packidx_cache[i]->path_packidx,
1358 		    path_packidx) == 0) {
1359 			*packidx = repo->packidx_cache[i];
1360 			return NULL;
1361 		}
1362 	}
1363 	/* No luck. Search the filesystem. */
1364 
1365 	err = got_packidx_open(packidx, got_repo_get_fd(repo),
1366 	    path_packidx, 0);
1367 	if (err)
1368 		return err;
1369 
1370 	err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1371 	if (err)
1372 		goto done;
1373 
1374 	err = cache_packidx(repo, *packidx, path_packidx);
1375 done:
1376 	if (err) {
1377 		got_packidx_close(*packidx);
1378 		*packidx = NULL;
1379 	}
1380 	return err;
1381 }
1382 
1383 static const struct got_error *
1384 read_packfile_hdr(int fd, struct got_packidx *packidx)
1385 {
1386 	const struct got_error *err = NULL;
1387 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
1388 	struct got_packfile_hdr hdr;
1389 	ssize_t n;
1390 
1391 	n = read(fd, &hdr, sizeof(hdr));
1392 	if (n < 0)
1393 		return got_error_from_errno("read");
1394 	if (n != sizeof(hdr))
1395 		return got_error(GOT_ERR_BAD_PACKFILE);
1396 
1397 	if (be32toh(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
1398 	    be32toh(hdr.version) != GOT_PACKFILE_VERSION ||
1399 	    be32toh(hdr.nobjects) != totobj)
1400 		err = got_error(GOT_ERR_BAD_PACKFILE);
1401 
1402 	return err;
1403 }
1404 
1405 static const struct got_error *
1406 open_packfile(int *fd, struct got_repository *repo,
1407     const char *relpath, struct got_packidx *packidx)
1408 {
1409 	const struct got_error *err = NULL;
1410 
1411 	*fd = openat(got_repo_get_fd(repo), relpath,
1412 	    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1413 	if (*fd == -1)
1414 		return got_error_from_errno_fmt("openat: %s/%s",
1415 		    got_repo_get_path_git_dir(repo), relpath);
1416 
1417 	if (packidx) {
1418 		err = read_packfile_hdr(*fd, packidx);
1419 		if (err) {
1420 			close(*fd);
1421 			*fd = -1;
1422 		}
1423 	}
1424 
1425 	return err;
1426 }
1427 
1428 const struct got_error *
1429 got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
1430     const char *path_packfile, struct got_packidx *packidx)
1431 {
1432 	const struct got_error *err = NULL;
1433 	struct got_pack *pack = NULL;
1434 	struct stat sb;
1435 	size_t i;
1436 
1437 	if (packp)
1438 		*packp = NULL;
1439 
1440 	for (i = 0; i < repo->pack_cache_size; i++) {
1441 		pack = &repo->packs[i];
1442 		if (pack->path_packfile == NULL)
1443 			break;
1444 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1445 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1446 	}
1447 
1448 	if (i == repo->pack_cache_size) {
1449 		struct got_pack tmp;
1450 		do {
1451 			i--;
1452 		} while (i > 0 && repo->pinned_pack >= 0 &&
1453 		    i == repo->pinned_pack);
1454 		err = got_pack_close(&repo->packs[i]);
1455 		if (err)
1456 			return err;
1457 		if (ftruncate(repo->packs[i].basefd, 0L) == -1)
1458 			return got_error_from_errno("ftruncate");
1459 		if (ftruncate(repo->packs[i].accumfd, 0L) == -1)
1460 			return got_error_from_errno("ftruncate");
1461 		memcpy(&tmp, &repo->packs[i], sizeof(tmp));
1462 		memcpy(&repo->packs[i], &repo->packs[0],
1463 		    sizeof(repo->packs[i]));
1464 		memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0]));
1465 		if (repo->pinned_pack == 0)
1466 			repo->pinned_pack = i;
1467 		else if (repo->pinned_pack == i)
1468 			repo->pinned_pack = 0;
1469 		i = 0;
1470 	}
1471 
1472 	pack = &repo->packs[i];
1473 
1474 	pack->path_packfile = strdup(path_packfile);
1475 	if (pack->path_packfile == NULL) {
1476 		err = got_error_from_errno("strdup");
1477 		goto done;
1478 	}
1479 
1480 	err = open_packfile(&pack->fd, repo, path_packfile, packidx);
1481 	if (err)
1482 		goto done;
1483 
1484 	if (fstat(pack->fd, &sb) != 0) {
1485 		err = got_error_from_errno("fstat");
1486 		goto done;
1487 	}
1488 	pack->filesize = sb.st_size;
1489 
1490 	pack->privsep_child = NULL;
1491 
1492 	err = got_delta_cache_alloc(&pack->delta_cache);
1493 	if (err)
1494 		goto done;
1495 
1496 #ifndef GOT_PACK_NO_MMAP
1497 	if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1498 		pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1499 		    pack->fd, 0);
1500 		if (pack->map == MAP_FAILED) {
1501 			if (errno != ENOMEM) {
1502 				err = got_error_from_errno("mmap");
1503 				goto done;
1504 			}
1505 			pack->map = NULL; /* fall back to read(2) */
1506 		}
1507 	}
1508 #endif
1509 done:
1510 	if (err) {
1511 		if (pack)
1512 			got_pack_close(pack);
1513 	} else if (packp)
1514 		*packp = pack;
1515 	return err;
1516 }
1517 
1518 struct got_pack *
1519 got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
1520 {
1521 	struct got_pack *pack = NULL;
1522 	size_t i;
1523 
1524 	for (i = 0; i < repo->pack_cache_size; i++) {
1525 		pack = &repo->packs[i];
1526 		if (pack->path_packfile == NULL)
1527 			break;
1528 		if (strcmp(pack->path_packfile, path_packfile) == 0)
1529 			return pack;
1530 	}
1531 
1532 	return NULL;
1533 }
1534 
1535 const struct got_error *
1536 got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx,
1537     struct got_pack *pack)
1538 {
1539 	size_t i;
1540 	int pinned_pack = -1, pinned_packidx = -1;
1541 
1542 	for (i = 0; i < repo->pack_cache_size; i++) {
1543 		if (repo->packidx_cache[i] &&
1544 		    strcmp(repo->packidx_cache[i]->path_packidx,
1545 		    packidx->path_packidx) == 0)
1546 			pinned_packidx = i;
1547 		if (repo->packs[i].path_packfile &&
1548 		    strcmp(repo->packs[i].path_packfile,
1549 		    pack->path_packfile) == 0)
1550 			pinned_pack = i;
1551 	}
1552 
1553 	if (pinned_packidx == -1 || pinned_pack == -1)
1554 		return got_error(GOT_ERR_PIN_PACK);
1555 
1556 	repo->pinned_pack = pinned_pack;
1557 	repo->pinned_packidx = pinned_packidx;
1558 	if (repo->packs[pinned_pack].privsep_child)
1559 		repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid;
1560 	return NULL;
1561 }
1562 
1563 struct got_pack *
1564 got_repo_get_pinned_pack(struct got_repository *repo)
1565 {
1566 	if (repo->pinned_pack >= 0 &&
1567 	    repo->pinned_pack < repo->pack_cache_size)
1568 		return &repo->packs[repo->pinned_pack];
1569 
1570 	return NULL;
1571 }
1572 
1573 void
1574 got_repo_unpin_pack(struct got_repository *repo)
1575 {
1576 	repo->pinned_packidx = -1;
1577 	repo->pinned_pack = -1;
1578 	repo->pinned_pid = 0;
1579 }
1580 
1581 const struct got_error *
1582 got_repo_init(const char *repo_path, const char *head_name)
1583 {
1584 	const struct got_error *err = NULL;
1585 	const char *dirnames[] = {
1586 		GOT_OBJECTS_DIR,
1587 		GOT_OBJECTS_PACK_DIR,
1588 		GOT_REFS_DIR,
1589 	};
1590 	const char *description_str = "Unnamed repository; "
1591 	    "edit this file 'description' to name the repository.";
1592 	const char *headref = "ref: refs/heads/";
1593 	const char *gitconfig_str = "[core]\n"
1594 	    "\trepositoryformatversion = 0\n"
1595 	    "\tfilemode = true\n"
1596 	    "\tbare = true\n";
1597 	char *headref_str, *path;
1598 	size_t i;
1599 
1600 	if (!got_path_dir_is_empty(repo_path))
1601 		return got_error(GOT_ERR_DIR_NOT_EMPTY);
1602 
1603 	for (i = 0; i < nitems(dirnames); i++) {
1604 		if (asprintf(&path, "%s/%s", repo_path, dirnames[i]) == -1) {
1605 			return got_error_from_errno("asprintf");
1606 		}
1607 		err = got_path_mkdir(path);
1608 		free(path);
1609 		if (err)
1610 			return err;
1611 	}
1612 
1613 	if (asprintf(&path, "%s/%s", repo_path, "description") == -1)
1614 		return got_error_from_errno("asprintf");
1615 	err = got_path_create_file(path, description_str);
1616 	free(path);
1617 	if (err)
1618 		return err;
1619 
1620 	if (asprintf(&path, "%s/%s", repo_path, GOT_HEAD_FILE) == -1)
1621 		return got_error_from_errno("asprintf");
1622 	if (asprintf(&headref_str, "%s%s", headref,
1623 	    head_name ? head_name : "main") == -1) {
1624 		free(path);
1625 		return got_error_from_errno("asprintf");
1626 	}
1627 	err = got_path_create_file(path, headref_str);
1628 	free(headref_str);
1629 	free(path);
1630 	if (err)
1631 		return err;
1632 
1633 	if (asprintf(&path, "%s/%s", repo_path, "config") == -1)
1634 		return got_error_from_errno("asprintf");
1635 	err = got_path_create_file(path, gitconfig_str);
1636 	free(path);
1637 	if (err)
1638 		return err;
1639 
1640 	return NULL;
1641 }
1642 
1643 static const struct got_error *
1644 match_packed_object(struct got_object_id **unique_id,
1645     struct got_repository *repo, const char *id_str_prefix, int obj_type)
1646 {
1647 	const struct got_error *err = NULL;
1648 	struct got_object_id_queue matched_ids;
1649 	struct got_pathlist_entry *pe;
1650 
1651 	STAILQ_INIT(&matched_ids);
1652 
1653 	err = refresh_packidx_paths(repo);
1654 	if (err)
1655 		return err;
1656 
1657 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1658 		const char *path_packidx = pe->path;
1659 		struct got_packidx *packidx;
1660 		struct got_object_qid *qid;
1661 
1662 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
1663 		    path_packidx, 0);
1664 		if (err)
1665 			break;
1666 
1667 		err = got_packidx_match_id_str_prefix(&matched_ids,
1668 		    packidx, id_str_prefix);
1669 		if (err) {
1670 			got_packidx_close(packidx);
1671 			break;
1672 		}
1673 		err = got_packidx_close(packidx);
1674 		if (err)
1675 			break;
1676 
1677 		STAILQ_FOREACH(qid, &matched_ids, entry) {
1678 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1679 				int matched_type;
1680 				err = got_object_get_type(&matched_type, repo,
1681 				    &qid->id);
1682 				if (err)
1683 					goto done;
1684 				if (matched_type != obj_type)
1685 					continue;
1686 			}
1687 			if (*unique_id == NULL) {
1688 				*unique_id = got_object_id_dup(&qid->id);
1689 				if (*unique_id == NULL) {
1690 					err = got_error_from_errno("malloc");
1691 					goto done;
1692 				}
1693 			} else {
1694 				if (got_object_id_cmp(*unique_id,
1695 				    &qid->id) == 0)
1696 					continue; /* packed multiple times */
1697 				err = got_error(GOT_ERR_AMBIGUOUS_ID);
1698 				goto done;
1699 			}
1700 		}
1701 	}
1702 done:
1703 	got_object_id_queue_free(&matched_ids);
1704 	if (err) {
1705 		free(*unique_id);
1706 		*unique_id = NULL;
1707 	}
1708 	return err;
1709 }
1710 
1711 static const struct got_error *
1712 match_loose_object(struct got_object_id **unique_id, const char *path_objects,
1713     const char *object_dir, const char *id_str_prefix, int obj_type,
1714     struct got_repository *repo)
1715 {
1716 	const struct got_error *err = NULL;
1717 	char *path, *id_str = NULL;
1718 	DIR *dir = NULL;
1719 	struct dirent *dent;
1720 	struct got_object_id id;
1721 
1722 	if (asprintf(&path, "%s/%s", path_objects, object_dir) == -1) {
1723 		err = got_error_from_errno("asprintf");
1724 		goto done;
1725 	}
1726 
1727 	dir = opendir(path);
1728 	if (dir == NULL) {
1729 		if (errno == ENOENT) {
1730 			err = NULL;
1731 			goto done;
1732 		}
1733 		err = got_error_from_errno2("opendir", path);
1734 		goto done;
1735 	}
1736 	while ((dent = readdir(dir)) != NULL) {
1737 		int cmp;
1738 
1739 		free(id_str);
1740 		id_str = NULL;
1741 
1742 		if (strcmp(dent->d_name, ".") == 0 ||
1743 		    strcmp(dent->d_name, "..") == 0)
1744 			continue;
1745 
1746 		if (asprintf(&id_str, "%s%s", object_dir, dent->d_name) == -1) {
1747 			err = got_error_from_errno("asprintf");
1748 			goto done;
1749 		}
1750 
1751 		if (!got_parse_sha1_digest(id.sha1, id_str))
1752 			continue;
1753 
1754 		/*
1755 		 * Directory entries do not necessarily appear in
1756 		 * sorted order, so we must iterate over all of them.
1757 		 */
1758 		cmp = strncmp(id_str, id_str_prefix, strlen(id_str_prefix));
1759 		if (cmp != 0)
1760 			continue;
1761 
1762 		if (*unique_id == NULL) {
1763 			if (obj_type != GOT_OBJ_TYPE_ANY) {
1764 				int matched_type;
1765 				err = got_object_get_type(&matched_type, repo,
1766 				    &id);
1767 				if (err)
1768 					goto done;
1769 				if (matched_type != obj_type)
1770 					continue;
1771 			}
1772 			*unique_id = got_object_id_dup(&id);
1773 			if (*unique_id == NULL) {
1774 				err = got_error_from_errno("got_object_id_dup");
1775 				goto done;
1776 			}
1777 		} else {
1778 			if (got_object_id_cmp(*unique_id, &id) == 0)
1779 				continue; /* both packed and loose */
1780 			err = got_error(GOT_ERR_AMBIGUOUS_ID);
1781 			goto done;
1782 		}
1783 	}
1784 done:
1785 	if (dir && closedir(dir) != 0 && err == NULL)
1786 		err = got_error_from_errno("closedir");
1787 	if (err) {
1788 		free(*unique_id);
1789 		*unique_id = NULL;
1790 	}
1791 	free(id_str);
1792 	free(path);
1793 	return err;
1794 }
1795 
1796 const struct got_error *
1797 got_repo_match_object_id_prefix(struct got_object_id **id,
1798     const char *id_str_prefix, int obj_type, struct got_repository *repo)
1799 {
1800 	const struct got_error *err = NULL;
1801 	char *path_objects = NULL, *object_dir = NULL;
1802 	size_t len;
1803 	int i;
1804 
1805 	*id = NULL;
1806 
1807 	path_objects = got_repo_get_path_objects(repo);
1808 
1809 	len = strlen(id_str_prefix);
1810 	if (len > SHA1_DIGEST_STRING_LENGTH - 1) {
1811 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1812 		goto done;
1813 	}
1814 
1815 	for (i = 0; i < len; i++) {
1816 		if (isxdigit((unsigned char)id_str_prefix[i]))
1817 			continue;
1818 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1819 		goto done;
1820 	}
1821 
1822 	if (len >= 2) {
1823 		err = match_packed_object(id, repo, id_str_prefix, obj_type);
1824 		if (err)
1825 			goto done;
1826 		object_dir = strndup(id_str_prefix, 2);
1827 		if (object_dir == NULL) {
1828 			err = got_error_from_errno("strdup");
1829 			goto done;
1830 		}
1831 		err = match_loose_object(id, path_objects, object_dir,
1832 		    id_str_prefix, obj_type, repo);
1833 	} else if (len == 1) {
1834 		int i;
1835 		for (i = 0; i < 0xf; i++) {
1836 			if (asprintf(&object_dir, "%s%.1x", id_str_prefix, i)
1837 			    == -1) {
1838 				err = got_error_from_errno("asprintf");
1839 				goto done;
1840 			}
1841 			err = match_packed_object(id, repo, object_dir,
1842 			    obj_type);
1843 			if (err)
1844 				goto done;
1845 			err = match_loose_object(id, path_objects, object_dir,
1846 			    id_str_prefix, obj_type, repo);
1847 			if (err)
1848 				goto done;
1849 		}
1850 	} else {
1851 		err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1852 		goto done;
1853 	}
1854 done:
1855 	free(path_objects);
1856 	free(object_dir);
1857 	if (err) {
1858 		free(*id);
1859 		*id = NULL;
1860 	} else if (*id == NULL) {
1861 		switch (obj_type) {
1862 		case GOT_OBJ_TYPE_BLOB:
1863 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1864 			    GOT_OBJ_LABEL_BLOB, id_str_prefix);
1865 			break;
1866 		case GOT_OBJ_TYPE_TREE:
1867 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1868 			    GOT_OBJ_LABEL_TREE, id_str_prefix);
1869 			break;
1870 		case GOT_OBJ_TYPE_COMMIT:
1871 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1872 			    GOT_OBJ_LABEL_COMMIT, id_str_prefix);
1873 			break;
1874 		case GOT_OBJ_TYPE_TAG:
1875 			err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1876 			    GOT_OBJ_LABEL_TAG, id_str_prefix);
1877 			break;
1878 		default:
1879 			err = got_error_path(id_str_prefix, GOT_ERR_NO_OBJ);
1880 			break;
1881 		}
1882 	}
1883 
1884 	return err;
1885 }
1886 
1887 const struct got_error *
1888 got_repo_match_object_id(struct got_object_id **id, char **label,
1889     const char *id_str, int obj_type, struct got_reflist_head *refs,
1890     struct got_repository *repo)
1891 {
1892 	const struct got_error *err;
1893 	struct got_tag_object *tag;
1894 	struct got_reference *ref = NULL;
1895 
1896 	*id = NULL;
1897 	if (label)
1898 		*label = NULL;
1899 
1900 	if (refs) {
1901 		err = got_repo_object_match_tag(&tag, id_str, obj_type,
1902 		    refs, repo);
1903 		if (err == NULL) {
1904 			*id = got_object_id_dup(
1905 			    got_object_tag_get_object_id(tag));
1906 			if (*id == NULL)
1907 				err = got_error_from_errno("got_object_id_dup");
1908 			else if (label && asprintf(label, "refs/tags/%s",
1909 			    got_object_tag_get_name(tag)) == -1) {
1910 				err = got_error_from_errno("asprintf");
1911 				free(*id);
1912 				*id = NULL;
1913 			}
1914 			got_object_tag_close(tag);
1915 			return err;
1916 		} else if (err->code != GOT_ERR_OBJ_TYPE &&
1917 		    err->code != GOT_ERR_NO_OBJ)
1918 			return err;
1919 	}
1920 
1921 	err = got_ref_open(&ref, repo, id_str, 0);
1922 	if (err == NULL) {
1923 		err = got_ref_resolve(id, repo, ref);
1924 		if (err)
1925 			goto done;
1926 		if (label) {
1927 			*label = strdup(got_ref_get_name(ref));
1928 			if (*label == NULL) {
1929 				err = got_error_from_errno("strdup");
1930 				goto done;
1931 			}
1932 		}
1933 	} else {
1934 		if (err->code != GOT_ERR_NOT_REF &&
1935 		    err->code != GOT_ERR_BAD_REF_NAME)
1936 			goto done;
1937 		err = got_repo_match_object_id_prefix(id, id_str,
1938 		    obj_type, repo);
1939 		if (err) {
1940 			if (err->code == GOT_ERR_BAD_OBJ_ID_STR)
1941 				err = got_error_not_ref(id_str);
1942 			goto done;
1943 		}
1944 		if (label) {
1945 			err = got_object_id_str(label, *id);
1946 			if (*label == NULL) {
1947 				err = got_error_from_errno("strdup");
1948 				goto done;
1949 			}
1950 		}
1951 	}
1952 done:
1953 	if (ref)
1954 		got_ref_close(ref);
1955 	return err;
1956 }
1957 
1958 const struct got_error *
1959 got_repo_object_match_tag(struct got_tag_object **tag, const char *name,
1960     int obj_type, struct got_reflist_head *refs, struct got_repository *repo)
1961 {
1962 	const struct got_error *err = NULL;
1963 	struct got_reflist_entry *re;
1964 	struct got_object_id *tag_id;
1965 	int name_is_absolute = (strncmp(name, "refs/", 5) == 0);
1966 
1967 	*tag = NULL;
1968 
1969 	TAILQ_FOREACH(re, refs, entry) {
1970 		const char *refname;
1971 		refname = got_ref_get_name(re->ref);
1972 		if (got_ref_is_symbolic(re->ref))
1973 			continue;
1974 		if (strncmp(refname, "refs/tags/", 10) != 0)
1975 			continue;
1976 		if (!name_is_absolute)
1977 			refname += strlen("refs/tags/");
1978 		if (strcmp(refname, name) != 0)
1979 			continue;
1980 		err = got_ref_resolve(&tag_id, repo, re->ref);
1981 		if (err)
1982 			break;
1983 		err = got_object_open_as_tag(tag, repo, tag_id);
1984 		free(tag_id);
1985 		if (err)
1986 			break;
1987 		if (obj_type == GOT_OBJ_TYPE_ANY ||
1988 		    got_object_tag_get_object_type(*tag) == obj_type)
1989 			break;
1990 		got_object_tag_close(*tag);
1991 		*tag = NULL;
1992 	}
1993 
1994 	if (err == NULL && *tag == NULL)
1995 		err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1996 		    GOT_OBJ_LABEL_TAG, name);
1997 	return err;
1998 }
1999 
2000 static const struct got_error *
2001 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
2002     const char *name, mode_t mode, struct got_object_id *blob_id)
2003 {
2004 	const struct got_error *err = NULL;
2005 
2006 	 *new_te = NULL;
2007 
2008 	*new_te = calloc(1, sizeof(**new_te));
2009 	if (*new_te == NULL)
2010 		return got_error_from_errno("calloc");
2011 
2012 	if (strlcpy((*new_te)->name, name, sizeof((*new_te)->name)) >=
2013 	    sizeof((*new_te)->name)) {
2014 		err = got_error(GOT_ERR_NO_SPACE);
2015 		goto done;
2016 	}
2017 
2018 	if (S_ISLNK(mode)) {
2019 		(*new_te)->mode = S_IFLNK;
2020 	} else {
2021 		(*new_te)->mode = S_IFREG;
2022 		(*new_te)->mode |= (mode & (S_IRWXU | S_IRWXG | S_IRWXO));
2023 	}
2024 	memcpy(&(*new_te)->id, blob_id, sizeof((*new_te)->id));
2025 done:
2026 	if (err && *new_te) {
2027 		free(*new_te);
2028 		*new_te = NULL;
2029 	}
2030 	return err;
2031 }
2032 
2033 static const struct got_error *
2034 import_file(struct got_tree_entry **new_te, struct dirent *de,
2035     const char *path, struct got_repository *repo)
2036 {
2037 	const struct got_error *err;
2038 	struct got_object_id *blob_id = NULL;
2039 	char *filepath;
2040 	struct stat sb;
2041 
2042 	if (asprintf(&filepath, "%s%s%s", path,
2043 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2044 		return got_error_from_errno("asprintf");
2045 
2046 	if (lstat(filepath, &sb) != 0) {
2047 		err = got_error_from_errno2("lstat", path);
2048 		goto done;
2049 	}
2050 
2051 	err = got_object_blob_create(&blob_id, filepath, repo);
2052 	if (err)
2053 		goto done;
2054 
2055 	err = alloc_added_blob_tree_entry(new_te, de->d_name, sb.st_mode,
2056 	    blob_id);
2057 done:
2058 	free(filepath);
2059 	if (err)
2060 		free(blob_id);
2061 	return err;
2062 }
2063 
2064 static const struct got_error *
2065 insert_tree_entry(struct got_tree_entry *new_te,
2066     struct got_pathlist_head *paths)
2067 {
2068 	const struct got_error *err = NULL;
2069 	struct got_pathlist_entry *new_pe;
2070 
2071 	err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
2072 	if (err)
2073 		return err;
2074 	if (new_pe == NULL)
2075 		return got_error(GOT_ERR_TREE_DUP_ENTRY);
2076 	return NULL;
2077 }
2078 
2079 static const struct got_error *write_tree(struct got_object_id **,
2080     const char *, struct got_pathlist_head *, struct got_repository *,
2081     got_repo_import_cb progress_cb, void *progress_arg);
2082 
2083 static const struct got_error *
2084 import_subdir(struct got_tree_entry **new_te, struct dirent *de,
2085     const char *path, struct got_pathlist_head *ignores,
2086     struct got_repository *repo,
2087     got_repo_import_cb progress_cb, void *progress_arg)
2088 {
2089 	const struct got_error *err;
2090 	struct got_object_id *id = NULL;
2091 	char *subdirpath;
2092 
2093 	if (asprintf(&subdirpath, "%s%s%s", path,
2094 	    path[0] == '\0' ? "" : "/", de->d_name) == -1)
2095 		return got_error_from_errno("asprintf");
2096 
2097 	(*new_te) = calloc(1, sizeof(**new_te));
2098 	if (*new_te == NULL)
2099 		return got_error_from_errno("calloc");
2100 	(*new_te)->mode = S_IFDIR;
2101 	if (strlcpy((*new_te)->name, de->d_name, sizeof((*new_te)->name)) >=
2102 	    sizeof((*new_te)->name)) {
2103 		err = got_error(GOT_ERR_NO_SPACE);
2104 		goto done;
2105 	}
2106 	err = write_tree(&id, subdirpath, ignores,  repo,
2107 	    progress_cb, progress_arg);
2108 	if (err)
2109 		goto done;
2110 	memcpy(&(*new_te)->id, id, sizeof((*new_te)->id));
2111 
2112 done:
2113 	free(id);
2114 	free(subdirpath);
2115 	if (err) {
2116 		free(*new_te);
2117 		*new_te = NULL;
2118 	}
2119 	return err;
2120 }
2121 
2122 static const struct got_error *
2123 write_tree(struct got_object_id **new_tree_id, const char *path_dir,
2124     struct got_pathlist_head *ignores, struct got_repository *repo,
2125     got_repo_import_cb progress_cb, void *progress_arg)
2126 {
2127 	const struct got_error *err = NULL;
2128 	DIR *dir;
2129 	struct dirent *de;
2130 	int nentries;
2131 	struct got_tree_entry *new_te = NULL;
2132 	struct got_pathlist_head paths;
2133 	struct got_pathlist_entry *pe;
2134 
2135 	*new_tree_id = NULL;
2136 
2137 	TAILQ_INIT(&paths);
2138 
2139 	dir = opendir(path_dir);
2140 	if (dir == NULL) {
2141 		err = got_error_from_errno2("opendir", path_dir);
2142 		goto done;
2143 	}
2144 
2145 	nentries = 0;
2146 	while ((de = readdir(dir)) != NULL) {
2147 		int ignore = 0;
2148 		int type;
2149 
2150 		if (strcmp(de->d_name, ".") == 0 ||
2151 		    strcmp(de->d_name, "..") == 0)
2152 			continue;
2153 
2154 		err = got_path_dirent_type(&type, path_dir, de);
2155 		if (err)
2156 			goto done;
2157 
2158 		TAILQ_FOREACH(pe, ignores, entry) {
2159 			if (type == DT_DIR && pe->path_len > 0 &&
2160 			    pe->path[pe->path_len - 1] == '/') {
2161 				char stripped[PATH_MAX];
2162 
2163 				if (strlcpy(stripped, pe->path,
2164 				    sizeof(stripped)) >= sizeof(stripped)) {
2165 					err = got_error(GOT_ERR_NO_SPACE);
2166 					goto done;
2167 				}
2168 				got_path_strip_trailing_slashes(stripped);
2169 				if (fnmatch(stripped, de->d_name, 0) == 0) {
2170 					ignore = 1;
2171 					break;
2172 				}
2173 			} else if (fnmatch(pe->path, de->d_name, 0) == 0) {
2174 				ignore = 1;
2175 				break;
2176 			}
2177 		}
2178 		if (ignore)
2179 			continue;
2180 
2181 		if (type == DT_DIR) {
2182 			err = import_subdir(&new_te, de, path_dir,
2183 			    ignores, repo, progress_cb, progress_arg);
2184 			if (err) {
2185 				if (err->code != GOT_ERR_NO_TREE_ENTRY)
2186 					goto done;
2187 				err = NULL;
2188 				continue;
2189 			}
2190 		} else if (type == DT_REG || type == DT_LNK) {
2191 			err = import_file(&new_te, de, path_dir, repo);
2192 			if (err)
2193 				goto done;
2194 		} else
2195 			continue;
2196 
2197 		err = insert_tree_entry(new_te, &paths);
2198 		if (err)
2199 			goto done;
2200 		nentries++;
2201 	}
2202 
2203 	if (TAILQ_EMPTY(&paths)) {
2204 		err = got_error_msg(GOT_ERR_NO_TREE_ENTRY,
2205 		    "cannot create tree without any entries");
2206 		goto done;
2207 	}
2208 
2209 	TAILQ_FOREACH(pe, &paths, entry) {
2210 		struct got_tree_entry *te = pe->data;
2211 		char *path;
2212 		if (!S_ISREG(te->mode) && !S_ISLNK(te->mode))
2213 			continue;
2214 		if (asprintf(&path, "%s/%s", path_dir, pe->path) == -1) {
2215 			err = got_error_from_errno("asprintf");
2216 			goto done;
2217 		}
2218 		err = (*progress_cb)(progress_arg, path);
2219 		free(path);
2220 		if (err)
2221 			goto done;
2222 	}
2223 
2224 	err = got_object_tree_create(new_tree_id, &paths, nentries, repo);
2225 done:
2226 	if (dir)
2227 		closedir(dir);
2228 	got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2229 	return err;
2230 }
2231 
2232 const struct got_error *
2233 got_repo_import(struct got_object_id **new_commit_id, const char *path_dir,
2234     const char *logmsg, const char *author, struct got_pathlist_head *ignores,
2235     struct got_repository *repo, got_repo_import_cb progress_cb,
2236     void *progress_arg)
2237 {
2238 	const struct got_error *err;
2239 	struct got_object_id *new_tree_id;
2240 
2241 	err = write_tree(&new_tree_id, path_dir, ignores, repo,
2242 	    progress_cb, progress_arg);
2243 	if (err)
2244 		return err;
2245 
2246 	err = got_object_commit_create(new_commit_id, new_tree_id, NULL, 0,
2247 	    author, time(NULL), author, time(NULL), logmsg, repo);
2248 	free(new_tree_id);
2249 	return err;
2250 }
2251 
2252 const struct got_error *
2253 got_repo_get_loose_object_info(int *nobjects, off_t *ondisk_size,
2254     struct got_repository *repo)
2255 {
2256 	const struct got_error *err = NULL;
2257 	char *path_objects = NULL, *path = NULL;
2258 	DIR *dir = NULL;
2259 	struct got_object_id id;
2260 	int i;
2261 
2262 	*nobjects = 0;
2263 	*ondisk_size = 0;
2264 
2265 	path_objects = got_repo_get_path_objects(repo);
2266 	if (path_objects == NULL)
2267 		return got_error_from_errno("got_repo_get_path_objects");
2268 
2269 	for (i = 0; i <= 0xff; i++) {
2270 		struct dirent *dent;
2271 
2272 		if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
2273 			err = got_error_from_errno("asprintf");
2274 			break;
2275 		}
2276 
2277 		dir = opendir(path);
2278 		if (dir == NULL) {
2279 			if (errno == ENOENT) {
2280 				err = NULL;
2281 				continue;
2282 			}
2283 			err = got_error_from_errno2("opendir", path);
2284 			break;
2285 		}
2286 
2287 		while ((dent = readdir(dir)) != NULL) {
2288 			char *id_str;
2289 			int fd;
2290 			struct stat sb;
2291 
2292 			if (strcmp(dent->d_name, ".") == 0 ||
2293 			    strcmp(dent->d_name, "..") == 0)
2294 				continue;
2295 
2296 			if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
2297 				err = got_error_from_errno("asprintf");
2298 				goto done;
2299 			}
2300 
2301 			if (!got_parse_sha1_digest(id.sha1, id_str)) {
2302 				free(id_str);
2303 				continue;
2304 			}
2305 			free(id_str);
2306 
2307 			err = got_object_open_loose_fd(&fd, &id, repo);
2308 			if (err)
2309 				goto done;
2310 
2311 			if (fstat(fd, &sb) == -1) {
2312 				err = got_error_from_errno("fstat");
2313 				close(fd);
2314 				goto done;
2315 			}
2316 			(*nobjects)++;
2317 			(*ondisk_size) += sb.st_size;
2318 
2319 			if (close(fd) == -1) {
2320 				err = got_error_from_errno("close");
2321 				goto done;
2322 			}
2323 		}
2324 
2325 		if (closedir(dir) != 0) {
2326 			err = got_error_from_errno("closedir");
2327 			goto done;
2328 		}
2329 		dir = NULL;
2330 
2331 		free(path);
2332 		path = NULL;
2333 	}
2334 done:
2335 	if (dir && closedir(dir) != 0 && err == NULL)
2336 		err = got_error_from_errno("closedir");
2337 
2338 	if (err) {
2339 		*nobjects = 0;
2340 		*ondisk_size = 0;
2341 	}
2342 	free(path_objects);
2343 	free(path);
2344 	return err;
2345 }
2346 
2347 const struct got_error *
2348 got_repo_get_packfile_info(int *npackfiles, int *nobjects,
2349     off_t *total_packsize, struct got_repository *repo)
2350 {
2351 	const struct got_error *err = NULL;
2352 	DIR *packdir = NULL;
2353 	struct dirent *dent;
2354 	struct got_packidx *packidx = NULL;
2355 	char *path_packidx;
2356 	char *path_packfile;
2357 	int packdir_fd;
2358 	struct stat sb;
2359 
2360 	*npackfiles = 0;
2361 	*nobjects = 0;
2362 	*total_packsize = 0;
2363 
2364 	packdir_fd = openat(got_repo_get_fd(repo),
2365 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
2366 	if (packdir_fd == -1) {
2367 		return got_error_from_errno_fmt("openat: %s/%s",
2368 		    got_repo_get_path_git_dir(repo),
2369 		    GOT_OBJECTS_PACK_DIR);
2370 	}
2371 
2372 	packdir = fdopendir(packdir_fd);
2373 	if (packdir == NULL) {
2374 		err = got_error_from_errno("fdopendir");
2375 		goto done;
2376 	}
2377 
2378 	while ((dent = readdir(packdir)) != NULL) {
2379 		if (!got_repo_is_packidx_filename(dent->d_name,
2380 		    strlen(dent->d_name)))
2381 			continue;
2382 
2383 		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
2384 		    dent->d_name) == -1) {
2385 			err = got_error_from_errno("asprintf");
2386 			goto done;
2387 		}
2388 
2389 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
2390 		    path_packidx, 0);
2391 		free(path_packidx);
2392 		if (err)
2393 			goto done;
2394 
2395 		if (fstat(packidx->fd, &sb) == -1)
2396 			goto done;
2397 		*total_packsize += sb.st_size;
2398 
2399 		err = got_packidx_get_packfile_path(&path_packfile,
2400 		    packidx->path_packidx);
2401 		if (err)
2402 			goto done;
2403 
2404 		if (fstatat(got_repo_get_fd(repo), path_packfile, &sb,
2405 		    0) == -1) {
2406 			free(path_packfile);
2407 			goto done;
2408 		}
2409 		free(path_packfile);
2410 		*total_packsize += sb.st_size;
2411 
2412 		*nobjects += be32toh(packidx->hdr.fanout_table[0xff]);
2413 
2414 		(*npackfiles)++;
2415 
2416 		got_packidx_close(packidx);
2417 		packidx = NULL;
2418 	}
2419 done:
2420 	if (packidx)
2421 		got_packidx_close(packidx);
2422 	if (packdir && closedir(packdir) != 0 && err == NULL)
2423 		err = got_error_from_errno("closedir");
2424 	if (err) {
2425 		*npackfiles = 0;
2426 		*nobjects = 0;
2427 		*total_packsize = 0;
2428 	}
2429 	return err;
2430 }
2431 
2432 RB_GENERATE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
2433     got_packidx_bloom_filter_cmp);