1 /*
2  * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/uio.h>
19 #include <sys/stat.h>
20 #include <sys/socket.h>
21 #include <sys/wait.h>
22 
23 #include <dirent.h>
24 #include <errno.h>
25 #include <fcntl.h>
26 #include <stdint.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <limits.h>
31 #include <unistd.h>
32 
33 #include "got_error.h"
34 #include "got_cancel.h"
35 #include "got_object.h"
36 #include "got_reference.h"
37 #include "got_repository.h"
38 #include "got_repository_admin.h"
39 #include "got_opentemp.h"
40 #include "got_path.h"
41 
42 #include "got_lib_delta.h"
43 #include "got_lib_object.h"
44 #include "got_lib_object_idset.h"
45 #include "got_lib_object_cache.h"
46 #include "got_lib_pack.h"
47 #include "got_lib_privsep.h"
48 #include "got_lib_repository.h"
49 #include "got_lib_pack_create.h"
50 #include "got_lib_sha1.h"
51 #include "got_lib_lockfile.h"
52 
53 #ifndef nitems
54 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
55 #endif
56 
57 static const struct got_error *
58 get_reflist_object_ids(struct got_object_id ***ids, int *nobjects,
59     unsigned int wanted_obj_type_mask, struct got_reflist_head *refs,
60     struct got_repository *repo,
61     got_cancel_cb cancel_cb, void *cancel_arg)
62 {
63 	const struct got_error *err = NULL;
64 	const size_t alloc_chunksz = 256;
65 	size_t nalloc;
66 	struct got_reflist_entry *re;
67 	int i;
68 
69 	*ids = NULL;
70 	*nobjects = 0;
71 
72 	*ids = reallocarray(NULL, alloc_chunksz, sizeof(struct got_object_id *));
73 	if (*ids == NULL)
74 		return got_error_from_errno("reallocarray");
75 	nalloc = alloc_chunksz;
76 
77 	TAILQ_FOREACH(re, refs, entry) {
78 		struct got_object_id *id;
79 
80 		if (cancel_cb) {
81 			err = cancel_cb(cancel_arg);
82 			if (err)
83 				goto done;
84 		}
85 
86 		err = got_ref_resolve(&id, repo, re->ref);
87 		if (err)
88 			goto done;
89 
90 		if (wanted_obj_type_mask != GOT_OBJ_TYPE_ANY) {
91 			int obj_type;
92 			err = got_object_get_type(&obj_type, repo, id);
93 			if (err)
94 				goto done;
95 			if ((wanted_obj_type_mask & (1 << obj_type)) == 0) {
96 				free(id);
97 				id = NULL;
98 				continue;
99 			}
100 		}
101 
102 		if (nalloc <= *nobjects) {
103 			struct got_object_id **new;
104 			new = recallocarray(*ids, nalloc,
105 			    nalloc + alloc_chunksz,
106 			    sizeof(struct got_object_id *));
107 			if (new == NULL) {
108 				err = got_error_from_errno(
109 				    "recallocarray");
110 				goto done;
111 			}
112 			*ids = new;
113 			nalloc += alloc_chunksz;
114 		}
115 		(*ids)[*nobjects] = id;
116 		if ((*ids)[*nobjects] == NULL) {
117 			err = got_error_from_errno("got_object_id_dup");
118 			goto done;
119 		}
120 		(*nobjects)++;
121 	}
122 done:
123 	if (err) {
124 		for (i = 0; i < *nobjects; i++)
125 			free((*ids)[i]);
126 		free(*ids);
127 		*ids = NULL;
128 		*nobjects = 0;
129 	}
130 	return err;
131 }
132 
133 const struct got_error *
134 got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash,
135     struct got_reflist_head *include_refs,
136     struct got_reflist_head *exclude_refs, struct got_repository *repo,
137     int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
138     got_cancel_cb cancel_cb, void *cancel_arg)
139 {
140 	const struct got_error *err = NULL;
141 	struct got_object_id **ours = NULL, **theirs = NULL;
142 	int nours = 0, ntheirs = 0, packfd = -1, i;
143 	char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL;
144 	char *sha1_str = NULL;
145 
146 	*packfile = NULL;
147 	*pack_hash = NULL;
148 
149 	if (asprintf(&path, "%s/%s/packing.pack",
150 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) {
151 		err = got_error_from_errno("asprintf");
152 		goto done;
153 	}
154 	err = got_opentemp_named_fd(&tmpfile_path, &packfd, path);
155 	if (err)
156 		goto done;
157 
158 	if (fchmod(packfd, GOT_DEFAULT_FILE_MODE) != 0) {
159 		err = got_error_from_errno2("fchmod", tmpfile_path);
160 		goto done;
161 	}
162 
163 	*packfile = fdopen(packfd, "w");
164 	if (*packfile == NULL) {
165 		err = got_error_from_errno2("fdopen", tmpfile_path);
166 		goto done;
167 	}
168 	packfd = -1;
169 
170 	err = get_reflist_object_ids(&ours, &nours,
171 	    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
172 	    include_refs, repo, cancel_cb, cancel_arg);
173 	if (err)
174 		goto done;
175 
176 	if (nours == 0) {
177 		err = got_error(GOT_ERR_CANNOT_PACK);
178 		goto done;
179 	}
180 
181 	if (!TAILQ_EMPTY(exclude_refs)) {
182 		err = get_reflist_object_ids(&theirs, &ntheirs,
183 		    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
184 		    exclude_refs, repo,
185 		    cancel_cb, cancel_arg);
186 		if (err)
187 			goto done;
188 	}
189 
190 	*pack_hash = calloc(1, sizeof(**pack_hash));
191 	if (*pack_hash == NULL) {
192 		err = got_error_from_errno("calloc");
193 		goto done;
194 	}
195 
196 	err = got_pack_create((*pack_hash)->sha1, *packfile, theirs, ntheirs,
197 	    ours, nours, repo, loose_obj_only, 0, progress_cb, progress_arg,
198 	    cancel_cb, cancel_arg);
199 	if (err)
200 		goto done;
201 
202 	err = got_object_id_str(&sha1_str, *pack_hash);
203 	if (err)
204 		goto done;
205 	if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
206 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR,
207 	    sha1_str) == -1) {
208 		err = got_error_from_errno("asprintf");
209 		goto done;
210 	}
211 
212 	if (fflush(*packfile) == -1) {
213 		err = got_error_from_errno("fflush");
214 		goto done;
215 	}
216 	if (fseek(*packfile, 0L, SEEK_SET) == -1) {
217 		err = got_error_from_errno("fseek");
218 		goto done;
219 	}
220 	if (rename(tmpfile_path, packfile_path) == -1) {
221 		err = got_error_from_errno3("rename", tmpfile_path,
222 		    packfile_path);
223 		goto done;
224 	}
225 	free(tmpfile_path);
226 	tmpfile_path = NULL;
227 done:
228 	for (i = 0; i < nours; i++)
229 		free(ours[i]);
230 	free(ours);
231 	for (i = 0; i < ntheirs; i++)
232 		free(theirs[i]);
233 	free(theirs);
234 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
235 		err = got_error_from_errno2("close", packfile_path);
236 	if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL)
237 		err = got_error_from_errno2("unlink", tmpfile_path);
238 	free(tmpfile_path);
239 	free(packfile_path);
240 	free(sha1_str);
241 	free(path);
242 	if (err) {
243 		free(*pack_hash);
244 		*pack_hash = NULL;
245 		if (*packfile)
246 			fclose(*packfile);
247 		*packfile = NULL;
248 	}
249 	return err;
250 }
251 
252 const struct got_error *
253 got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
254     struct got_repository *repo,
255     got_pack_index_progress_cb progress_cb, void *progress_arg,
256     got_cancel_cb cancel_cb, void *cancel_arg)
257 {
258 	size_t i;
259 	char *path;
260 	int imsg_idxfds[2];
261 	int npackfd = -1, idxfd = -1, nidxfd = -1;
262 	int tmpfds[3];
263 	int idxstatus, done = 0;
264 	const struct got_error *err;
265 	struct imsgbuf idxibuf;
266 	pid_t idxpid;
267 	char *tmpidxpath = NULL;
268 	char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL;
269 	const char *repo_path = got_repo_get_path_git_dir(repo);
270 	struct stat sb;
271 
272 	for (i = 0; i < nitems(tmpfds); i++)
273 		tmpfds[i] = -1;
274 
275 	if (asprintf(&path, "%s/%s/indexing.idx",
276 	    repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
277 		err = got_error_from_errno("asprintf");
278 		goto done;
279 	}
280 	err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path);
281 	free(path);
282 	if (err)
283 		goto done;
284 	if (fchmod(idxfd, GOT_DEFAULT_FILE_MODE) != 0) {
285 		err = got_error_from_errno2("fchmod", tmpidxpath);
286 		goto done;
287 	}
288 
289 	nidxfd = dup(idxfd);
290 	if (nidxfd == -1) {
291 		err = got_error_from_errno("dup");
292 		goto done;
293 	}
294 
295 	for (i = 0; i < nitems(tmpfds); i++) {
296 		tmpfds[i] = got_opentempfd();
297 		if (tmpfds[i] == -1) {
298 			err = got_error_from_errno("got_opentempfd");
299 			goto done;
300 		}
301 	}
302 
303 	err = got_object_id_str(&id_str, pack_hash);
304 	if (err)
305 		goto done;
306 
307 	if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
308 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
309 		err = got_error_from_errno("asprintf");
310 		goto done;
311 	}
312 
313 	if (fstat(fileno(packfile), &sb) == -1) {
314 		err = got_error_from_errno2("fstat", packfile_path);
315 		goto done;
316 	}
317 
318 	if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
319 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
320 		err = got_error_from_errno("asprintf");
321 		goto done;
322 	}
323 
324 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
325 		err = got_error_from_errno("socketpair");
326 		goto done;
327 	}
328 	idxpid = fork();
329 	if (idxpid == -1) {
330 		err= got_error_from_errno("fork");
331 		goto done;
332 	} else if (idxpid == 0)
333 		got_privsep_exec_child(imsg_idxfds,
334 		    GOT_PATH_PROG_INDEX_PACK, packfile_path);
335 	if (close(imsg_idxfds[1]) == -1) {
336 		err = got_error_from_errno("close");
337 		goto done;
338 	}
339 	imsg_init(&idxibuf, imsg_idxfds[0]);
340 
341 	npackfd = dup(fileno(packfile));
342 	if (npackfd == -1) {
343 		err = got_error_from_errno("dup");
344 		goto done;
345 	}
346 	err = got_privsep_send_index_pack_req(&idxibuf, pack_hash->sha1,
347 	    npackfd);
348 	if (err != NULL)
349 		goto done;
350 	npackfd = -1;
351 	err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
352 	if (err != NULL)
353 		goto done;
354 	nidxfd = -1;
355 	for (i = 0; i < nitems(tmpfds); i++) {
356 		err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
357 		if (err != NULL)
358 			goto done;
359 		tmpfds[i] = -1;
360 	}
361 	done = 0;
362 	while (!done) {
363 		int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
364 
365 		if (cancel_cb) {
366 			err = cancel_cb(cancel_arg);
367 			if (err)
368 				goto done;
369 		}
370 
371 		err = got_privsep_recv_index_progress(&done, &nobj_total,
372 		    &nobj_indexed, &nobj_loose, &nobj_resolved,
373 		    &idxibuf);
374 		if (err != NULL)
375 			goto done;
376 		if (nobj_indexed != 0) {
377 			err = progress_cb(progress_arg, sb.st_size,
378 			    nobj_total, nobj_indexed, nobj_loose,
379 			    nobj_resolved);
380 			if (err)
381 				break;
382 		}
383 		imsg_clear(&idxibuf);
384 	}
385 	if (close(imsg_idxfds[0]) == -1) {
386 		err = got_error_from_errno("close");
387 		goto done;
388 	}
389 	if (waitpid(idxpid, &idxstatus, 0) == -1) {
390 		err = got_error_from_errno("waitpid");
391 		goto done;
392 	}
393 
394 	if (rename(tmpidxpath, idxpath) == -1) {
395 		err = got_error_from_errno3("rename", tmpidxpath, idxpath);
396 		goto done;
397 	}
398 	free(tmpidxpath);
399 	tmpidxpath = NULL;
400 
401 done:
402 	if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
403 		err = got_error_from_errno2("unlink", tmpidxpath);
404 	if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
405 		err = got_error_from_errno("close");
406 	if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
407 		err = got_error_from_errno("close");
408 	for (i = 0; i < nitems(tmpfds); i++) {
409 		if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
410 			err = got_error_from_errno("close");
411 	}
412 	free(tmpidxpath);
413 	free(idxpath);
414 	free(packfile_path);
415 	return err;
416 }
417 
418 const struct got_error *
419 got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash,
420     struct got_repository *repo, const char *packfile_path)
421 {
422 	const struct got_error *err = NULL;
423 	const char *packdir_path = NULL;
424 	char *packfile_name = NULL, *p, *dot;
425 	struct got_object_id id;
426 	int packfd = -1;
427 
428 	*packfile = NULL;
429 	*pack_hash = NULL;
430 
431 	packdir_path = got_repo_get_path_objects_pack(repo);
432 	if (packdir_path == NULL)
433 		return got_error_from_errno("got_repo_get_path_objects_pack");
434 
435 	if (!got_path_is_child(packfile_path, packdir_path,
436 	    strlen(packdir_path))) {
437 		err = got_error_path(packfile_path, GOT_ERR_BAD_PATH);
438 		goto done;
439 
440 	}
441 
442 	err = got_path_basename(&packfile_name, packfile_path);
443 	if (err)
444 		goto done;
445 	p = packfile_name;
446 
447 	if (strncmp(p, "pack-", 5) != 0) {
448 		err = got_error_fmt(GOT_ERR_BAD_PATH,
449 		    "'%s' is not a valid pack file name",
450 		    packfile_name);
451 		goto done;
452 	}
453 	p += 5;
454 	dot = strchr(p, '.');
455 	if (dot == NULL) {
456 		err = got_error_fmt(GOT_ERR_BAD_PATH,
457 		    "'%s' is not a valid pack file name",
458 		    packfile_name);
459 		goto done;
460 	}
461 	if (strcmp(dot + 1, "pack") != 0) {
462 		err = got_error_fmt(GOT_ERR_BAD_PATH,
463 		    "'%s' is not a valid pack file name",
464 		    packfile_name);
465 		goto done;
466 	}
467 	*dot = '\0';
468 	if (!got_parse_sha1_digest(id.sha1, p)) {
469 		err = got_error_fmt(GOT_ERR_BAD_PATH,
470 		    "'%s' is not a valid pack file name",
471 		    packfile_name);
472 		goto done;
473 	}
474 
475 	*pack_hash = got_object_id_dup(&id);
476 	if (*pack_hash == NULL) {
477 		err = got_error_from_errno("got_object_id_dup");
478 		goto done;
479 	}
480 
481 	packfd = open(packfile_path, O_RDONLY | O_NOFOLLOW);
482 	if (packfd == -1) {
483 		err = got_error_from_errno2("open", packfile_path);
484 		goto done;
485 	}
486 
487 	*packfile = fdopen(packfd, "r");
488 	if (*packfile == NULL) {
489 		err = got_error_from_errno2("fdopen", packfile_path);
490 		goto done;
491 	}
492 	packfd = -1;
493 done:
494 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
495 		err = got_error_from_errno2("close", packfile_path);
496 	free(packfile_name);
497 	if (err) {
498 		free(*pack_hash);
499 		*pack_hash = NULL;
500 	}
501 	return err;
502 }
503 
504 const struct got_error *
505 got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash,
506     struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg,
507     got_cancel_cb cancel_cb, void *cancel_arg)
508 {
509 	const struct got_error *err = NULL;
510 	char *id_str = NULL, *idxpath = NULL, *packpath = NULL;
511 	struct got_packidx *packidx = NULL;
512 	struct got_pack *pack = NULL;
513 	uint32_t nobj, i;
514 
515 	err = got_object_id_str(&id_str, pack_hash);
516 	if (err)
517 		goto done;
518 
519 	if (asprintf(&packpath, "%s/pack-%s.pack",
520 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
521 		err = got_error_from_errno("asprintf");
522 		goto done;
523 	}
524 	if (asprintf(&idxpath, "%s/pack-%s.idx",
525 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
526 		err = got_error_from_errno("asprintf");
527 		goto done;
528 	}
529 
530 	err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1);
531 	if (err)
532 		goto done;
533 
534 	err = got_repo_cache_pack(&pack, repo, packpath, packidx);
535 	if (err)
536 		goto done;
537 
538 	nobj = be32toh(packidx->hdr.fanout_table[0xff]);
539 	for (i = 0; i < nobj; i++) {
540 		struct got_packidx_object_id *oid;
541 		struct got_object_id id, base_id;
542 		off_t offset, base_offset = 0;
543 		uint8_t type;
544 		uint64_t size;
545 		size_t tslen, len;
546 
547 		if (cancel_cb) {
548 			err = cancel_cb(cancel_arg);
549 			if (err)
550 				break;
551 		}
552 		oid = &packidx->hdr.sorted_ids[i];
553 		memcpy(id.sha1, oid->sha1, SHA1_DIGEST_LENGTH);
554 
555 		offset = got_packidx_get_object_offset(packidx, i);
556 		if (offset == -1) {
557 			err = got_error(GOT_ERR_BAD_PACKIDX);
558 			goto done;
559 		}
560 
561 		err = got_pack_parse_object_type_and_size(&type, &size, &tslen,
562 		    pack, offset);
563 		if (err)
564 			goto done;
565 
566 		switch (type) {
567 		case GOT_OBJ_TYPE_OFFSET_DELTA:
568 			err = got_pack_parse_offset_delta(&base_offset, &len,
569 			    pack, offset, tslen);
570 			if (err)
571 				goto done;
572 			break;
573 		case GOT_OBJ_TYPE_REF_DELTA:
574 			err = got_pack_parse_ref_delta(&base_id,
575 			    pack, offset, tslen);
576 			if (err)
577 				goto done;
578 			break;
579 		}
580 		err = (*list_cb)(list_arg, &id, type, offset, size,
581 		    base_offset, &base_id);
582 		if (err)
583 			goto done;
584 	}
585 
586 done:
587 	free(id_str);
588 	free(idxpath);
589 	free(packpath);
590 	if (packidx)
591 		got_packidx_close(packidx);
592 	return err;
593 }
594 
595 static const struct got_error *
596 get_loose_object_ids(struct got_object_idset **loose_ids, off_t *ondisk_size,
597     got_cleanup_progress_cb progress_cb, void *progress_arg,
598     struct got_repository *repo)
599 {
600 	const struct got_error *err = NULL;
601 	char *path_objects = NULL, *path = NULL;
602 	DIR *dir = NULL;
603 	struct got_object *obj = NULL;
604 	struct got_object_id id;
605 	int i, fd = -1;
606 	struct stat sb;
607 
608 	*ondisk_size = 0;
609 	*loose_ids = got_object_idset_alloc();
610 	if (*loose_ids == NULL)
611 		return got_error_from_errno("got_object_idset_alloc");
612 
613 	path_objects = got_repo_get_path_objects(repo);
614 	if (path_objects == NULL) {
615 		err = got_error_from_errno("got_repo_get_path_objects");
616 		goto done;
617 	}
618 
619 	for (i = 0; i <= 0xff; i++) {
620 		struct dirent *dent;
621 
622 		if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
623 			err = got_error_from_errno("asprintf");
624 			break;
625 		}
626 
627 		dir = opendir(path);
628 		if (dir == NULL) {
629 			if (errno == ENOENT) {
630 				err = NULL;
631 				continue;
632 			}
633 			err = got_error_from_errno2("opendir", path);
634 			break;
635 		}
636 
637 		while ((dent = readdir(dir)) != NULL) {
638 			char *id_str;
639 
640 			if (strcmp(dent->d_name, ".") == 0 ||
641 			    strcmp(dent->d_name, "..") == 0)
642 				continue;
643 
644 			if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
645 				err = got_error_from_errno("asprintf");
646 				goto done;
647 			}
648 
649 			memset(&id, 0, sizeof(id));
650 			if (!got_parse_sha1_digest(id.sha1, id_str)) {
651 				free(id_str);
652 				continue;
653 			}
654 			free(id_str);
655 
656 			err = got_object_open_loose_fd(&fd, &id, repo);
657 			if (err)
658 				goto done;
659 			if (fstat(fd, &sb) == -1) {
660 				err = got_error_from_errno("fstat");
661 				goto done;
662 			}
663 			err = got_object_read_header_privsep(&obj, &id, repo,
664 			    fd);
665 			if (err)
666 				goto done;
667 			fd = -1; /* already closed */
668 
669 			switch (obj->type) {
670 			case GOT_OBJ_TYPE_COMMIT:
671 			case GOT_OBJ_TYPE_TREE:
672 			case GOT_OBJ_TYPE_BLOB:
673 			case GOT_OBJ_TYPE_TAG:
674 				break;
675 			default:
676 				err = got_error_fmt(GOT_ERR_OBJ_TYPE,
677 				    "%d", obj->type);
678 				goto done;
679 			}
680 			got_object_close(obj);
681 			obj = NULL;
682 			(*ondisk_size) += sb.st_size;
683 			err = got_object_idset_add(*loose_ids, &id, NULL);
684 			if (err)
685 				goto done;
686 			if (progress_cb) {
687 				err = progress_cb(progress_arg,
688 				    got_object_idset_num_elements(*loose_ids),
689 				    -1, -1);
690 				if (err)
691 					goto done;
692 			}
693 		}
694 
695 		if (closedir(dir) != 0) {
696 			err = got_error_from_errno("closedir");
697 			goto done;
698 		}
699 		dir = NULL;
700 
701 		free(path);
702 		path = NULL;
703 	}
704 done:
705 	if (dir && closedir(dir) != 0 && err == NULL)
706 		err = got_error_from_errno("closedir");
707 	if (fd != -1 && close(fd) == -1 && err == NULL)
708 		err = got_error_from_errno("close");
709 	if (err) {
710 		got_object_idset_free(*loose_ids);
711 		*loose_ids = NULL;
712 	}
713 	if (obj)
714 		got_object_close(obj);
715 	free(path_objects);
716 	free(path);
717 	return err;
718 }
719 
720 static const struct got_error *
721 preserve_loose_object(struct got_object_idset *loose_ids,
722     struct got_object_id *id, struct got_repository *repo, int *npacked)
723 {
724 	const struct got_error *err = NULL;
725 	struct got_object *obj;
726 
727 	if (!got_object_idset_contains(loose_ids, id))
728 		return NULL;
729 
730 	/*
731 	 * Try to open this object from a pack file. This ensures that
732 	 * we do in fact have a valid packed copy of the object. Otherwise
733 	 * we should not delete the loose representation of this object.
734 	 */
735 	err = got_object_open_packed(&obj, id, repo);
736 	if (err == NULL) {
737 		got_object_close(obj);
738 		/*
739 		 * The object is referenced and packed.
740 		 * We can purge the redundantly stored loose object.
741 		 */
742 		(*npacked)++;
743 		return NULL;
744 	} else if (err->code != GOT_ERR_NO_OBJ)
745 		return err;
746 
747 	/*
748 	 * This object is referenced and not packed.
749 	 * Remove it from our purge set.
750 	 */
751 	return got_object_idset_remove(NULL, loose_ids, id);
752 }
753 
754 static const struct got_error *
755 load_tree_entries(struct got_object_id_queue *ids,
756     struct got_object_idset *loose_ids,
757     struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
758     const char *dpath, struct got_repository *repo, int *npacked,
759     got_cancel_cb cancel_cb, void *cancel_arg)
760 {
761 	const struct got_error *err;
762 	struct got_tree_object *tree;
763 	char *p = NULL;
764 	int i;
765 
766 	err = got_object_open_as_tree(&tree, repo, tree_id);
767 	if (err)
768 		return err;
769 
770 	for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
771 		struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
772 		struct got_object_id *id = got_tree_entry_get_id(e);
773 		mode_t mode = got_tree_entry_get_mode(e);
774 
775 		if (cancel_cb) {
776 			err = (*cancel_cb)(cancel_arg);
777 			if (err)
778 				break;
779 		}
780 
781 		if (got_object_tree_entry_is_symlink(e) ||
782 		    got_object_tree_entry_is_submodule(e) ||
783 		    got_object_idset_contains(traversed_ids, id))
784 			continue;
785 
786 		if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "",
787 		    got_tree_entry_get_name(e)) == -1) {
788 			err = got_error_from_errno("asprintf");
789 			break;
790 		}
791 
792 		if (S_ISDIR(mode)) {
793 			struct got_object_qid *qid;
794 			err = got_object_qid_alloc(&qid, id);
795 			if (err)
796 				break;
797 			STAILQ_INSERT_TAIL(ids, qid, entry);
798 		} else if (S_ISREG(mode)) {
799 			/* This blob is referenced. */
800 			err = preserve_loose_object(loose_ids, id, repo,
801 			    npacked);
802 			if (err)
803 				break;
804 			err = got_object_idset_add(traversed_ids, id, NULL);
805 			if (err)
806 				break;
807 			
808 		}
809 		free(p);
810 		p = NULL;
811 	}
812 
813 	got_object_tree_close(tree);
814 	free(p);
815 	return err;
816 }
817 
818 static const struct got_error *
819 load_tree(struct got_object_idset *loose_ids,
820     struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
821     const char *dpath, struct got_repository *repo, int *npacked,
822     got_cancel_cb cancel_cb, void *cancel_arg)
823 {
824 	const struct got_error *err = NULL;
825 	struct got_object_id_queue tree_ids;
826 	struct got_object_qid *qid;
827 
828 	err = got_object_qid_alloc(&qid, tree_id);
829 	if (err)
830 		return err;
831 
832 	STAILQ_INIT(&tree_ids);
833 	STAILQ_INSERT_TAIL(&tree_ids, qid, entry);
834 
835 	while (!STAILQ_EMPTY(&tree_ids)) {
836 		if (cancel_cb) {
837 			err = (*cancel_cb)(cancel_arg);
838 			if (err)
839 				break;
840 		}
841 
842 		qid = STAILQ_FIRST(&tree_ids);
843 		STAILQ_REMOVE_HEAD(&tree_ids, entry);
844 
845 		if (got_object_idset_contains(traversed_ids, qid->id)) {
846 			got_object_qid_free(qid);
847 			continue;
848 		}
849 
850 		err = got_object_idset_add(traversed_ids, qid->id, NULL);
851 		if (err) {
852 			got_object_qid_free(qid);
853 			break;
854 		}
855 
856 		/* This tree is referenced. */
857 		err = preserve_loose_object(loose_ids, qid->id, repo, npacked);
858 		if (err)
859 			break;
860 
861 		err = load_tree_entries(&tree_ids, loose_ids, traversed_ids,
862 		    qid->id, dpath, repo, npacked, cancel_cb, cancel_arg);
863 		got_object_qid_free(qid);
864 		if (err)
865 			break;
866 	}
867 
868 	got_object_id_queue_free(&tree_ids);
869 	return err;
870 }
871 
872 static const struct got_error *
873 load_commit_or_tag(struct got_object_idset *loose_ids, int *ncommits,
874     int *npacked, struct got_object_idset *traversed_ids,
875     struct got_object_id *id, struct got_repository *repo,
876     got_cleanup_progress_cb progress_cb, void *progress_arg, int nloose,
877     got_cancel_cb cancel_cb, void *cancel_arg)
878 {
879 	const struct got_error *err;
880 	struct got_commit_object *commit = NULL;
881 	struct got_tag_object *tag = NULL;
882 	struct got_object_id *tree_id = NULL;
883 	struct got_object_id_queue ids;
884 	struct got_object_qid *qid;
885 	int obj_type;
886 
887 	err = got_object_qid_alloc(&qid, id);
888 	if (err)
889 		return err;
890 
891 	STAILQ_INIT(&ids);
892 	STAILQ_INSERT_TAIL(&ids, qid, entry);
893 
894 	while (!STAILQ_EMPTY(&ids)) {
895 		if (cancel_cb) {
896 			err = (*cancel_cb)(cancel_arg);
897 			if (err)
898 				break;
899 		}
900 
901 		qid = STAILQ_FIRST(&ids);
902 		STAILQ_REMOVE_HEAD(&ids, entry);
903 
904 		if (got_object_idset_contains(traversed_ids, qid->id)) {
905 			got_object_qid_free(qid);
906 			qid = NULL;
907 			continue;
908 		}
909 
910 		err = got_object_idset_add(traversed_ids, qid->id, NULL);
911 		if (err)
912 			break;
913 
914 		/* This commit or tag is referenced. */
915 		err = preserve_loose_object(loose_ids, qid->id, repo, npacked);
916 		if (err)
917 			break;
918 
919 		err = got_object_get_type(&obj_type, repo, qid->id);
920 		if (err)
921 			break;
922 		switch (obj_type) {
923 		case GOT_OBJ_TYPE_COMMIT:
924 			err = got_object_open_as_commit(&commit, repo, qid->id);
925 			if (err)
926 				goto done;
927 			break;
928 		case GOT_OBJ_TYPE_TAG:
929 			err = got_object_open_as_tag(&tag, repo, qid->id);
930 			if (err)
931 				goto done;
932 			break;
933 		default:
934 			/* should not happen */
935 			err = got_error(GOT_ERR_OBJ_TYPE);
936 			goto done;
937 		}
938 
939 		/* Find a tree object to scan. */
940 		if (commit) {
941 			tree_id = got_object_commit_get_tree_id(commit);
942 		} else if (tag) {
943 			obj_type = got_object_tag_get_object_type(tag);
944 			switch (obj_type) {
945 			case GOT_OBJ_TYPE_COMMIT:
946 				err = got_object_open_as_commit(&commit, repo,
947 				    got_object_tag_get_object_id(tag));
948 				if (err)
949 					goto done;
950 				tree_id = got_object_commit_get_tree_id(commit);
951 				break;
952 			case GOT_OBJ_TYPE_TREE:
953 				tree_id = got_object_tag_get_object_id(tag);
954 				break;
955 			default:
956 				/*
957 				 * Tag points at something other than a
958 				 * commit or tree. Leave this weird tag object
959 				 * and the object it points to on disk.
960 				 */
961 				err = got_object_idset_remove(NULL, loose_ids,
962 				    qid->id);
963 				if (err && err->code != GOT_ERR_NO_OBJ)
964 					goto done;
965 				err = got_object_idset_remove(NULL, loose_ids,
966 				    got_object_tag_get_object_id(tag));
967 				if (err && err->code != GOT_ERR_NO_OBJ)
968 					goto done;
969 				err = NULL;
970 				break;
971 			}
972 		}
973 
974 		if (tree_id) {
975 			err = load_tree(loose_ids, traversed_ids, tree_id, "",
976 			    repo, npacked, cancel_cb, cancel_arg);
977 			if (err)
978 				break;
979 		}
980 
981 		if (commit || tag)
982 			(*ncommits)++; /* scanned tags are counted as commits */
983 
984 		if (progress_cb) {
985 			err = progress_cb(progress_arg, nloose, *ncommits, -1);
986 			if (err)
987 				break;
988 		}
989  
990 		if (commit) {
991 			/* Find parent commits to scan. */
992 			const struct got_object_id_queue *parent_ids;
993 			parent_ids = got_object_commit_get_parent_ids(commit);
994 			err = got_object_id_queue_copy(parent_ids, &ids);
995 			if (err)
996 				break;
997 			got_object_commit_close(commit);
998 			commit = NULL;
999 		}
1000 		if (tag) {
1001 			got_object_tag_close(tag);
1002 			tag = NULL;
1003 		}
1004 		got_object_qid_free(qid);
1005 		qid = NULL;
1006 	}
1007 done:
1008 	if (qid)
1009 		got_object_qid_free(qid);
1010 	if (commit)
1011 		got_object_commit_close(commit);
1012 	if (tag)
1013 		got_object_tag_close(tag);
1014 	got_object_id_queue_free(&ids);
1015 	return err;
1016 }
1017 
1018 struct purge_loose_object_arg {
1019 	struct got_repository *repo;
1020 	got_cleanup_progress_cb progress_cb;
1021 	void *progress_arg;
1022 	int nloose;
1023 	int ncommits;
1024 	int npurged;
1025 	off_t size_purged;
1026 	int dry_run;
1027 	time_t max_mtime;
1028 	int ignore_mtime;
1029 };
1030 
1031 static const struct got_error *
1032 purge_loose_object(struct got_object_id *id, void *data, void *arg)
1033 {
1034 	struct purge_loose_object_arg *a = arg;
1035 	const struct got_error *err, *unlock_err = NULL;
1036 	char *path = NULL;
1037 	int fd = -1;
1038 	struct stat sb;
1039 	struct got_lockfile *lf = NULL;
1040 
1041 	err = got_object_get_path(&path, id, a->repo);
1042 	if (err)
1043 		return err;
1044 
1045 	err = got_object_open_loose_fd(&fd, id, a->repo);
1046 	if (err)
1047 		goto done;
1048 
1049 	if (fstat(fd, &sb) == -1) {
1050 		err = got_error_from_errno("fstat");
1051 		goto done;
1052 	}
1053 
1054 	/*
1055 	 * Do not delete objects which are younger than our maximum
1056 	 * modification time threshold. This prevents a race where
1057 	 * new objects which are being added to the repository
1058 	 * concurrently would be deleted.
1059 	 */
1060 	if (a->ignore_mtime || sb.st_mtime <= a->max_mtime) {
1061 		if (!a->dry_run) {
1062 			err = got_lockfile_lock(&lf, path, -1);
1063 			if (err)
1064 				goto done;
1065 			if (unlink(path) == -1) {
1066 				err = got_error_from_errno2("unlink", path);
1067 				goto done;
1068 			}
1069 		}
1070 
1071 		a->npurged++;
1072 		a->size_purged += sb.st_size;
1073 		if (a->progress_cb) {
1074 			err = a->progress_cb(a->progress_arg, a->nloose,
1075 			    a->ncommits, a->npurged);
1076 		}
1077 	}
1078 done:
1079 	if (fd != -1 && close(fd) == -1 && err == NULL)
1080 		err = got_error_from_errno("close");
1081 	free(path);
1082 	if (lf)
1083 		unlock_err = got_lockfile_unlock(lf, -1);
1084 	return err ? err : unlock_err;
1085 }
1086 
1087 const struct got_error *
1088 got_repo_purge_unreferenced_loose_objects(struct got_repository *repo,
1089     off_t *size_before, off_t *size_after, int *npacked, int dry_run,
1090     int ignore_mtime, got_cleanup_progress_cb progress_cb, void *progress_arg,
1091     got_cancel_cb cancel_cb, void *cancel_arg)
1092 {
1093 	const struct got_error *err;
1094 	struct got_object_idset *loose_ids;
1095 	struct got_object_idset *traversed_ids;
1096 	struct got_object_id **referenced_ids;
1097 	int i, nreferenced, nloose, ncommits = 0;
1098 	struct got_reflist_head refs;
1099 	struct got_reflist_entry *re;
1100 	struct purge_loose_object_arg arg;
1101 	time_t max_mtime = 0;
1102 
1103 	TAILQ_INIT(&refs);
1104 
1105 	*size_before = 0;
1106 	*size_after = 0;
1107 	*npacked = 0;
1108 
1109 	err = get_loose_object_ids(&loose_ids, size_before,
1110 	    progress_cb, progress_arg, repo);
1111 	if (err)
1112 		return err;
1113 	nloose = got_object_idset_num_elements(loose_ids);
1114 	if (nloose == 0) {
1115 		got_object_idset_free(loose_ids);
1116 		return NULL;
1117 	}
1118 
1119 	traversed_ids = got_object_idset_alloc();
1120 	if (traversed_ids == NULL) {
1121 		err = got_error_from_errno("got_object_idset_alloc");
1122 		goto done;
1123 	}
1124 
1125 	err = got_ref_list(&refs, repo, "", got_ref_cmp_by_name, NULL);
1126 	if (err)
1127 		goto done;
1128 	if (!ignore_mtime) {
1129 		TAILQ_FOREACH(re, &refs, entry) {
1130 			time_t mtime = got_ref_get_mtime(re->ref);
1131 			if (mtime > max_mtime)
1132 				max_mtime = mtime;
1133 		}
1134 		/*
1135 		 * For safety, keep objects created within 10 minutes
1136 		 * before the youngest reference was created.
1137 		 */
1138 		if (max_mtime >= 600)
1139 			max_mtime -= 600;
1140 	}
1141 
1142 	err = get_reflist_object_ids(&referenced_ids, &nreferenced,
1143 	    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
1144 	    &refs, repo, cancel_cb, cancel_arg);
1145 	if (err)
1146 		goto done;
1147 
1148 	for (i = 0; i < nreferenced; i++) {
1149 		struct got_object_id *id = referenced_ids[i];
1150 		err = load_commit_or_tag(loose_ids, &ncommits, npacked,
1151 		    traversed_ids, id, repo, progress_cb, progress_arg, nloose, 
1152 		    cancel_cb, cancel_arg);
1153 		if (err)
1154 			goto done;
1155 	}
1156 
1157 	/* Produce a final progress report in case no objects can be purged. */
1158 	if (got_object_idset_num_elements(loose_ids) == 0 && progress_cb) {
1159 		err = progress_cb(progress_arg, nloose, ncommits, 0);
1160 		if (err)
1161 			goto done;
1162 	}
1163 
1164 	/* Any remaining loose objects are unreferenced and can be purged. */
1165 	arg.repo = repo;
1166 	arg.progress_arg = progress_arg;
1167 	arg.progress_cb = progress_cb;
1168 	arg.nloose = nloose;
1169 	arg.npurged = 0;
1170 	arg.size_purged = 0;
1171 	arg.ncommits = ncommits;
1172 	arg.dry_run = dry_run;
1173 	arg.max_mtime = max_mtime;
1174 	arg.ignore_mtime = ignore_mtime;
1175 	err = got_object_idset_for_each(loose_ids, purge_loose_object, &arg);
1176 	if (err)
1177 		goto done;
1178 	*size_after = *size_before - arg.size_purged;
1179 done:
1180 	got_object_idset_free(loose_ids);
1181 	got_object_idset_free(traversed_ids);
1182 	return err;
1183 }
1184 
1185 static const struct got_error *
1186 remove_packidx(int dir_fd, const char *relpath)
1187 {
1188 	const struct got_error *err, *unlock_err;
1189 	struct got_lockfile *lf;
1190 
1191 	err = got_lockfile_lock(&lf, relpath, dir_fd);
1192 	if (err)
1193 		return err;
1194 	if (unlinkat(dir_fd, relpath, 0) == -1)
1195 		err = got_error_from_errno("unlinkat");
1196 	unlock_err = got_lockfile_unlock(lf, dir_fd);
1197 	return err ? err : unlock_err;
1198 }
1199 
1200 const struct got_error *
1201 got_repo_remove_lonely_packidx(struct got_repository *repo, int dry_run,
1202     got_lonely_packidx_progress_cb progress_cb, void *progress_arg,
1203     got_cancel_cb cancel_cb, void *cancel_arg)
1204 {
1205 	const struct got_error *err;
1206 	DIR *packdir = NULL;
1207 	struct dirent *dent;
1208 	char *pack_relpath = NULL;
1209 	int packdir_fd;
1210 	struct stat sb;
1211 
1212 	packdir_fd = openat(got_repo_get_fd(repo),
1213 	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
1214 	if (packdir_fd == -1) {
1215 		if (errno == ENOENT)
1216 			return NULL;
1217 		return got_error_from_errno_fmt("openat: %s/%s",
1218 		    got_repo_get_path_git_dir(repo),
1219 		    GOT_OBJECTS_PACK_DIR);
1220 	}
1221 
1222 	packdir = fdopendir(packdir_fd);
1223 	if (packdir == NULL) {
1224 		err = got_error_from_errno("fdopendir");
1225 		goto done;
1226 	}
1227 
1228 	while ((dent = readdir(packdir)) != NULL) {
1229 		if (cancel_cb) {
1230 			err = cancel_cb(cancel_arg);
1231 			if (err)
1232 				goto done;
1233 		}
1234 
1235 		if (!got_repo_is_packidx_filename(dent->d_name,
1236 		    strlen(dent->d_name)))
1237 			continue;
1238 
1239 		err = got_packidx_get_packfile_path(&pack_relpath,
1240 		    dent->d_name);
1241 		if (err)
1242 			goto done;
1243 
1244 		if (fstatat(packdir_fd, pack_relpath, &sb, 0) != -1) {
1245 			free(pack_relpath);
1246 			pack_relpath = NULL;
1247 			continue;
1248 		}
1249 		if (errno != ENOENT) {
1250 			err = got_error_from_errno_fmt("fstatat: %s/%s/%s",
1251 			    got_repo_get_path_git_dir(repo),
1252 			    GOT_OBJECTS_PACK_DIR,
1253 			    pack_relpath);
1254 			goto done;
1255 		}
1256 
1257 		if (!dry_run) {
1258 			err = remove_packidx(packdir_fd, dent->d_name);
1259 			if (err)
1260 				goto done;
1261 		}
1262 		if (progress_cb) {
1263 			char *path;
1264 			if (asprintf(&path, "%s/%s/%s",
1265 			    got_repo_get_path_git_dir(repo),
1266 			    GOT_OBJECTS_PACK_DIR,
1267 			    dent->d_name) == -1) {
1268 				err = got_error_from_errno("asprintf");
1269 				goto done;
1270 			}
1271 			err = progress_cb(progress_arg, path);
1272 			free(path);
1273 			if (err)
1274 				goto done;
1275 		}
1276 		free(pack_relpath);
1277 		pack_relpath = NULL;
1278 	}
1279 done:
1280 	if (packdir && closedir(packdir) != 0 && err == NULL)
1281 		err = got_error_from_errno("closedir");
1282 	free(pack_relpath);
1283 	return err;
1284 }