commit 6322ae439e52def47d663fb644a96236c81ad92d
from: Omar Polo <op@omarpolo.com>
via: Thomas Adam <thomas@xteddy.org>
date: Tue Jul 11 21:54:18 2023 UTC

take reachability in consideration when cleaning redundant packfiles

This wraps the cleaning up of loose objects and redundant pack files
under a new functions, making the _prepare() and _complete() functions
unnecessary.  It walks the reachable commits unconditionally since
that information is always needed and adapt how we purge loose
objects after this change.

The progress function is changed too and we log `ncommits' first,
followed by the number of loose objects, followed by the rest.

Pack files are now considered redundant when all their objects are
provided by a bigger pack or are unreachable.

ok stsp

commit - cfbfa60cc722e92165914178267754f432b46ccb
commit + 6322ae439e52def47d663fb644a96236c81ad92d
blob - 6a1990e9ed9ab7446753f40e6d36eeae5ef24647
blob + afce9e2f9b1e5a863de3014628402fe232f36894
--- gotadmin/gotadmin.c
+++ gotadmin/gotadmin.c
@@ -1153,31 +1153,31 @@ struct got_cleanup_progress_arg {
 };
 
 static const struct got_error *
-cleanup_progress(void *arg, int nloose, int ncommits, int npurged,
+cleanup_progress(void *arg, int ncommits, int nloose, int npurged,
     int nredundant)
 {
 	struct got_cleanup_progress_arg *a = arg;
 	int print_loose = 0, print_commits = 0, print_purged = 0;
 	int print_redundant = 0;
 
-	if (a->last_nloose != nloose) {
-		print_loose = 1;
-		a->last_nloose = nloose;
-	}
 	if (a->last_ncommits != ncommits) {
-		print_loose = 1;
 		print_commits = 1;
 		a->last_ncommits = ncommits;
 	}
-	if (a->last_npurged != npurged) {
+	if (a->last_nloose != nloose) {
+		print_commits = 1;
 		print_loose = 1;
+		a->last_nloose = nloose;
+	}
+	if (a->last_npurged != npurged) {
 		print_commits = 1;
+		print_loose = 1;
 		print_purged = 1;
 		a->last_npurged = npurged;
 	}
 	if (a->last_nredundant != nredundant) {
-		print_loose = 1;
 		print_commits = 1;
+		print_loose = 1;
 		print_purged = 1;
 		print_redundant = 1;
 		a->last_nredundant = nredundant;
@@ -1188,11 +1188,11 @@ cleanup_progress(void *arg, int nloose, int ncommits, 
 
 	if (print_loose || print_commits || print_purged || print_redundant)
 		printf("\r");
-	if (print_loose)
-		printf("%d loose object%s", nloose, nloose == 1 ? "" : "s");
 	if (print_commits)
-		printf("; %d commit%s scanned", ncommits,
+		printf("%d commit%s scanned", ncommits,
 		    ncommits == 1 ? "" : "s");
+	if (print_loose)
+		printf("; %d loose object%s", nloose, nloose == 1 ? "" : "s");
 	if (print_purged || print_redundant) {
 		if (a->dry_run) {
 			printf("; could purge %d object%s", npurged,
@@ -1250,7 +1250,6 @@ cmd_cleanup(int argc, char *argv[])
 	int ch, dry_run = 0, verbosity = 0;
 	int ncommits = 0, nloose = 0, npacked = 0;
 	int remove_lonely_packidx = 0, ignore_mtime = 0;
-	struct got_lockfile *lock = NULL;
 	struct got_cleanup_progress_arg cpa;
 	struct got_lonely_packidx_progress_arg lpa;
 	off_t loose_before, loose_after;
@@ -1322,10 +1321,6 @@ cmd_cleanup(int argc, char *argv[])
 		goto done;
 	}
 
-	error = got_repo_cleanup_prepare(repo, &lock);
-	if (error)
-		goto done;
-
 	if (remove_lonely_packidx) {
 		memset(&lpa, 0, sizeof(lpa));
 		lpa.dry_run = dry_run;
@@ -1336,25 +1331,16 @@ cmd_cleanup(int argc, char *argv[])
 	}
 
 	memset(&cpa, 0, sizeof(cpa));
-	cpa.last_ncommits = -1;
+	cpa.last_nloose = -1;
 	cpa.last_npurged = -1;
 	cpa.last_nredundant = -1;
 	cpa.dry_run = dry_run;
 	cpa.verbosity = verbosity;
 
-	error = got_repo_purge_unreferenced_loose_objects(repo,
-	    &loose_before, &loose_after, &ncommits, &nloose, &npacked,
+	error = got_repo_cleanup(repo, &loose_before, &loose_after,
+	    &pack_before, &pack_after, &ncommits, &nloose, &npacked,
 	    dry_run, ignore_mtime, cleanup_progress, &cpa,
 	    check_cancelled, NULL);
-	if (error) {
-		if (cpa.printed_something)
-			printf("\n");
-		goto done;
-	}
-
-	error = got_repo_purge_redundant_packfiles(repo, &pack_before,
-	    &pack_after, dry_run, ncommits, nloose, npacked,
-	    cleanup_progress, &cpa, check_cancelled, NULL);
 	if (cpa.printed_something)
 		printf("\n");
 	if (error)
@@ -1397,7 +1383,6 @@ cmd_cleanup(int argc, char *argv[])
 	}
 
 done:
-	got_repo_cleanup_complete(repo, lock);
 	if (repo)
 		got_repo_close(repo);
 	if (pack_fds) {
blob - ea8cf37270d81a00cc006814c1e95430045f8277
blob + 7175f86e3c1f00b9b4d870af1638dc3f1068d951
--- include/got_repository_admin.h
+++ include/got_repository_admin.h
@@ -14,8 +14,6 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-struct got_lockfile;
-
 /* A callback function which gets invoked with progress information to print. */
 typedef const struct got_error *(*got_pack_progress_cb)(void *arg,
     int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits,
@@ -70,21 +68,9 @@ got_repo_list_pack(FILE *packfile, struct got_object_i
     struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg,
     got_cancel_cb cancel_cb, void *cancel_arg);
 
-/*
- * Prepare for removing loose objects or redundant packfiles.
- *
- * These functions do the necessary locking in order to avoid
- * concurrent operation to irremediably damage the repository.
- */
-const struct got_error *
-got_repo_cleanup_prepare(struct got_repository *, struct got_lockfile **);
-
-const struct got_error *
-got_repo_cleanup_complete(struct got_repository *, struct got_lockfile *);
-
 /* A callback function which gets invoked with cleanup information to print. */
 typedef const struct got_error *(*got_cleanup_progress_cb)(void *arg,
-    int nloose, int ncommits, int npurged, int nredundant);
+    int ncommits, int nloose, int npurged, int nredundant);
 
 /*
  * Walk objects reachable via references to determine whether any loose
@@ -92,24 +78,21 @@ typedef const struct got_error *(*got_cleanup_progress
  * unless the dry_run parameter is set.
  * Do not remove objects with a modification timestamp above an
  * implementation-defined timestamp threshold, unless ignore_mtime is set.
- * Return the disk space size occupied by loose objects before and after
- * the operation.
+ * Remove packfiles which objects are either unreachable or provided
+ * by biggest pack files.
+ * Return the disk space size occupied by loose objects and pack files
+ * before and after the operation.
  * Return the number of loose objects which are also stored in a pack file.
  */
 const struct got_error *
-got_repo_purge_unreferenced_loose_objects(struct got_repository *repo,
-    off_t *size_before, off_t *size_after, int *ncommits, int *nloose,
+got_repo_cleanup(struct got_repository *repo,
+    off_t *loose_before, off_t *loose_after,
+    off_t *pack_before, off_t *pack_after,
+    int *ncommits, int *nloose,
     int *npacked, int dry_run, int ignore_mtime,
     got_cleanup_progress_cb progress_cb, void *progress_arg,
     got_cancel_cb cancel_cb, void *cancel_arg);
 
-const struct got_error *
-got_repo_purge_redundant_packfiles(struct got_repository *repo,
-    off_t *before, off_t *size_after, int dry_run,
-    int nloose, int ncommits, int npurged,
-    got_cleanup_progress_cb progress_cb, void *progress_arg,
-    got_cancel_cb cancel_cb, void *cancel_arg);
-
 /* A callback function which gets invoked with cleanup information to print. */
 typedef const struct got_error *(*got_lonely_packidx_progress_cb)(void *arg,
     const char *path);
blob - f9c82566a2ca6c4603e1de84ce22eb4b3ac21a8b
blob + fe2e32cfde3f61a36fccce72cc9fcfd32e30488e
--- lib/repository_admin.c
+++ lib/repository_admin.c
@@ -610,9 +610,8 @@ done:
 	return err;
 }
 
-const struct got_error *
-got_repo_cleanup_prepare(struct got_repository *repo,
-    struct got_lockfile **lk)
+static const struct got_error *
+repo_cleanup_lock(struct got_repository *repo, struct got_lockfile **lk)
 {
 	const struct got_error *err;
 	char myname[_POSIX_HOST_NAME_MAX + 1];
@@ -634,22 +633,12 @@ got_repo_cleanup_prepare(struct got_repository *repo,
 		return got_error_from_errno("dprintf");
 
 	return NULL;
-}
-
-const struct got_error *
-got_repo_cleanup_complete(struct got_repository *repo,
-    struct got_lockfile *lk)
-{
-	if (lk == NULL)
-		return NULL;
-
-	return got_lockfile_unlock(lk, got_repo_get_fd(repo));
 }
 
 static const struct got_error *
 report_cleanup_progress(got_cleanup_progress_cb progress_cb,
     void *progress_arg, struct got_ratelimit *rl,
-    int nloose, int ncommits, int npurged, int nredundant)
+    int ncommits, int nloose, int npurged, int nredundant)
 {
 	const struct got_error *err;
 	int elapsed;
@@ -661,11 +650,13 @@ report_cleanup_progress(got_cleanup_progress_cb progre
 	if (err || !elapsed)
 		return err;
 
-	return progress_cb(progress_arg, nloose, ncommits, npurged, nredundant);
+	return progress_cb(progress_arg, ncommits, nloose, npurged,
+	    nredundant);
 }
 
 static const struct got_error *
-get_loose_object_ids(struct got_object_idset **loose_ids, off_t *ondisk_size,
+get_loose_object_ids(struct got_object_idset **loose_ids,
+    off_t *ondisk_size, int ncommits,
     got_cleanup_progress_cb progress_cb, void *progress_arg,
     struct got_ratelimit *rl, struct got_repository *repo)
 {
@@ -755,9 +746,9 @@ get_loose_object_ids(struct got_object_idset **loose_i
 			if (err)
 				goto done;
 			err = report_cleanup_progress(progress_cb,
-			    progress_arg, rl,
+			    progress_arg, rl, ncommits,
 			    got_object_idset_num_elements(*loose_ids),
-			    -1, -1, -1);
+			    -1, -1);
 			if (err)
 				goto done;
 		}
@@ -785,47 +776,12 @@ done:
 	free(path_objects);
 	free(path);
 	return err;
-}
-
-static const struct got_error *
-preserve_loose_object(struct got_object_idset *loose_ids,
-    struct got_object_id *id, struct got_repository *repo, int *npacked)
-{
-	const struct got_error *err = NULL;
-	struct got_object *obj;
-
-	if (!got_object_idset_contains(loose_ids, id))
-		return NULL;
-
-	/*
-	 * Try to open this object from a pack file. This ensures that
-	 * we do in fact have a valid packed copy of the object. Otherwise
-	 * we should not delete the loose representation of this object.
-	 */
-	err = got_object_open_packed(&obj, id, repo);
-	if (err == NULL) {
-		got_object_close(obj);
-		/*
-		 * The object is referenced and packed.
-		 * We can purge the redundantly stored loose object.
-		 */
-		(*npacked)++;
-		return NULL;
-	} else if (err->code != GOT_ERR_NO_OBJ)
-		return err;
-
-	/*
-	 * This object is referenced and not packed.
-	 * Remove it from our purge set.
-	 */
-	return got_object_idset_remove(NULL, loose_ids, id);
 }
 
 static const struct got_error *
 load_tree_entries(struct got_object_id_queue *ids,
-    struct got_object_idset *loose_ids,
     struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
-    const char *dpath, struct got_repository *repo, int *npacked,
+    const char *dpath, struct got_repository *repo,
     got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
@@ -867,10 +823,6 @@ load_tree_entries(struct got_object_id_queue *ids,
 			STAILQ_INSERT_TAIL(ids, qid, entry);
 		} else if (S_ISREG(mode)) {
 			/* This blob is referenced. */
-			err = preserve_loose_object(loose_ids, id, repo,
-			    npacked);
-			if (err)
-				break;
 			err = got_object_idset_add(traversed_ids, id, NULL);
 			if (err)
 				break;
@@ -885,9 +837,9 @@ load_tree_entries(struct got_object_id_queue *ids,
 }
 
 static const struct got_error *
-load_tree(struct got_object_idset *loose_ids,
-    struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
-    const char *dpath, struct got_repository *repo, int *npacked,
+load_tree(struct got_object_idset *traversed_ids,
+    struct got_object_id *tree_id,
+    const char *dpath, struct got_repository *repo,
     got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err = NULL;
@@ -922,13 +874,8 @@ load_tree(struct got_object_idset *loose_ids,
 			break;
 		}
 
-		/* This tree is referenced. */
-		err = preserve_loose_object(loose_ids, &qid->id, repo, npacked);
-		if (err)
-			break;
-
-		err = load_tree_entries(&tree_ids, loose_ids, traversed_ids,
-		    &qid->id, dpath, repo, npacked, cancel_cb, cancel_arg);
+		err = load_tree_entries(&tree_ids, traversed_ids,
+		    &qid->id, dpath, repo, cancel_cb, cancel_arg);
 		got_object_qid_free(qid);
 		if (err)
 			break;
@@ -939,12 +886,10 @@ load_tree(struct got_object_idset *loose_ids,
 }
 
 static const struct got_error *
-load_commit_or_tag(struct got_object_idset *loose_ids, int *ncommits,
-    int *npacked, struct got_object_idset *traversed_ids,
+load_commit_or_tag(int *ncommits, struct got_object_idset *traversed_ids,
     struct got_object_id *id, struct got_repository *repo,
     got_cleanup_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, int nloose,
-    got_cancel_cb cancel_cb, void *cancel_arg)
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
 	struct got_commit_object *commit = NULL;
@@ -978,11 +923,6 @@ load_commit_or_tag(struct got_object_idset *loose_ids,
 		}
 
 		err = got_object_idset_add(traversed_ids, &qid->id, NULL);
-		if (err)
-			break;
-
-		/* This commit or tag is referenced. */
-		err = preserve_loose_object(loose_ids, &qid->id, repo, npacked);
 		if (err)
 			break;
 
@@ -1027,24 +967,22 @@ load_commit_or_tag(struct got_object_idset *loose_ids,
 				/*
 				 * Tag points at something other than a
 				 * commit or tree. Leave this weird tag object
-				 * and the object it points to on disk.
+				 * and the object it points to.
 				 */
-				err = got_object_idset_remove(NULL, loose_ids,
-				    &qid->id);
-				if (err && err->code != GOT_ERR_NO_OBJ)
+				if (got_object_idset_contains(traversed_ids,
+				    got_object_tag_get_object_id(tag)))
+					break;
+				err = got_object_idset_add(traversed_ids,
+				    got_object_tag_get_object_id(tag), NULL);
+				if (err)
 					goto done;
-				err = got_object_idset_remove(NULL, loose_ids,
-				    got_object_tag_get_object_id(tag));
-				if (err && err->code != GOT_ERR_NO_OBJ)
-					goto done;
-				err = NULL;
 				break;
 			}
 		}
 
 		if (tree_id) {
-			err = load_tree(loose_ids, traversed_ids, tree_id, "",
-			    repo, npacked, cancel_cb, cancel_arg);
+			err = load_tree(traversed_ids, tree_id, "",
+			    repo, cancel_cb, cancel_arg);
 			if (err)
 				break;
 		}
@@ -1053,7 +991,7 @@ load_commit_or_tag(struct got_object_idset *loose_ids,
 			(*ncommits)++; /* scanned tags are counted as commits */
 
 		err = report_cleanup_progress(progress_cb, progress_arg, rl,
-		    nloose, *ncommits, -1, -1);
+		    *ncommits, -1, -1, -1);
 		if (err)
 			break;
 
@@ -1085,13 +1023,35 @@ done:
 	return err;
 }
 
+static const struct got_error *
+is_object_packed(int *packed, struct got_repository *repo,
+    struct got_object_id *id)
+{
+	const struct got_error *err;
+	struct got_object *obj;
+
+	*packed = 0;
+
+	err = got_object_open_packed(&obj, id, repo);
+	if (err) {
+		if (err->code == GOT_ERR_NO_OBJ)
+			err = NULL;
+		return err;
+	}
+	got_object_close(obj);
+	*packed = 1;
+	return NULL;
+}
+
 struct purge_loose_object_arg {
 	struct got_repository *repo;
 	got_cleanup_progress_cb progress_cb;
 	void *progress_arg;
 	struct got_ratelimit *rl;
+	struct got_object_idset *traversed_ids;
 	int nloose;
 	int ncommits;
+	int npacked;
 	int npurged;
 	off_t size_purged;
 	int dry_run;
@@ -1105,10 +1065,20 @@ purge_loose_object(struct got_object_id *id, void *dat
 	struct purge_loose_object_arg *a = arg;
 	const struct got_error *err, *unlock_err = NULL;
 	char *path = NULL;
-	int fd = -1;
+	int packed, fd = -1;
 	struct stat sb;
 	struct got_lockfile *lf = NULL;
 
+	err = is_object_packed(&packed, a->repo, id);
+	if (err)
+		return err;
+
+	if (!packed && got_object_idset_contains(a->traversed_ids, id))
+		return NULL;
+
+	if (packed)
+		a->npacked++;
+
 	err = got_object_get_path(&path, id, a->repo);
 	if (err)
 		return err;
@@ -1142,7 +1112,7 @@ purge_loose_object(struct got_object_id *id, void *dat
 		a->npurged++;
 		a->size_purged += sb.st_size;
 		err = report_cleanup_progress(a->progress_cb, a->progress_arg,
-		    a->rl, a->nloose, a->ncommits, a->npurged, -1);
+		    a->rl, a->ncommits, a->nloose, a->npurged, -1);
 		if (err)
 			goto done;
 	}
@@ -1155,33 +1125,21 @@ done:
 	return err ? err : unlock_err;
 }
 
-const struct got_error *
-got_repo_purge_unreferenced_loose_objects(struct got_repository *repo,
-    off_t *size_before, off_t *size_after, int *ncommits, int *nloose,
-    int *npacked, int dry_run, int ignore_mtime,
+static const struct got_error *
+repo_purge_unreferenced_loose_objects(struct got_repository *repo,
+    struct got_object_idset *traversed_ids,
+    off_t *size_before, off_t *size_after, int ncommits, int *nloose,
+    int *npacked, int *npurged, int dry_run, int ignore_mtime,
+    time_t max_mtime, struct got_ratelimit *rl,
     got_cleanup_progress_cb progress_cb, void *progress_arg,
     got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
 	struct got_object_idset *loose_ids;
-	struct got_object_idset *traversed_ids;
-	struct got_object_id **referenced_ids;
-	int i, nreferenced;
-	struct got_reflist_head refs;
-	struct got_reflist_entry *re;
 	struct purge_loose_object_arg arg;
-	time_t max_mtime = 0;
-	struct got_ratelimit rl;
 
-	TAILQ_INIT(&refs);
-	got_ratelimit_init(&rl, 0, 500);
-
-	*size_before = 0;
-	*size_after = 0;
-	*npacked = 0;
-
-	err = get_loose_object_ids(&loose_ids, size_before,
-	    progress_cb, progress_arg, &rl, repo);
+	err = get_loose_object_ids(&loose_ids, size_before, ncommits,
+	    progress_cb, progress_arg, rl, repo);
 	if (err)
 		return err;
 	*nloose = got_object_idset_num_elements(loose_ids);
@@ -1195,71 +1153,36 @@ got_repo_purge_unreferenced_loose_objects(struct got_r
 		return NULL;
 	}
 
-	traversed_ids = got_object_idset_alloc();
-	if (traversed_ids == NULL) {
-		err = got_error_from_errno("got_object_idset_alloc");
-		goto done;
-	}
-
-	err = got_ref_list(&refs, repo, "", got_ref_cmp_by_name, NULL);
-	if (err)
-		goto done;
-	if (!ignore_mtime) {
-		TAILQ_FOREACH(re, &refs, entry) {
-			time_t mtime = got_ref_get_mtime(re->ref);
-			if (mtime > max_mtime)
-				max_mtime = mtime;
-		}
-		/*
-		 * For safety, keep objects created within 10 minutes
-		 * before the youngest reference was created.
-		 */
-		if (max_mtime >= 600)
-			max_mtime -= 600;
-	}
-
-	err = get_reflist_object_ids(&referenced_ids, &nreferenced,
-	    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
-	    &refs, repo, cancel_cb, cancel_arg);
-	if (err)
-		goto done;
-
-	for (i = 0; i < nreferenced; i++) {
-		struct got_object_id *id = referenced_ids[i];
-		err = load_commit_or_tag(loose_ids, ncommits, npacked,
-		    traversed_ids, id, repo, progress_cb, progress_arg, &rl,
-		    *nloose, cancel_cb, cancel_arg);
-		if (err)
-			goto done;
-	}
-
-	/* Any remaining loose objects are unreferenced and can be purged. */
+	memset(&arg, 0, sizeof(arg));
 	arg.repo = repo;
 	arg.progress_arg = progress_arg;
 	arg.progress_cb = progress_cb;
-	arg.rl = &rl;
+	arg.rl = rl;
+	arg.traversed_ids = traversed_ids;
 	arg.nloose = *nloose;
+	arg.npacked = 0;
 	arg.npurged = 0;
 	arg.size_purged = 0;
-	arg.ncommits = *ncommits;
 	arg.dry_run = dry_run;
 	arg.max_mtime = max_mtime;
 	arg.ignore_mtime = ignore_mtime;
 	err = got_object_idset_for_each(loose_ids, purge_loose_object, &arg);
 	if (err)
 		goto done;
+
 	*size_after = *size_before - arg.size_purged;
+	*npacked = arg.npacked;
+	*npurged = arg.npurged;
 
 	/* Produce a final progress report. */
 	if (progress_cb) {
-		err = progress_cb(progress_arg, *nloose, *ncommits, arg.npurged,
-		    -1);
+		err = progress_cb(progress_arg, ncommits, *nloose,
+		    arg.npurged, -1);
 		if (err)
 			goto done;
 	}
 done:
 	got_object_idset_free(loose_ids);
-	got_object_idset_free(traversed_ids);
 	return err;
 }
 
@@ -1325,6 +1248,7 @@ purge_redundant_pack(struct got_repository *repo, cons
 
 static const struct got_error *
 pack_is_redundant(int *redundant, struct got_repository *repo,
+    struct got_object_idset *traversed_ids,
     const char *packidx_path, struct got_object_idset *idset)
 {
 	const struct got_error *err;
@@ -1349,6 +1273,9 @@ pack_is_redundant(int *redundant, struct got_repositor
 		if (got_object_idset_contains(idset, &id))
 			continue;
 
+		if (!got_object_idset_contains(traversed_ids, &id))
+			continue;
+
 		*redundant = 0;
 		err = got_object_idset_add(idset, &id, NULL);
 		if (err)
@@ -1377,10 +1304,11 @@ pack_info_cmp(const void *a, const void *b)
 	return 1;
 }
 
-const struct got_error *
-got_repo_purge_redundant_packfiles(struct got_repository *repo,
+static const struct got_error *
+repo_purge_redundant_packfiles(struct got_repository *repo,
+    struct got_object_idset *traversed_ids,
     off_t *size_before, off_t *size_after, int dry_run,
-    int nloose, int ncommits, int npurged,
+    int nloose, int ncommits, int npurged, struct got_ratelimit *rl,
     got_cleanup_progress_cb progress_cb, void *progress_arg,
     got_cancel_cb cancel_cb, void *cancel_arg)
 {
@@ -1391,13 +1319,7 @@ got_repo_purge_redundant_packfiles(struct got_reposito
 	struct got_pathlist_entry *pe;
 	size_t i, npacks;
 	int remove, redundant_packs = 0;
-	struct got_ratelimit rl;
 
-	got_ratelimit_init(&rl, 0, 500);
-
-	*size_before = 0;
-	*size_after = 0;
-
 	npacks = 0;
 	TAILQ_FOREACH(pe, &repo->packidx_paths, entry)
 		npacks++;
@@ -1434,7 +1356,8 @@ got_repo_purge_redundant_packfiles(struct got_reposito
 				break;
 		}
 
-		err = pack_is_redundant(&remove, repo, sorted[i].path, idset);
+		err = pack_is_redundant(&remove, repo, traversed_ids,
+		    sorted[i].path, idset);
 		if (err)
 			goto done;
 		err = purge_redundant_pack(repo, sorted[i].path, dry_run,
@@ -1444,14 +1367,14 @@ got_repo_purge_redundant_packfiles(struct got_reposito
 		if (!remove)
 			continue;
 		err = report_cleanup_progress(progress_cb, progress_arg,
-		    &rl, nloose, ncommits, npurged, ++redundant_packs);
+		    rl, ncommits, nloose, npurged, ++redundant_packs);
 		if (err)
 			goto done;
 	}
 
 	/* Produce a final progress report. */
 	if (progress_cb) {
-		err = progress_cb(progress_arg, nloose, ncommits, npurged,
+		err = progress_cb(progress_arg, ncommits, nloose, npurged,
 		    redundant_packs);
 		if (err)
 			goto done;
@@ -1460,6 +1383,102 @@ got_repo_purge_redundant_packfiles(struct got_reposito
 	free(sorted);
 	if (idset)
 		got_object_idset_free(idset);
+	return err;
+}
+
+const struct got_error *
+got_repo_cleanup(struct got_repository *repo,
+    off_t *loose_before, off_t *loose_after,
+    off_t *pack_before, off_t *pack_after,
+    int *ncommits, int *nloose, int *npacked, int dry_run, int ignore_mtime,
+    got_cleanup_progress_cb progress_cb, void *progress_arg,
+    got_cancel_cb cancel_cb, void *cancel_arg)
+{
+	const struct got_error *unlock_err, *err = NULL;
+	struct got_lockfile *lk = NULL;
+	struct got_ratelimit rl;
+	struct got_reflist_head refs;
+	struct got_object_idset *traversed_ids = NULL;
+	struct got_reflist_entry *re;
+	struct got_object_id **referenced_ids;
+	int i, nreferenced;
+	int npurged = 0;
+	time_t max_mtime = 0;
+
+	TAILQ_INIT(&refs);
+	got_ratelimit_init(&rl, 0, 500);
+
+	*loose_before = 0;
+	*loose_after = 0;
+	*pack_before = 0;
+	*pack_after = 0;
+	*ncommits = 0;
+	*nloose = 0;
+	*npacked = 0;
+
+	err = repo_cleanup_lock(repo, &lk);
+	if (err)
+		return err;
+
+	traversed_ids = got_object_idset_alloc();
+	if (traversed_ids == NULL) {
+		err = got_error_from_errno("got_object_idset_alloc");
+		goto done;
+	}
+
+	err = got_ref_list(&refs, repo, "", got_ref_cmp_by_name, NULL);
+	if (err)
+		goto done;
+	if (!ignore_mtime) {
+		TAILQ_FOREACH(re, &refs, entry) {
+			time_t mtime = got_ref_get_mtime(re->ref);
+			if (mtime > max_mtime)
+				max_mtime = mtime;
+		}
+		/*
+		 * For safety, keep objects created within 10 minutes
+		 * before the youngest reference was created.
+		 */
+		if (max_mtime >= 600)
+			max_mtime -= 600;
+	}
+
+	err = get_reflist_object_ids(&referenced_ids, &nreferenced,
+	    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
+	    &refs, repo, cancel_cb, cancel_arg);
+	if (err)
+		goto done;
+
+	for (i = 0; i < nreferenced; i++) {
+		struct got_object_id *id = referenced_ids[i];
+		err = load_commit_or_tag(ncommits, traversed_ids,
+		    id, repo, progress_cb, progress_arg, &rl,
+		    cancel_cb, cancel_arg);
+		if (err)
+			goto done;
+	}
+
+	err = repo_purge_unreferenced_loose_objects(repo, traversed_ids,
+	    loose_before, loose_after, *ncommits, nloose, npacked, &npurged,
+	    dry_run, ignore_mtime, max_mtime, &rl, progress_cb, progress_arg,
+	    cancel_cb, cancel_arg);
+	if (err)
+		goto done;
+
+	err = repo_purge_redundant_packfiles(repo, traversed_ids,
+	    pack_before, pack_after, dry_run, *nloose, *ncommits, npurged,
+	    &rl, progress_cb, progress_arg, cancel_cb, cancel_arg);
+	if (err)
+		goto done;
+
+ done:
+	if (lk) {
+		unlock_err = got_lockfile_unlock(lk, got_repo_get_fd(repo));
+		if (err == NULL)
+			err = unlock_err;
+	}
+	if (traversed_ids)
+		got_object_idset_free(traversed_ids);
 	return err;
 }
 
blob - f6ddbb9657a29cfb443245457d038aee5b06d364
blob + 757a755adad5a2bca886449ea29bc283017dd221
--- regress/cmdline/cleanup.sh
+++ regress/cmdline/cleanup.sh
@@ -265,17 +265,16 @@ test_cleanup_redundant_pack_files() {
 	# create two packs with all the objects
 	gotadmin pack -a -r "$testroot/repo" >/dev/null
 	gotadmin pack -a -r "$testroot/repo" >/dev/null
-
-	gotadmin cleanup -r "$testroot/repo" | grep 'pack files? purged' \
-		| tail -1 > $testroot/stdout
 
-	echo "5 pack files purged" > $testroot/stdout.expected
-	if cmp -s "$testroot/stdout.expected" "$testroot/stdout"; then
-		diff -u "$testroot/stdout.expected" "$testroot/stdout"
-		test_done "$testroot" 1
-		return 1
-	fi
+	# create another one with unreachable objects
+	(cd "$testroot/repo" && git checkout -q -b tempbranch)
+	echo "modified alpha on tempbranch" >$testroot/repo/alpha
+	git_commit "$testroot/repo" -m "edit alpha on tempbranch"
+	gotadmin pack -a -r "$testroot/repo" >/dev/null
+	(cd "$testroot/repo" && git checkout -q master)
+	(cd "$testroot/repo" && got branch -d tempbranch) >/dev/null
 
+	gotadmin cleanup -q -r "$testroot/repo"
 	n=$(gotadmin info -r "$testroot/repo" | awk '/^pack files/{print $3}')
 	if [ "$n" -ne 2 ]; then
 		echo "expected 2 pack files left, $n found instead" >&2
@@ -302,16 +301,8 @@ test_cleanup_redundant_pack_files() {
 	done
 	gotadmin pack -r "$testroot/repo" >/dev/null
 
-	gotadmin cleanup -r "$testroot/repo" | grep 'pack files? purged' \
-	    | tail -1 > $testroot/stdout
+	gotadmin cleanup -q -r "$testroot/repo"
 
-	echo "0 pack files purged" > $testroot/stdout.expected
-	if cmp -s "$testroot/stdout.expected" "$testroot/stdout"; then
-		diff -u "$testroot/stdout.expected" "$testroot/stdout"
-		test_done "$testroot" 1
-		return 1
-	fi
-
 	n=$(gotadmin info -r "$testroot/repo" | awk '/^pack files/{print $3}')
 	if [ "$n" -ne 3 ]; then
 		echo "expected 3 pack files left, $n found instead" >&2
@@ -333,16 +324,7 @@ test_cleanup_redundant_pack_files() {
 
 	gotadmin pack -a -x master -r "$testroot/repo" >/dev/null
 
-	gotadmin cleanup -r "$testroot/repo" | grep 'pack files? purged' \
-		| tail -1 > $testroot/stdout
-
-	echo "6 pack files purged" > $testroot/stdout.expected
-	if cmp -s "$testroot/stdout.expected" "$testroot/stdout"; then
-		diff -u "$testroot/stdout.expected" "$testroot/stdout"
-		test_done "$testroot" 1
-		return 1
-	fi
-
+	gotadmin cleanup -q -r "$testroot/repo"
 	n=$(gotadmin info -r "$testroot/repo" | awk '/^pack files/{print $3}')
 	if [ "$n" -ne 3 ]; then
 		echo "expected 3 pack files left, $n found instead" >&2