commit de47d040c7458d8bddac77ba76a8a5e61aa8eb44 from: Stefan Sperling via: Thomas Adam date: Tue Mar 22 17:53:45 2022 UTC cache a list of known pack index files when the repository is opened Avoids overhead due to readdir calls while searching a pack index. ok op@ commit - eaf99875e8a296a7a1128d804339ed1935f8d2a0 commit + de47d040c7458d8bddac77ba76a8a5e61aa8eb44 blob - 798762e89557feb475f650e62e549abe41eb43c4 blob + 5fb86bb46e7a4329db5383980128ed5c1b627aec --- lib/got_lib_repository.h +++ lib/got_lib_repository.h @@ -51,6 +51,8 @@ struct got_repository { char *path_git_dir; int gitdir_fd; + struct got_pathlist_head packidx_paths; + /* The pack index cache speeds up search for packed objects. */ struct got_packidx *packidx_cache[GOT_PACK_CACHE_SIZE]; blob - 138499bf96babfa8fcd49411e9ae49826a51ae96 blob + 96b95856b52b6f229cbee1d31aba3cdde0b726b3 --- lib/pack_create.c +++ lib/pack_create.c @@ -487,20 +487,14 @@ static const struct got_error * find_pack_for_reuse(struct got_packidx **best_packidx, struct got_repository *repo) { - const struct got_error *err; - struct got_pathlist_head packidx_paths; + const struct got_error *err = NULL; struct got_pathlist_entry *pe; const char *best_packidx_path = NULL; int nobj_max = 0; - TAILQ_INIT(&packidx_paths); *best_packidx = NULL; - err = got_repo_list_packidx(&packidx_paths, repo); - if (err) - return err; - - TAILQ_FOREACH(pe, &packidx_paths, entry) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { const char *path_packidx = pe->path; struct got_packidx *packidx; int nobj; @@ -521,9 +515,6 @@ find_pack_for_reuse(struct got_packidx **best_packidx, repo); } - TAILQ_FOREACH(pe, &packidx_paths, entry) - free((void *)pe->path); - got_pathlist_free(&packidx_paths); return err; } blob - a7c2e0f246cc3c3568df8293679707567e5ec8e1 blob + dc7435908d610d55270466c158a0dea7033d6b19 --- lib/repository.c +++ lib/repository.c @@ -664,6 +664,7 @@ got_repo_open(struct got_repository **repop, const cha } RB_INIT(&repo->packidx_bloom_filters); + TAILQ_INIT(&repo->packidx_paths); for (i = 0; i < nitems(repo->privsep_children); i++) { memset(&repo->privsep_children[i], 0, @@ -744,6 +745,8 @@ got_repo_open(struct got_repository **repop, const cha goto done; } } + + err = got_repo_list_packidx(&repo->packidx_paths, repo); done: if (err) got_repo_close(repo); @@ -758,6 +761,7 @@ got_repo_close(struct got_repository *repo) { const struct got_error *err = NULL, *child_err; struct got_packidx_bloom_filter *bf; + struct got_pathlist_entry *pe; size_t i; for (i = 0; i < repo->pack_cache_size; i++) { @@ -818,6 +822,10 @@ got_repo_close(struct got_repository *repo) for (i = 0; i < repo->nextensions; i++) free(repo->extensions[i]); free(repo->extensions); + + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) + free((void *)pe->path); + got_pathlist_free(&repo->packidx_paths); free(repo); return err; @@ -1086,11 +1094,8 @@ got_repo_search_packidx(struct got_packidx **packidx, struct got_repository *repo, struct got_object_id *id) { const struct got_error *err; - DIR *packdir = NULL; - struct dirent *dent; - char *path_packidx; + struct got_pathlist_entry *pe; size_t i; - int packdir_fd; /* Search pack index cache. */ for (i = 0; i < repo->pack_cache_size; i++) { @@ -1117,42 +1122,13 @@ got_repo_search_packidx(struct got_packidx **packidx, } /* No luck. Search the filesystem. */ - packdir_fd = openat(got_repo_get_fd(repo), - GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC); - if (packdir_fd == -1) { - if (errno == ENOENT) - err = got_error_no_obj(id); - else - err = got_error_from_errno_fmt("openat: %s/%s", - got_repo_get_path_git_dir(repo), - GOT_OBJECTS_PACK_DIR); - goto done; - } - - packdir = fdopendir(packdir_fd); - if (packdir == NULL) { - err = got_error_from_errno("fdopendir"); - goto done; - } - - while ((dent = readdir(packdir)) != NULL) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { + const char *path_packidx = pe->path; int is_cached = 0; - if (!got_repo_is_packidx_filename(dent->d_name, - strlen(dent->d_name))) - continue; - - if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR, - dent->d_name) == -1) { - err = got_error_from_errno("asprintf"); - goto done; - } - if (!got_repo_check_packidx_bloom_filter(repo, - path_packidx, id)) { - free(path_packidx); + pe->path, id)) continue; /* object will not be found in this index */ - } for (i = 0; i < repo->pack_cache_size; i++) { if (repo->packidx_cache[i] == NULL) @@ -1163,26 +1139,19 @@ got_repo_search_packidx(struct got_packidx **packidx, break; } } - if (is_cached) { - free(path_packidx); + if (is_cached) continue; /* already searched */ - } err = got_packidx_open(packidx, got_repo_get_fd(repo), path_packidx, 0); - if (err) { - free(path_packidx); + if (err) goto done; - } err = add_packidx_bloom_filter(repo, *packidx, path_packidx); - if (err) { - free(path_packidx); + if (err) goto done; - } err = cache_packidx(repo, *packidx, path_packidx); - free(path_packidx); if (err) goto done; @@ -1195,8 +1164,6 @@ got_repo_search_packidx(struct got_packidx **packidx, err = got_error_no_obj(id); done: - if (packdir && closedir(packdir) != 0 && err == NULL) - err = got_error_from_errno("closedir"); return err; } @@ -1481,47 +1448,18 @@ match_packed_object(struct got_object_id **unique_id, struct got_repository *repo, const char *id_str_prefix, int obj_type) { const struct got_error *err = NULL; - DIR *packdir = NULL; - struct dirent *dent; - char *path_packidx; - struct got_object_id_queue matched_ids; - int packdir_fd; + struct got_object_id_queue matched_ids; + struct got_pathlist_entry *pe; STAILQ_INIT(&matched_ids); - packdir_fd = openat(got_repo_get_fd(repo), - GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC); - if (packdir_fd == -1) { - if (errno != ENOENT) { - err = got_error_from_errno2("openat", - GOT_OBJECTS_PACK_DIR); - } - goto done; - } - - packdir = fdopendir(packdir_fd); - if (packdir == NULL) { - err = got_error_from_errno("fdopendir"); - goto done; - } - - while ((dent = readdir(packdir)) != NULL) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { + const char *path_packidx = pe->path; struct got_packidx *packidx; struct got_object_qid *qid; - - if (!got_repo_is_packidx_filename(dent->d_name, - strlen(dent->d_name))) - continue; - if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR, - dent->d_name) == -1) { - err = got_error_from_errno("strdup"); - break; - } - err = got_packidx_open(&packidx, got_repo_get_fd(repo), path_packidx, 0); - free(path_packidx); if (err) break; @@ -1561,8 +1499,6 @@ match_packed_object(struct got_object_id **unique_id, } done: got_object_id_queue_free(&matched_ids); - if (packdir && closedir(packdir) != 0 && err == NULL) - err = got_error_from_errno("closedir"); if (err) { free(*unique_id); *unique_id = NULL;