commit 394b0db70a9cfc5f89b616d8c88d8e2e602346db from: Omar Polo date: Tue Jul 16 12:18:43 2024 UTC support to create/index sha256 packfiles in a few cases, an array of SHA1_DIGEST_LENGTH has been converted to a struct got_object_id to simplify the handling. commit - 515d451f3ad6aff3d0b4148d7e6234bddbe17447 commit + 394b0db70a9cfc5f89b616d8c88d8e2e602346db blob - 1cfaea31afa52c9ffd7e5e5c14d8605279ead7cd blob + 67c89a7abe4949fa112ddc68388c7ac88006368e --- gotd/repo_read.c +++ gotd/repo_read.c @@ -601,8 +601,8 @@ send_packfile(struct imsg *imsg, struct gotd_imsgev *i { const struct got_error *err = NULL; struct repo_read_client *client = &repo_read_client; - uint8_t packsha1[SHA1_DIGEST_LENGTH]; - char hex[SHA1_DIGEST_STRING_LENGTH]; + struct got_object_id packhash; + char hex[GOT_HASH_DIGEST_STRING_MAXLEN]; FILE *delta_cache = NULL; struct imsgbuf ibuf; struct repo_read_pack_progress_arg pa; @@ -642,7 +642,7 @@ send_packfile(struct imsg *imsg, struct gotd_imsgev *i if (err) goto done; - err = got_pack_create(packsha1, client->pack_pipe, delta_cache, + err = got_pack_create(&packhash, client->pack_pipe, delta_cache, have_ids.ids, have_ids.nids, want_ids.ids, want_ids.nids, repo_read.repo, 0, 1, 0, pack_progress, &pa, &rl, check_cancelled, NULL); @@ -650,7 +650,8 @@ send_packfile(struct imsg *imsg, struct gotd_imsgev *i goto done; if (log_getverbose() > 0 && - got_sha1_digest_to_str(packsha1, hex, sizeof(hex))) + got_hash_digest_to_str(packhash.hash, hex, sizeof(hex), + packhash.algo)) log_debug("sent pack-%s.pack", hex); if (gotd_imsg_compose_event(iev, GOTD_IMSG_PACKFILE_DONE, blob - ccba40109ee7f3669dde9c2c13b51d72d39158b2 blob + 979d980b5f2f473086fcf0fbb3cf6c3715d21692 --- gotd/repo_write.c +++ gotd/repo_write.c @@ -1205,6 +1205,7 @@ recv_packfile(int *have_packfile, struct imsg *imsg) const struct got_error *err = NULL, *unpack_err; struct repo_write_client *client = &repo_write_client; struct gotd_imsg_recv_packfile ireq; + struct got_object_id id; FILE *tempfiles[3] = { NULL, NULL, NULL }; struct repo_tempfile { int fd; @@ -1322,10 +1323,14 @@ recv_packfile(int *have_packfile, struct imsg *imsg) pack->filesize = pack_filesize; *have_packfile = 1; + memset(&id, 0, sizeof(id)); + memcpy(&id.hash, client->pack_sha1, SHA1_DIGEST_LENGTH); + id.algo = GOT_HASH_SHA1; + log_debug("begin indexing pack (%lld bytes in size)", (long long)pack->filesize); err = got_pack_index(pack, client->packidx_fd, - tempfiles[0], tempfiles[1], tempfiles[2], client->pack_sha1, + tempfiles[0], tempfiles[1], tempfiles[2], &id, pack_index_progress, NULL, &rl); if (err) goto done; blob - 845ac0dd1d3f99357c7a70e09b1f47d509ac1770 blob + 91d6de482305d9f00034d9d07270422fafb7415f --- lib/dump.c +++ lib/dump.c @@ -36,6 +36,7 @@ #include "got_repository_dump.h" #include "got_lib_delta.h" +#include "got_lib_hash.h" #include "got_lib_object.h" #include "got_lib_object_idset.h" #include "got_lib_ratelimit.h" @@ -86,7 +87,7 @@ got_repo_dump(FILE *out, struct got_reflist_head *incl { const struct got_error *err = NULL; struct got_ratelimit rl; - uint8_t packsha[SHA1_DIGEST_LENGTH]; + struct got_object_id packhash; FILE *delta_cache = NULL; struct got_reflist_entry *e; struct got_object_id *id = NULL; @@ -171,7 +172,7 @@ got_repo_dump(FILE *out, struct got_reflist_head *incl goto done; } - err = got_pack_create(&packsha[0], fileno(out), delta_cache, + err = got_pack_create(&packhash, fileno(out), delta_cache, theirs.ids, theirs.len, ours.ids, ours.len, repo, 0, 0, 0, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); blob - 00d3d6ab13e69cc6ad1cabc03980761c02cf2933 blob + ba7667f6986fa052b7c6f36d5d7ff65c5a87bff8 --- lib/fetch.c +++ lib/fetch.c @@ -455,8 +455,7 @@ got_fetch_pack(struct got_object_id **pack_hash, struc err = got_error_from_errno("dup"); goto done; } - err = got_privsep_send_index_pack_req(&idxibuf, (*pack_hash)->hash, - npackfd); + err = got_privsep_send_index_pack_req(&idxibuf, *pack_hash, npackfd); if (err != NULL) goto done; npackfd = -1; blob - a46287669ce58c4c526c341b77350db16c5fa714 blob + 8bf80e3bec6bdb3edf09b007a268b9fb8eb96d0b --- lib/got_lib_pack_create.h +++ lib/got_lib_pack_create.h @@ -18,11 +18,11 @@ * Write pack file data into the provided open packfile handle, for all * objects reachable via the commits listed in 'ours'. * Exclude any objects for commits listed in 'theirs' if 'theirs' is not NULL. - * Return the SHA1 digest of the resulting pack file in pack_sha1 which must - * be pre-allocated by the caller with at least SHA1_DIGEST_LENGTH bytes. + * Return the hash digest of the resulting pack file in pack_hash which must + * be pre-allocated by the caller with at least GOT_HASH_DIGEST_MAXLEN bytes. */ -const struct got_error *got_pack_create(uint8_t *pack_sha1, int packfd, - FILE *delta_cache, struct got_object_id **theirs, int ntheirs, +const struct got_error *got_pack_create(struct got_object_id *pack_hash, + int packfd, FILE *delta_cache, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, int loose_obj_only, int allow_empty, int force_refdelta, got_pack_progress_cb progress_cb, void *progress_arg, blob - 8fcb52f1333aada44e211d3b89366979118e617d blob + 4d1cbbfbbce6fbe35b75112686cc7784cb6122e9 --- lib/got_lib_pack_index.h +++ lib/got_lib_pack_index.h @@ -23,6 +23,6 @@ const struct got_error *got_pack_hwrite(int, void *, i const struct got_error * got_pack_index(struct got_pack *pack, int idxfd, FILE *tmpfile, FILE *delta_base_file, FILE *delta_accum_file, - uint8_t *pack_sha1_expected, + struct got_object_id *pack_hash_expected, got_pack_index_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl); blob - ea11d107da95548038e25310c49d564804b1c3d4 blob + 50f01ee324448ac6a14453532234d56701962df1 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -492,7 +492,7 @@ struct got_imsg_send_ref_status { /* Structure for GOT_IMSG_IDXPACK_REQUEST data. */ struct got_imsg_index_pack_request { - uint8_t pack_hash[SHA1_DIGEST_LENGTH]; + struct got_object_id id; } __attribute__((__packed__)); /* Structure for GOT_IMSG_IDXPACK_PROGRESS data. */ @@ -701,7 +701,7 @@ const struct got_error *got_privsep_send_tmpfd(struct const struct got_error *got_privsep_send_obj(struct imsgbuf *, struct got_object *); const struct got_error *got_privsep_send_index_pack_req(struct imsgbuf *, - uint8_t *, int); + struct got_object_id *, int); const struct got_error *got_privsep_send_index_pack_outfd(struct imsgbuf *, int); const struct got_error *got_privsep_recv_index_progress(int *, int *, int *, blob - 962e7a4169b31da0b55afd2771071eb95ab32c11 blob + 410b0e9b2cbce22df12461bda7f7e9b4cdff10aa --- lib/load.c +++ lib/load.c @@ -106,19 +106,20 @@ load_report_progress(got_load_progress_cb progress_cb, } static const struct got_error * -copypack(FILE *in, int outfd, off_t *tot, - struct got_object_id *id, struct got_ratelimit *rl, +copypack(FILE *in, int outfd, off_t *tot, struct got_object_id *id, + enum got_hash_algorithm algo, struct got_ratelimit *rl, got_load_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; struct got_hash hash; struct got_object_id expected_id; - char buf[BUFSIZ], sha1buf[SHA1_DIGEST_LENGTH]; - size_t r, sha1buflen = 0; + char buf[BUFSIZ], hashbuf[GOT_HASH_DIGEST_MAXLEN]; + size_t r, digest_len, hashlen = 0; *tot = 0; - got_hash_init(&hash, GOT_HASH_SHA1); + digest_len = got_hash_digest_length(algo); + got_hash_init(&hash, algo); for (;;) { err = cancel_cb(cancel_arg); @@ -130,27 +131,27 @@ copypack(FILE *in, int outfd, off_t *tot, break; /* - * An expected SHA1 checksum sits at the end of the - * pack file. Since we don't know the file size ahead - * of time we have to keep SHA1_DIGEST_LENGTH bytes - * buffered and avoid mixing those bytes int our hash - * computation until we know for sure that additional - * pack file data bytes follow. + * An expected a checksum sits at the end of the pack + * file. Since we don't know the file size ahead of + * time we have to keep digest_len bytes buffered and + * avoid mixing those bytes int our hash computation + * until we know for sure that additional pack file + * data bytes follow. * * We can assume that BUFSIZE is greater than - * SHA1_DIGEST_LENGTH and that a short read means that - * we've reached EOF. + * digest_len and that a short read means that we've + * reached EOF. */ - if (r >= sizeof(sha1buf)) { - *tot += sha1buflen; - got_hash_update(&hash, sha1buf, sha1buflen); - if (write(outfd, sha1buf, sha1buflen) == -1) + if (r >= digest_len) { + *tot += hashlen; + got_hash_update(&hash, hashbuf, hashlen); + if (write(outfd, hashbuf, hashlen) == -1) return got_error_from_errno("write"); - r -= sizeof(sha1buf); - memcpy(sha1buf, &buf[r], sizeof(sha1buf)); - sha1buflen = sizeof(sha1buf); + r -= digest_len; + memcpy(hashbuf, &buf[r], digest_len); + hashlen = digest_len; *tot += r; got_hash_update(&hash, buf, r); @@ -165,37 +166,37 @@ copypack(FILE *in, int outfd, off_t *tot, continue; } - if (sha1buflen == 0) + if (hashlen == 0) return got_error(GOT_ERR_BAD_PACKFILE); /* short read, we've reached EOF */ *tot += r; - got_hash_update(&hash, sha1buf, r); - if (write(outfd, sha1buf, r) == -1) + got_hash_update(&hash, hashbuf, r); + if (write(outfd, hashbuf, r) == -1) return got_error_from_errno("write"); - memmove(&sha1buf[0], &sha1buf[r], sizeof(sha1buf) - r); - memcpy(&sha1buf[sizeof(sha1buf) - r], buf, r); + memmove(&hashbuf[0], &hashbuf[r], digest_len - r); + memcpy(&hashbuf[digest_len - r], buf, r); break; } - if (sha1buflen == 0) + if (hashlen == 0) return got_error(GOT_ERR_BAD_PACKFILE); got_hash_final_object_id(&hash, id); - /* XXX SHA256 */ memset(&expected_id, 0, sizeof(expected_id)); - memcpy(&expected_id.hash, sha1buf, sizeof(expected_id.hash)); + expected_id.algo = algo; + memcpy(&expected_id.hash, hashbuf, digest_len); if (got_object_id_cmp(id, &expected_id) != 0) return got_error(GOT_ERR_PACKIDX_CSUM); /* re-add the expected hash at the end of the pack */ - if (write(outfd, sha1buf, sizeof(sha1buf)) == -1) + if (write(outfd, hashbuf, digest_len) == -1) return got_error_from_errno("write"); - *tot += sizeof(sha1buf); + *tot += digest_len; err = progress_cb(progress_arg, *tot, 0, 0, 0, 0); if (err) return err; @@ -223,16 +224,18 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs char *line = NULL; size_t linesize = 0; ssize_t linelen; - size_t i; + size_t i, digest_len; ssize_t n; off_t packsiz; int tmpfds[3] = {-1, -1, -1}; int imsg_idxfds[2] = {-1, -1}; int ch, done, nobj, idxstatus; pid_t idxpid; + enum got_hash_algorithm algo; got_ratelimit_init(&rl, 0, 500); - + algo = got_repo_get_object_format(repo); + digest_len = got_hash_digest_length(algo); repo_path = got_repo_get_path_git_dir(repo); linelen = getline(&line, &linesize, in); @@ -264,7 +267,7 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs if (line[linelen - 1] == '\n') line[linelen - 1] = '\0'; - if (!got_parse_object_id(&id, line, GOT_HASH_SHA1)) { + if (!got_parse_object_id(&id, line, algo)) { err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR); goto done; } @@ -309,7 +312,7 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs goto done; } - if (!got_parse_object_id(id, line, GOT_HASH_SHA1)) { + if (!got_parse_object_id(id, line, algo)) { free(id); err = got_error(GOT_ERR_BAD_OBJ_ID_STR); goto done; @@ -341,7 +344,7 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs if (err) goto done; - err = copypack(in, packfd, &packsiz, &id, &rl, + err = copypack(in, packfd, &packsiz, &id, algo, &rl, progress_cb, progress_arg, cancel_cb, cancel_arg); if (err) goto done; @@ -352,7 +355,7 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs } /* Safety checks on the pack' content. */ - if (packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) { + if (packsiz <= ssizeof(pack_hdr) + digest_len) { err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file"); goto done; } @@ -378,13 +381,13 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs } nobj = be32toh(pack_hdr.nobjects); if (nobj == 0 && - packsiz > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) { + packsiz > ssizeof(pack_hdr) + digest_len) { err = got_error_msg(GOT_ERR_BAD_PACKFILE, "bad pack file with zero objects"); goto done; } if (nobj != 0 && - packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) { + packsiz <= ssizeof(pack_hdr) + digest_len) { err = got_error_msg(GOT_ERR_BAD_PACKFILE, "empty pack file with non-zero object count"); goto done; @@ -425,7 +428,7 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs imsg_idxfds[1] = -1; imsg_init(&idxibuf, imsg_idxfds[0]); - err = got_privsep_send_index_pack_req(&idxibuf, id.hash, packfd); + err = got_privsep_send_index_pack_req(&idxibuf, &id, packfd); if (err) goto done; packfd = -1; blob - fa93dc91c296087d5f21fbfafe66effd3f66b29e blob + 637ec2cd3baf5cc77c6593f1630808a6e0b7b58e --- lib/pack_create.c +++ lib/pack_create.c @@ -44,6 +44,7 @@ #include "got_object.h" #include "got_path.h" #include "got_reference.h" +#include "got_repository.h" #include "got_repository_admin.h" #include "got_lib_deltify.h" @@ -1558,6 +1559,7 @@ deltahdr(off_t *packfile_size, struct got_hash *ctx, i const struct got_error *err; char buf[32]; int nh; + size_t digest_len = got_hash_digest_length(m->prev->id.algo); if (m->prev->off != 0 && !force_refdelta) { err = packhdr(&nh, buf, sizeof(buf), @@ -1578,11 +1580,10 @@ deltahdr(off_t *packfile_size, struct got_hash *ctx, i if (err) return err; *packfile_size += nh; - err = hwrite(packfd, m->prev->id.hash, - sizeof(m->prev->id.hash), ctx); + err = hwrite(packfd, m->prev->id.hash, digest_len, ctx); if (err) return err; - *packfile_size += sizeof(m->prev->id.hash); + *packfile_size += digest_len; } return NULL; @@ -1684,8 +1685,9 @@ done: } static const struct got_error * -genpack(uint8_t *pack_sha1, int packfd, struct got_pack *reuse_pack, - FILE *delta_cache, struct got_pack_meta **deltify, int ndeltify, +genpack(struct got_object_id *pack_hash, int packfd, + struct got_pack *reuse_pack, FILE *delta_cache, + struct got_pack_meta **deltify, int ndeltify, struct got_pack_meta **reuse, int nreuse, int ncolored, int nfound, int ntrees, int nours, struct got_repository *repo, int force_refdelta, @@ -1704,9 +1706,16 @@ genpack(uint8_t *pack_sha1, int packfd, struct got_pac uint8_t *delta_cache_map = NULL; size_t delta_cache_size = 0; FILE *packfile = NULL; + enum got_hash_algorithm algo; + size_t digest_len; - got_hash_init(&ctx, GOT_HASH_SHA1); + algo = got_repo_get_object_format(repo); + digest_len = got_hash_digest_length(algo); + got_hash_init(&ctx, algo); + memset(pack_hash, 0, sizeof(*pack_hash)); + pack_hash->algo = algo; + #ifndef GOT_PACK_NO_MMAP delta_cache_fd = dup(fileno(delta_cache)); if (delta_cache_fd != -1) { @@ -1786,11 +1795,11 @@ genpack(uint8_t *pack_sha1, int packfd, struct got_pac goto done; } - got_hash_final(&ctx, pack_sha1); - err = got_poll_write_full(packfd, pack_sha1, SHA1_DIGEST_LENGTH); + got_hash_final_object_id(&ctx, pack_hash); + err = got_poll_write_full(packfd, pack_hash->hash, digest_len); if (err) goto done; - packfile_size += SHA1_DIGEST_LENGTH; + packfile_size += digest_len; packfile_size += sizeof(struct got_packfile_hdr); if (progress_cb) { err = progress_cb(progress_arg, ncolored, nfound, ntrees, @@ -1824,7 +1833,7 @@ add_meta_idset_cb(struct got_object_id *id, void *data } const struct got_error * -got_pack_create(uint8_t *packsha1, int packfd, FILE *delta_cache, +got_pack_create(struct got_object_id *packhash, int packfd, FILE *delta_cache, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, int loose_obj_only, int allow_empty, @@ -1936,7 +1945,7 @@ got_pack_create(uint8_t *packsha1, int packfd, FILE *d /* Pinned pack may have moved to different cache slot. */ reuse_pack = got_repo_get_pinned_pack(repo); - err = genpack(packsha1, packfd, reuse_pack, delta_cache, deltify.meta, + err = genpack(packhash, packfd, reuse_pack, delta_cache, deltify.meta, deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees, nours, repo, force_refdelta, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); blob - 22f7e59e0c4b66b5d2e157609322d172b0fb4461 blob + f5c3ffa28603a655efaca764f0ff6031de2ef9f5 --- lib/pack_index.c +++ lib/pack_index.c @@ -320,8 +320,8 @@ read_packed_object(struct got_pack *pack, struct got_i break; } else { /* - * XXX Seek back and get CRC and SHA1 of on-disk - * offset bytes. + * XXX Seek back and get CRC and hash digest + * of on-disk offset bytes. */ if (lseek(pack->fd, obj->off + obj->tslen, SEEK_SET) == -1) { @@ -441,9 +441,9 @@ done: /* Determine the slot in the pack index a given object ID should use. */ static int -find_object_idx(struct got_packidx *packidx, uint8_t *sha1) +find_object_idx(struct got_packidx *packidx, uint8_t *hash) { - u_int8_t id0 = sha1[0]; + u_int8_t id0 = hash[0]; uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]); int left = 0, right = nindexed - 1; int cmp = 0, i = 0; @@ -458,7 +458,7 @@ find_object_idx(struct got_packidx *packidx, uint8_t * i = ((left + right) / 2); oid = packidx->hdr.sorted_ids + i * digest_len; - cmp = memcmp(sha1, oid, digest_len); + cmp = memcmp(hash, oid, digest_len); if (cmp == 0) return -1; /* object already indexed */ else if (cmp > 0) @@ -480,9 +480,9 @@ print_packidx(struct got_packidx *packidx) fprintf(stderr, "object IDs:\n"); for (i = 0; i < nindexed; i++) { - char hex[SHA1_DIGEST_STRING_LENGTH]; - got_sha1_digest_to_str(packidx->hdr.sorted_ids + i * digest_len, - hex, sizeof(hex)); + char hex[GOT_HASH_DIGEST_STRING_MAXLEN]; + got_hash_digest_to_str(packidx->hdr.sorted_ids + i * digest_len, + hex, sizeof(hex), packidx->algo); fprintf(stderr, "%s\n", hex); } fprintf(stderr, "\n"); @@ -608,7 +608,8 @@ report_progress(uint32_t nobj_total, uint32_t nobj_ind const struct got_error * got_pack_index(struct got_pack *pack, int idxfd, FILE *tmpfile, - FILE *delta_base_file, FILE *delta_accum_file, uint8_t *pack_sha1_expected, + FILE *delta_base_file, FILE *delta_accum_file, + struct got_object_id *pack_hash_expected, got_pack_index_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl) { @@ -616,19 +617,20 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE struct got_packfile_hdr hdr; struct got_packidx packidx; char buf[8]; - char pack_sha1[SHA1_DIGEST_LENGTH]; + struct got_object_id pack_hash; uint32_t nobj, nvalid, nloose, nresolved = 0, i; struct got_indexed_object *objects = NULL, *obj; struct got_hash ctx; - uint8_t packidx_hash[SHA1_DIGEST_LENGTH]; + uint8_t packidx_hash[GOT_HASH_DIGEST_MAXLEN]; ssize_t r, w; int pass, have_ref_deltas = 0, first_delta_idx = -1; size_t mapoff = 0; int p_indexed = 0, last_p_indexed = -1; int p_resolved = 0, last_p_resolved = -1; + size_t digest_len = got_hash_digest_length(pack->algo); - /* Require that pack file header and SHA1 trailer are present. */ - if (pack->filesize < sizeof(hdr) + SHA1_DIGEST_LENGTH) + /* Require that pack file header and hash trailer are present. */ + if (pack->filesize < sizeof(hdr) + digest_len) return got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file"); @@ -655,7 +657,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE return got_error_msg(GOT_ERR_BAD_PACKFILE, "bad packfile with zero objects"); - /* We compute the SHA1 of pack file contents and verify later on. */ + /* We compute the hash of pack file contents and verify later on. */ got_hash_init(&ctx, pack->algo); got_hash_update(&ctx, &hdr, sizeof(hdr)); @@ -682,8 +684,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE err = got_error_from_errno("calloc"); goto done; } - packidx.hdr.sorted_ids = calloc(nobj, - got_hash_digest_length(pack->algo)); + packidx.hdr.sorted_ids = calloc(nobj, digest_len); if (packidx.hdr.sorted_ids == NULL) { err = got_error_from_errno("calloc"); goto done; @@ -698,6 +699,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE err = got_error_from_errno("calloc"); goto done; } + packidx.algo = pack->algo; /* Large offsets table is empty for pack files < 2 GB. */ if (pack->filesize >= GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) { packidx.hdr.large_offsets = calloc(nobj, sizeof(uint64_t)); @@ -780,14 +782,14 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE * Having done a full pass over the pack file and can now * verify its checksum. */ - got_hash_final(&ctx, pack_sha1); + got_hash_final_object_id(&ctx, &pack_hash); - if (memcmp(pack_sha1_expected, pack_sha1, SHA1_DIGEST_LENGTH) != 0) { + if (got_object_id_cmp(pack_hash_expected, &pack_hash) != 0) { err = got_error(GOT_ERR_PACKFILE_CSUM); goto done; } - /* Verify the SHA1 checksum stored at the end of the pack file. */ + /* Verify the hash checksum stored at the end of the pack file. */ if (pack->map) { if (pack->filesize > SIZE_MAX) { err = got_error_fmt(GOT_ERR_RANGE, @@ -796,26 +798,26 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE goto done; } - memcpy(pack_sha1_expected, pack->map + - pack->filesize - SHA1_DIGEST_LENGTH, - SHA1_DIGEST_LENGTH); + memcpy(pack_hash_expected, pack->map + + pack->filesize - digest_len, + digest_len); } else { ssize_t n; - if (lseek(pack->fd, -SHA1_DIGEST_LENGTH, SEEK_END) == -1) { + if (lseek(pack->fd, -digest_len, SEEK_END) == -1) { err = got_error_from_errno("lseek"); goto done; } - n = read(pack->fd, pack_sha1_expected, SHA1_DIGEST_LENGTH); + n = read(pack->fd, pack_hash_expected, digest_len); if (n == -1) { err = got_error_from_errno("read"); goto done; } - if (n != SHA1_DIGEST_LENGTH) { + if (n != digest_len) { err = got_error(GOT_ERR_IO); goto done; } } - if (memcmp(pack_sha1, pack_sha1_expected, SHA1_DIGEST_LENGTH) != 0) { + if (got_object_id_cmp(pack_hash_expected, &pack_hash) != 0) { err = got_error_msg(GOT_ERR_BAD_PACKFILE, "bad checksum in pack file trailer"); goto done; @@ -915,7 +917,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE free(objects); objects = NULL; - got_hash_init(&ctx, GOT_HASH_SHA1); + got_hash_init(&ctx, pack->algo); putbe32(buf, GOT_PACKIDX_V2_MAGIC); putbe32(buf + 4, GOT_PACKIDX_VERSION); err = got_pack_hwrite(idxfd, buf, 8, &ctx); @@ -926,7 +928,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE if (err) goto done; err = got_pack_hwrite(idxfd, packidx.hdr.sorted_ids, - nobj * SHA1_DIGEST_LENGTH, &ctx); + nobj * digest_len, &ctx); if (err) goto done; err = got_pack_hwrite(idxfd, packidx.hdr.crc32, @@ -943,17 +945,17 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE if (err) goto done; } - err = got_pack_hwrite(idxfd, pack_sha1, SHA1_DIGEST_LENGTH, &ctx); + err = got_pack_hwrite(idxfd, &pack_hash.hash, digest_len, &ctx); if (err) goto done; got_hash_final(&ctx, packidx_hash); - w = write(idxfd, packidx_hash, sizeof(packidx_hash)); + w = write(idxfd, packidx_hash, digest_len); if (w == -1) { err = got_error_from_errno("write"); goto done; } - if (w != sizeof(packidx_hash)) { + if (w != digest_len) { err = got_error(GOT_ERR_IO); goto done; } blob - 16b07ccae25800aac74814a8f28dc862b140de76 blob + 42906a3ad5dc8e31e2858ff79937b7a91ecb6ddb --- lib/privsep.c +++ lib/privsep.c @@ -1077,14 +1077,14 @@ got_privsep_recv_send_progress(int *done, off_t *bytes } const struct got_error * -got_privsep_send_index_pack_req(struct imsgbuf *ibuf, uint8_t *pack_sha1, +got_privsep_send_index_pack_req(struct imsgbuf *ibuf, struct got_object_id *id, int fd) { const struct got_error *err = NULL; /* Keep in sync with struct got_imsg_index_pack_request */ if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_REQUEST, 0, 0, fd, - pack_sha1, SHA1_DIGEST_LENGTH) == -1) { + id, sizeof(*id)) == -1) { err = got_error_from_errno("imsg_compose INDEX_REQUEST"); close(fd); return err; blob - 66d664aa6393a67bea7096826a3412604ea954ef blob + 7c31b2fe91e06a2c69dab4fb449f2bf62b1e09f1 --- lib/repository_admin.c +++ lib/repository_admin.c @@ -154,7 +154,7 @@ got_repo_pack_objects(FILE **packfile, struct got_obje struct got_object_id **ours = NULL, **theirs = NULL; int nours = 0, ntheirs = 0, packfd = -1, i; char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL; - char *sha1_str = NULL; + char *hash_str = NULL; FILE *delta_cache = NULL; struct got_ratelimit rl; @@ -208,20 +208,19 @@ got_repo_pack_objects(FILE **packfile, struct got_obje err = got_error_from_errno("calloc"); goto done; } - - err = got_pack_create((*pack_hash)->hash, packfd, delta_cache, + err = got_pack_create(*pack_hash, packfd, delta_cache, theirs, ntheirs, ours, nours, repo, loose_obj_only, 0, force_refdelta, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); if (err) goto done; - err = got_object_id_str(&sha1_str, *pack_hash); + err = got_object_id_str(&hash_str, *pack_hash); if (err) goto done; if (asprintf(&packfile_path, "%s/%s/pack-%s.pack", got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR, - sha1_str) == -1) { + hash_str) == -1) { err = got_error_from_errno("asprintf"); goto done; } @@ -259,7 +258,7 @@ done: err = got_error_from_errno2("unlink", tmpfile_path); free(tmpfile_path); free(packfile_path); - free(sha1_str); + free(hash_str); free(path); if (err) { free(*pack_hash); @@ -365,8 +364,7 @@ got_repo_index_pack(FILE *packfile, struct got_object_ err = got_error_from_errno("dup"); goto done; } - err = got_privsep_send_index_pack_req(&idxibuf, pack_hash->hash, - npackfd); + err = got_privsep_send_index_pack_req(&idxibuf, pack_hash, npackfd); if (err != NULL) goto done; npackfd = -1; blob - d6d1733a9530e64b2f68b2f1ec8489706f582b2a blob + a1473da8306f07e7f95614c32166941e804ca5f1 --- lib/send.c +++ lib/send.c @@ -356,7 +356,7 @@ got_send_pack(const char *remote_name, struct got_path int refs_to_send = 0, refs_to_delete = 0; off_t bytes_sent = 0, bytes_sent_cur = 0; struct pack_progress_arg ppa; - uint8_t packsha1[SHA1_DIGEST_LENGTH]; + struct got_object_id packhash; int packfd = -1; FILE *delta_cache = NULL; char *s = NULL; @@ -613,7 +613,7 @@ got_send_pack(const char *remote_name, struct got_path ppa.progress_cb = progress_cb; ppa.progress_arg = progress_arg; ppa.sendfd = sendfd; - err = got_pack_create(packsha1, packfd, delta_cache, + err = got_pack_create(&packhash, packfd, delta_cache, their_ids, ntheirs, our_ids, nours, repo, 0, 1, 0, pack_progress, &ppa, &rl, cancel_cb, cancel_arg); if (err) blob - 6891969704cec149dec63e3d8e89a50d6da5d5b9 blob + cc2cd7ee7882f8bf0a997b3e00e36419a15d76fc --- libexec/got-index-pack/got-index-pack.c +++ libexec/got-index-pack/got-index-pack.c @@ -82,11 +82,11 @@ main(int argc, char **argv) const struct got_error *err = NULL, *close_err; struct imsgbuf ibuf; struct imsg imsg; + struct got_object_id pack_hash; size_t i; int idxfd = -1, tmpfd = -1; FILE *tmpfiles[3]; struct got_pack pack; - uint8_t pack_hash[SHA1_DIGEST_LENGTH]; off_t packfile_size; struct got_ratelimit rl; #if 0 @@ -128,8 +128,9 @@ main(int argc, char **argv) err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; } - memcpy(pack_hash, imsg.data, sizeof(pack_hash)); + memcpy(&pack_hash, imsg.data, sizeof(pack_hash)); pack.fd = imsg_get_fd(&imsg); + pack.algo = pack_hash.algo; err = got_privsep_recv_imsg(&imsg, &ibuf, 0); if (err) @@ -194,7 +195,7 @@ main(int argc, char **argv) } #endif err = got_pack_index(&pack, idxfd, tmpfiles[0], tmpfiles[1], - tmpfiles[2], pack_hash, send_index_pack_progress, &ibuf, &rl); + tmpfiles[2], &pack_hash, send_index_pack_progress, &ibuf, &rl); done: close_err = got_pack_close(&pack); if (close_err && err == NULL) blob - 5e1ec6efa75876a9ed9e5f0aea1a6975639cbbfc blob + 34cc8e56ce949b897f9175c59785d3bfed510de9 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -463,7 +463,7 @@ tree_path_changed(int *changed, uint8_t **buf1, size_t *changed = 0; - idlen = got_hash_digest_length(GOT_HASH_SHA1); + idlen = got_hash_digest_length(pack->algo); /* We not do support comparing the root path. */ if (got_path_is_root_dir(path)) @@ -542,7 +542,7 @@ tree_path_changed(int *changed, uint8_t **buf1, size_t break; } - if (memcmp(pte1.id, pte2.id, SHA1_DIGEST_LENGTH) == 0) { + if (memcmp(pte1.id, pte2.id, pte1.idlen) == 0) { *changed = 0; break; } @@ -576,7 +576,7 @@ tree_path_changed(int *changed, uint8_t **buf1, size_t next_entry1 = *buf1; remain1 = *len1; - memcpy(id2.hash, pte2.id, SHA1_DIGEST_LENGTH); + memcpy(id2.hash, pte2.id, pte2.idlen); id2.algo = pack->algo; idx = got_packidx_get_object_idx(packidx, &id2); if (idx == -1) { @@ -1259,7 +1259,7 @@ enumerate_tree(int *have_all_entries, struct imsgbuf * } err = got_object_parse_tree(&entries, &nentries, - &nentries_alloc, buf, len, GOT_HASH_SHA1); + &nentries_alloc, buf, len, pack->algo); if (err) goto done; @@ -1279,7 +1279,7 @@ enumerate_tree(int *have_all_entries, struct imsgbuf * if (err) goto done; eqid->id.algo = pte->algo; - memcpy(eqid->id.hash, pte->id, sizeof(eqid->id.hash)); + memcpy(eqid->id.hash, pte->id, pte->idlen); if (got_object_idset_contains(idset, &eqid->id)) { got_object_qid_free(eqid);