commit ebc55e2dfc648063c434cc7a708ac24d266efbdb from: Stefan Sperling date: Mon Dec 24 13:18:56 2018 UTC eliminate got_object_open() round-trip when opening blobs commit - 09de383e6e63acb41f89665120f8f9746b637605 commit + ebc55e2dfc648063c434cc7a708ac24d266efbdb blob - 115b8ff9f9d012b61bcb855ef9c636acee91a445 blob + 9667c50953279012dfc4ed411a7bfeca643cf176 --- lib/got_lib_object_parse.h +++ lib/got_lib_object_parse.h @@ -17,10 +17,12 @@ const struct got_error *got_object_qid_alloc_partial(struct got_object_qid **); struct got_commit_object *got_object_commit_alloc_partial(void); struct got_tree_entry *got_alloc_tree_entry_partial(void); + +/* XXX these declarations don't belong here */ const struct got_error *got_object_read_header_privsep(struct got_object**, struct got_repository *repo, int); -const struct got_error *got_object_read_blob_privsep(size_t *, int, int, - struct got_repository *repo); +const struct got_error *got_object_read_blob_privsep(size_t *, size_t *, int, + int, struct got_repository *repo); const struct got_error *got_object_read_commit_privsep( struct got_commit_object **, int, struct got_repository *); const struct got_error *got_object_read_tree_privsep(struct got_tree_object **, blob - 74f300e5d407ceb34a8c02eb0bbb4911b3afc6e4 blob + dc25646a1852be26f74f2c2fc27edc56e9c5dc0a --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -165,6 +165,7 @@ struct got_imsg_tree_object { /* Structure for GOT_IMSG_BLOB. */ struct got_imsg_blob { size_t size; + size_t hdrlen; }; /* Structure for GOT_IMSG_TAG data. */ @@ -224,7 +225,8 @@ const struct got_error *got_privsep_send_tree_req(stru struct got_object_id *, int); const struct got_error *got_privsep_send_tag_req(struct imsgbuf *, int, struct got_object_id *, int); -const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int); +const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int, + struct got_object_id *, int); const struct got_error *got_privsep_send_blob_outfd(struct imsgbuf *, int); const struct got_error *got_privsep_send_tmpfd(struct imsgbuf *, int); const struct got_error *got_privsep_send_obj(struct imsgbuf *, @@ -241,8 +243,9 @@ const struct got_error *got_privsep_recv_tree(struct g struct imsgbuf *); const struct got_error *got_privsep_send_tree(struct imsgbuf *, struct got_tree_object *); -const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t); -const struct got_error *got_privsep_recv_blob(size_t *, struct imsgbuf *); +const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t, size_t); +const struct got_error *got_privsep_recv_blob(size_t *, size_t *, + struct imsgbuf *); const struct got_error *got_privsep_send_tag(struct imsgbuf *, struct got_tag_object *); const struct got_error *got_privsep_recv_tag(struct got_tag_object **, blob - 11a2f7424fefddd1c5a32be9dd1207fb3f48a881 blob + 40fe11e5a4707452c6f3ffd2f243f1cc391e9f72 --- lib/object.c +++ lib/object.c @@ -653,8 +653,9 @@ got_object_tree_get_entries(struct got_tree_object *tr } static const struct got_error * -read_packed_blob_privsep(size_t *size, int outfd, struct got_object *obj, - struct got_pack *pack) +request_packed_blob(size_t *size, size_t *hdrlen, int outfd, + struct got_pack *pack, struct got_packidx *packidx, int idx, + struct got_object_id *id) { const struct got_error *err = NULL; int outfd_child; @@ -671,7 +672,7 @@ read_packed_blob_privsep(size_t *size, int outfd, stru if (outfd_child == -1) return got_error_from_errno(); - err = got_privsep_send_obj_req(pack->privsep_child->ibuf, -1, obj); + err = got_privsep_send_blob_req(pack->privsep_child->ibuf, -1, id, idx); if (err) return err; @@ -698,7 +699,8 @@ read_packed_blob_privsep(size_t *size, int outfd, stru return err; } - err = got_privsep_recv_blob(size, pack->privsep_child->ibuf); + err = got_privsep_recv_blob(size, hdrlen, + pack->privsep_child->ibuf); if (err) return err; @@ -708,21 +710,34 @@ read_packed_blob_privsep(size_t *size, int outfd, stru return err; } -const struct got_error * -got_object_blob_open(struct got_blob_object **blob, - struct got_repository *repo, struct got_object *obj, size_t blocksize) +static const struct got_error * +read_packed_blob_privsep(size_t *size, size_t *hdrlen, int outfd, + struct got_pack *pack, struct got_packidx *packidx, int idx, + struct got_object_id *id) { const struct got_error *err = NULL; - int outfd; - size_t size; - struct stat sb; + + if (pack->privsep_child == NULL) { + err = start_pack_privsep_child(pack, packidx); + if (err) + return err; + } - if (obj->type != GOT_OBJ_TYPE_BLOB) - return got_error(GOT_ERR_OBJ_TYPE); + return request_packed_blob(size, hdrlen, outfd, pack, packidx, idx, id); +} - if (blocksize < obj->hdrlen) - return got_error(GOT_ERR_NO_SPACE); - +static const struct got_error * +open_blob(struct got_blob_object **blob, struct got_repository *repo, + struct got_object_id *id, size_t blocksize) +{ + const struct got_error *err = NULL; + struct got_packidx *packidx = NULL; + int idx; + char *path_packfile; + int outfd; + size_t size, hdrlen; + struct stat sb; + *blob = calloc(1, sizeof(**blob)); if (*blob == NULL) return got_error_from_errno(); @@ -736,47 +751,52 @@ got_object_blob_open(struct got_blob_object **blob, err = got_error_from_errno(); goto done; } - if (obj->flags & GOT_OBJ_FLAG_PACKED) { - struct got_pack *pack; - pack = got_repo_get_cached_pack(repo, obj->path_packfile); + + err = got_repo_search_packidx(&packidx, &idx, repo, id); + if (err == NULL) { + struct got_pack *pack = NULL; + + err = get_packfile_path(&path_packfile, packidx); + if (err) + goto done; + + pack = got_repo_get_cached_pack(repo, path_packfile); if (pack == NULL) { - err = got_repo_cache_pack(&pack, repo, - obj->path_packfile, NULL); + err = got_repo_cache_pack(&pack, repo, path_packfile, + packidx); if (err) goto done; } - err = read_packed_blob_privsep(&size, outfd, obj, pack); - if (err) - goto done; - obj->size = size; - } else { + err = read_packed_blob_privsep(&size, &hdrlen, outfd, pack, + packidx, idx, id); + } else if (err->code == GOT_ERR_NO_OBJ) { int infd; - err = open_loose_object(&infd, got_object_get_id(obj), repo); - if (err) - goto done; - - err = got_object_read_blob_privsep(&size, outfd, infd, repo); - close(infd); + err = open_loose_object(&infd, id, repo); if (err) goto done; - - if (size != obj->hdrlen + obj->size) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - goto done; - } + err = got_object_read_blob_privsep(&size, &hdrlen, outfd, + infd, repo); + close(infd); } + if (err) + goto done; if (fstat(outfd, &sb) == -1) { err = got_error_from_errno(); goto done; } - if (sb.st_size != obj->hdrlen + obj->size) { + if (sb.st_size != size) { err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; } + if (hdrlen >= size) { + err = got_error(GOT_ERR_BAD_OBJ_HDR); + goto done; + } + (*blob)->f = fdopen(outfd, "rb"); if ((*blob)->f == NULL) { err = got_error_from_errno(); @@ -784,9 +804,9 @@ got_object_blob_open(struct got_blob_object **blob, goto done; } - (*blob)->hdrlen = obj->hdrlen; + (*blob)->hdrlen = hdrlen; (*blob)->blocksize = blocksize; - memcpy(&(*blob)->id.sha1, obj->id.sha1, SHA1_DIGEST_LENGTH); + memcpy(&(*blob)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH); done: if (err) { @@ -807,23 +827,14 @@ got_object_open_as_blob(struct got_blob_object **blob, struct got_repository *repo, struct got_object_id *id, size_t blocksize) { - const struct got_error *err; - struct got_object *obj; + return open_blob(blob, repo, id, blocksize); +} - *blob = NULL; - - err = got_object_open(&obj, repo, id); - if (err) - return err; - if (obj->type != GOT_OBJ_TYPE_BLOB) { - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - - err = got_object_blob_open(blob, repo, obj, blocksize); -done: - got_object_close(obj); - return err; +const struct got_error * +got_object_blob_open(struct got_blob_object **blob, + struct got_repository *repo, struct got_object *obj, size_t blocksize) +{ + return open_blob(blob, repo, got_object_get_id(obj), blocksize); } void @@ -1368,7 +1379,8 @@ got_object_read_tree_privsep(struct got_tree_object ** } static const struct got_error * -request_blob(size_t *size, int outfd, int infd, struct imsgbuf *ibuf) +request_blob(size_t *size, size_t *hdrlen, int outfd, int infd, + struct imsgbuf *ibuf) { const struct got_error *err = NULL; int outfd_child; @@ -1377,7 +1389,7 @@ request_blob(size_t *size, int outfd, int infd, struct if (outfd_child == -1) return got_error_from_errno(); - err = got_privsep_send_blob_req(ibuf, infd); + err = got_privsep_send_blob_req(ibuf, infd, NULL, -1); if (err) return err; @@ -1387,7 +1399,7 @@ request_blob(size_t *size, int outfd, int infd, struct return err; } - err = got_privsep_recv_blob(size, ibuf); + err = got_privsep_recv_blob(size, hdrlen, ibuf); if (err) return err; @@ -1398,8 +1410,8 @@ request_blob(size_t *size, int outfd, int infd, struct } const struct got_error * -got_object_read_blob_privsep(size_t *size, int outfd, int infd, - struct got_repository *repo) +got_object_read_blob_privsep(size_t *size, size_t *hdrlen, + int outfd, int infd, struct got_repository *repo) { int imsg_fds[2]; pid_t pid; @@ -1407,7 +1419,7 @@ got_object_read_blob_privsep(size_t *size, int outfd, if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) { ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf; - return request_blob(size, outfd, infd, ibuf); + return request_blob(size, hdrlen, outfd, infd, ibuf); } ibuf = calloc(1, sizeof(*ibuf)); @@ -1433,7 +1445,7 @@ got_object_read_blob_privsep(size_t *size, int outfd, imsg_init(ibuf, imsg_fds[0]); repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf; - return request_blob(size, outfd, infd, ibuf); + return request_blob(size, hdrlen, outfd, infd, ibuf); } static const struct got_error * blob - 3c8114ca4f5d5b4026d90702aa47ffb3fc77cf2c blob + 0ae91bf1e8abddd5bbc2ed48b4324d3e02b023f3 --- lib/privsep.c +++ lib/privsep.c @@ -338,9 +338,23 @@ got_privsep_send_tag_req(struct imsgbuf *ibuf, int fd, } const struct got_error * -got_privsep_send_blob_req(struct imsgbuf *ibuf, int infd) +got_privsep_send_blob_req(struct imsgbuf *ibuf, int infd, + struct got_object_id *id, int pack_idx) { - if (imsg_compose(ibuf, GOT_IMSG_BLOB_REQUEST, 0, 0, infd, NULL, 0) + struct got_imsg_packed_object iobj, *iobjp; + size_t len; + + if (id) { /* blob is packed */ + iobj.idx = pack_idx; + memcpy(iobj.id, id->sha1, sizeof(iobj.id)); + iobjp = &iobj; + len = sizeof(iobj); + } else { + iobjp = NULL; + len = 0; + } + + if (imsg_compose(ibuf, GOT_IMSG_BLOB_REQUEST, 0, 0, infd, iobjp, len) == -1) return got_error_from_errno(); @@ -852,11 +866,12 @@ done: } const struct got_error * -got_privsep_send_blob(struct imsgbuf *ibuf, size_t size) +got_privsep_send_blob(struct imsgbuf *ibuf, size_t size, size_t hdrlen) { struct got_imsg_blob iblob; iblob.size = size; + iblob.hdrlen = hdrlen; /* Data has already been written to file descriptor. */ if (imsg_compose(ibuf, GOT_IMSG_BLOB, 0, 0, -1, &iblob, sizeof(iblob)) @@ -867,7 +882,7 @@ got_privsep_send_blob(struct imsgbuf *ibuf, size_t siz } const struct got_error * -got_privsep_recv_blob(size_t *size, struct imsgbuf *ibuf) +got_privsep_recv_blob(size_t *size, size_t *hdrlen, struct imsgbuf *ibuf) { const struct got_error *err = NULL; struct imsg imsg; @@ -888,6 +903,7 @@ got_privsep_recv_blob(size_t *size, struct imsgbuf *ib } iblob = imsg.data; *size = iblob->size; + *hdrlen = iblob->hdrlen; /* Data has been written to file descriptor. */ break; default: blob - d45817fbc3bc2a3022fa504990abbe4ad1fc9bdc blob + bcd6bb4ba2b60ed90712b29a2e72129204bee3cb --- libexec/got-read-blob/got-read-blob.c +++ libexec/got-read-blob/got-read-blob.c @@ -71,6 +71,7 @@ main(int argc, char *argv[]) struct imsg imsg, imsg_outfd; FILE *f = NULL; size_t size; + struct got_object *obj = NULL; memset(&imsg, 0, sizeof(imsg)); imsg.fd = -1; @@ -97,11 +98,6 @@ main(int argc, char *argv[]) goto done; } - datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - if (datalen != 0) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - goto done; - } if (imsg.fd == -1) { err = got_error(GOT_ERR_PRIVSEP_NO_FD); goto done; @@ -132,6 +128,15 @@ main(int argc, char *argv[]) goto done; } + err = got_object_read_header(&obj, imsg.fd); + if (err) + goto done; + + if (lseek(imsg.fd, SEEK_SET, 0) == -1) { + err = got_error_from_errno(); + goto done; + } + f = fdopen(imsg.fd, "rb"); if (f == NULL) { err = got_error_from_errno(); @@ -142,7 +147,7 @@ main(int argc, char *argv[]) if (err) goto done; - err = got_privsep_send_blob(&ibuf, size); + err = got_privsep_send_blob(&ibuf, size, obj->hdrlen); done: if (f) fclose(f); @@ -152,6 +157,8 @@ done: close(imsg_outfd.fd); imsg_free(&imsg); imsg_free(&imsg_outfd); + if (obj) + got_object_close(obj); if (err) break; } blob - a195548c98a53ac04849c666f664141ec7834607 blob + eec0b58eb918e3e41527914b3675a5b9a7b62032 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -84,39 +84,6 @@ done: } static const struct got_error * -get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf, - struct got_pack *pack, struct got_packidx *packidx, - struct got_object_cache *objcache, int type) -{ - const struct got_error *err = NULL; - struct got_object *iobj; - - err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf); - if (err) - return err; - - if (iobj->type != type) { - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - - if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0) - return got_error(GOT_ERR_OBJ_NOT_PACKED); - - *obj = got_object_cache_get(objcache, &iobj->id); - if (*obj == NULL) { - err = got_packfile_open_object(obj, pack, packidx, - iobj->pack_idx, &iobj->id); - if (err) - goto done; - } - (*obj)->refcnt++; -done: - got_object_close(iobj); - return err; -} - -static const struct got_error * commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, struct got_packidx *packidx, struct got_object_cache *objcache) { @@ -250,11 +217,19 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_packidx *packidx, struct got_object_cache *objcache) { const struct got_error *err = NULL; + struct got_imsg_packed_object iobj; struct got_object *obj = NULL; FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL; + struct got_object_id id; + size_t datalen; - err = get_object(&obj, imsg, ibuf, pack, packidx, objcache, - GOT_OBJ_TYPE_BLOB); + datalen = imsg->hdr.len - IMSG_HEADER_SIZE; + if (datalen != sizeof(iobj)) + return got_error(GOT_ERR_PRIVSEP_LEN); + memcpy(&iobj, imsg->data, sizeof(iobj)); + memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH); + + err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id); if (err) return err; @@ -273,7 +248,7 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, if (err) goto done; - err = got_privsep_send_blob(ibuf, obj->size); + err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen); done: if (outfile) fclose(outfile);