Blob


1 /*
2 * Copyright (c) 2018, 2019 Ori Bernstein <ori@openbsd.org>
3 * Copyright (c) 2021 Stefan Sperling <stsp@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
18 #include "got_compat.h"
20 #include <sys/types.h>
21 #include <sys/stat.h>
22 #include <sys/queue.h>
23 #include <sys/uio.h>
24 #include <sys/socket.h>
25 #include <sys/wait.h>
26 #include <sys/resource.h>
27 #include <sys/socket.h>
29 #include <errno.h>
30 #include <err.h>
31 #include <fcntl.h>
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <stdint.h>
36 #include <unistd.h>
37 #include <zlib.h>
38 #include <ctype.h>
39 #include <limits.h>
40 #include <time.h>
42 #include "got_error.h"
43 #include "got_reference.h"
44 #include "got_repository.h"
45 #include "got_path.h"
46 #include "got_cancel.h"
47 #include "got_worktree.h"
48 #include "got_object.h"
49 #include "got_opentemp.h"
50 #include "got_send.h"
51 #include "got_repository_admin.h"
52 #include "got_commit_graph.h"
54 #include "got_lib_delta.h"
55 #include "got_lib_inflate.h"
56 #include "got_lib_object.h"
57 #include "got_lib_object_parse.h"
58 #include "got_lib_object_create.h"
59 #include "got_lib_pack.h"
60 #include "got_lib_hash.h"
61 #include "got_lib_privsep.h"
62 #include "got_lib_object_cache.h"
63 #include "got_lib_repository.h"
64 #include "got_lib_ratelimit.h"
65 #include "got_lib_pack_create.h"
66 #include "got_lib_dial.h"
68 #ifndef nitems
69 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
70 #endif
72 #ifndef ssizeof
73 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
74 #endif
76 #ifndef MIN
77 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
78 #endif
80 const struct got_error *
81 got_send_connect(pid_t *sendpid, int *sendfd, const char *proto,
82 const char *host, const char *port, const char *server_path, int verbosity)
83 {
84 const struct got_error *err = NULL;
86 *sendpid = -1;
87 *sendfd = -1;
89 if (strcmp(proto, "ssh") == 0 || strcmp(proto, "git+ssh") == 0)
90 err = got_dial_ssh(sendpid, sendfd, host, port, server_path,
91 GOT_DIAL_CMD_SEND, verbosity);
92 else if (strcmp(proto, "git") == 0)
93 err = got_dial_git(sendfd, host, port, server_path,
94 GOT_DIAL_CMD_SEND);
95 else if (strcmp(proto, "http") == 0 || strcmp(proto, "git+http") == 0)
96 err = got_error_path(proto, GOT_ERR_NOT_IMPL);
97 else
98 err = got_error_path(proto, GOT_ERR_BAD_PROTO);
99 return err;
102 struct pack_progress_arg {
103 got_send_progress_cb progress_cb;
104 void *progress_arg;
106 int ncolored;
107 int nfound;
108 int ntrees;
109 off_t packfile_size;
110 int ncommits;
111 int nobj_total;
112 int nobj_deltify;
113 int nobj_written;
114 };
116 static const struct got_error *
117 pack_progress(void *arg, int ncolored, int nfound, int ntrees,
118 off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
119 int nobj_written)
121 const struct got_error *err;
122 struct pack_progress_arg *a = arg;
124 err = a->progress_cb(a->progress_arg, ncolored, nfound, ntrees,
125 packfile_size, ncommits, nobj_total, nobj_deltify,
126 nobj_written, 0, NULL, NULL, 0);
127 if (err)
128 return err;
130 a->ncolored= ncolored;
131 a->nfound = nfound;
132 a->ntrees = ntrees;
133 a->packfile_size = packfile_size;
134 a->ncommits = ncommits;
135 a->nobj_total = nobj_total;
136 a->nobj_deltify = nobj_deltify;
137 a->nobj_written = nobj_written;
138 return NULL;
141 static const struct got_error *
142 insert_ref(struct got_reflist_head *refs, const char *refname,
143 struct got_repository *repo)
145 const struct got_error *err;
146 struct got_reference *ref;
147 struct got_reflist_entry *new;
149 err = got_ref_open(&ref, repo, refname, 0);
150 if (err)
151 return err;
153 err = got_reflist_insert(&new, refs, ref, got_ref_cmp_by_name, NULL);
154 if (err || new == NULL /* duplicate */)
155 got_ref_close(ref);
157 return err;
160 static const struct got_error *
161 check_common_ancestry(const char *refname, struct got_object_id *my_id,
162 struct got_object_id *their_id, struct got_repository *repo,
163 got_cancel_cb cancel_cb, void *cancel_arg)
165 const struct got_error *err = NULL;
166 struct got_object_id *yca_id;
167 int obj_type;
169 err = got_object_get_type(&obj_type, repo, their_id);
170 if (err)
171 return err;
172 if (obj_type != GOT_OBJ_TYPE_COMMIT)
173 return got_error_fmt(GOT_ERR_OBJ_TYPE,
174 "bad object type on server for %s", refname);
176 err = got_commit_graph_find_youngest_common_ancestor(&yca_id,
177 my_id, their_id, 0, repo, cancel_cb, cancel_arg);
178 if (err)
179 return err;
180 if (yca_id == NULL)
181 return got_error_fmt(GOT_ERR_SEND_ANCESTRY, "%s", refname);
183 if (got_object_id_cmp(their_id, yca_id) != 0)
184 err = got_error_fmt(GOT_ERR_SEND_ANCESTRY, "%s", refname);
186 free(yca_id);
187 return err;
190 static const struct got_error *
191 realloc_ids(struct got_object_id ***ids, size_t *nalloc, size_t n)
193 struct got_object_id **new;
194 const size_t alloc_chunksz = 256;
196 if (*nalloc >= n)
197 return NULL;
199 new = recallocarray(*ids, *nalloc, *nalloc + alloc_chunksz,
200 sizeof(struct got_object_id));
201 if (new == NULL)
202 return got_error_from_errno("recallocarray");
204 *ids = new;
205 *nalloc += alloc_chunksz;
206 return NULL;
209 static struct got_reference *
210 find_ref(struct got_reflist_head *refs, const char *refname)
212 struct got_reflist_entry *re;
214 TAILQ_FOREACH(re, refs, entry) {
215 if (got_path_cmp(got_ref_get_name(re->ref), refname,
216 strlen(got_ref_get_name(re->ref)),
217 strlen(refname)) == 0) {
218 return re->ref;
222 return NULL;
225 static struct got_pathlist_entry *
226 find_their_ref(struct got_pathlist_head *their_refs, const char *refname)
228 struct got_pathlist_entry *pe;
230 TAILQ_FOREACH(pe, their_refs, entry) {
231 const char *their_refname = pe->path;
232 if (got_path_cmp(their_refname, refname,
233 strlen(their_refname), strlen(refname)) == 0) {
234 return pe;
238 return NULL;
241 static const struct got_error *
242 get_remote_refname(char **remote_refname, const char *remote_name,
243 const char *refname)
245 if (strncmp(refname, "refs/", 5) == 0)
246 refname += 5;
247 if (strncmp(refname, "heads/", 6) == 0)
248 refname += 6;
250 if (asprintf(remote_refname, "refs/remotes/%s/%s",
251 remote_name, refname) == -1)
252 return got_error_from_errno("asprintf");
254 return NULL;
257 static const struct got_error *
258 update_remote_ref(struct got_reference *my_ref, const char *remote_name,
259 struct got_repository *repo)
261 const struct got_error *err, *unlock_err;
262 struct got_object_id *my_id;
263 struct got_reference *ref = NULL;
264 char *remote_refname = NULL;
265 int ref_locked = 0;
267 err = got_ref_resolve(&my_id, repo, my_ref);
268 if (err)
269 return err;
271 err = get_remote_refname(&remote_refname, remote_name,
272 got_ref_get_name(my_ref));
273 if (err)
274 goto done;
276 err = got_ref_open(&ref, repo, remote_refname, 1 /* lock */);
277 if (err) {
278 if (err->code != GOT_ERR_NOT_REF)
279 goto done;
280 err = got_ref_alloc(&ref, remote_refname, my_id);
281 if (err)
282 goto done;
283 } else {
284 ref_locked = 1;
285 err = got_ref_change_ref(ref, my_id);
286 if (err)
287 goto done;
290 err = got_ref_write(ref, repo);
291 done:
292 if (ref) {
293 if (ref_locked) {
294 unlock_err = got_ref_unlock(ref);
295 if (unlock_err && err == NULL)
296 err = unlock_err;
298 got_ref_close(ref);
300 free(my_id);
301 free(remote_refname);
302 return err;
305 const struct got_error*
306 got_send_pack(const char *remote_name, struct got_pathlist_head *branch_names,
307 struct got_pathlist_head *tag_names,
308 struct got_pathlist_head *delete_branches,
309 int verbosity, int overwrite_refs, int sendfd,
310 struct got_repository *repo, got_send_progress_cb progress_cb,
311 void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg)
313 int imsg_sendfds[2];
314 int npackfd = -1, nsendfd = -1;
315 int sendstatus, done = 0;
316 const struct got_error *err;
317 struct imsgbuf sendibuf;
318 pid_t sendpid = -1;
319 struct got_reflist_head refs;
320 struct got_pathlist_head have_refs;
321 struct got_pathlist_head their_refs;
322 struct got_pathlist_entry *pe;
323 struct got_reflist_entry *re;
324 struct got_object_id **our_ids = NULL;
325 struct got_object_id **their_ids = NULL;
326 int i, nours = 0, ntheirs = 0;
327 size_t nalloc_ours = 0, nalloc_theirs = 0;
328 int refs_to_send = 0, refs_to_delete = 0;
329 off_t bytes_sent = 0, bytes_sent_cur = 0;
330 struct pack_progress_arg ppa;
331 uint8_t packsha1[SHA1_DIGEST_LENGTH];
332 int packfd = -1;
333 FILE *delta_cache = NULL;
335 TAILQ_INIT(&refs);
336 TAILQ_INIT(&have_refs);
337 TAILQ_INIT(&their_refs);
339 TAILQ_FOREACH(pe, branch_names, entry) {
340 const char *branchname = pe->path;
341 if (strncmp(branchname, "refs/heads/", 11) != 0) {
342 char *s;
343 if (asprintf(&s, "refs/heads/%s", branchname) == -1) {
344 err = got_error_from_errno("asprintf");
345 goto done;
347 err = insert_ref(&refs, s, repo);
348 free(s);
349 } else {
350 err = insert_ref(&refs, branchname, repo);
352 if (err)
353 goto done;
356 TAILQ_FOREACH(pe, delete_branches, entry) {
357 const char *branchname = pe->path;
358 struct got_reference *ref;
359 if (strncmp(branchname, "refs/heads/", 11) != 0) {
360 err = got_error_fmt(GOT_ERR_SEND_DELETE_REF, "%s",
361 branchname);
362 goto done;
364 ref = find_ref(&refs, branchname);
365 if (ref) {
366 err = got_error_fmt(GOT_ERR_SEND_DELETE_REF,
367 "changes on %s will be sent to server",
368 branchname);
369 goto done;
373 TAILQ_FOREACH(pe, tag_names, entry) {
374 const char *tagname = pe->path;
375 if (strncmp(tagname, "refs/tags/", 10) != 0) {
376 char *s;
377 if (asprintf(&s, "refs/tags/%s", tagname) == -1) {
378 err = got_error_from_errno("asprintf");
379 goto done;
381 err = insert_ref(&refs, s, repo);
382 free(s);
383 } else {
384 err = insert_ref(&refs, tagname, repo);
386 if (err)
387 goto done;
390 if (TAILQ_EMPTY(&refs) && TAILQ_EMPTY(delete_branches)) {
391 err = got_error(GOT_ERR_SEND_EMPTY);
392 goto done;
395 TAILQ_FOREACH(re, &refs, entry) {
396 struct got_object_id *id;
397 int obj_type;
399 if (got_ref_is_symbolic(re->ref)) {
400 err = got_error_fmt(GOT_ERR_BAD_REF_TYPE,
401 "cannot send symbolic reference %s",
402 got_ref_get_name(re->ref));
403 goto done;
406 err = got_ref_resolve(&id, repo, re->ref);
407 if (err)
408 goto done;
409 err = got_object_get_type(&obj_type, repo, id);
410 free(id);
411 if (err)
412 goto done;
413 switch (obj_type) {
414 case GOT_OBJ_TYPE_COMMIT:
415 case GOT_OBJ_TYPE_TAG:
416 break;
417 default:
418 err = got_error_fmt(GOT_ERR_OBJ_TYPE,
419 "cannot send %s", got_ref_get_name(re->ref));
420 goto done;
424 packfd = got_opentempfd();
425 if (packfd == -1) {
426 err = got_error_from_errno("got_opentempfd");
427 goto done;
430 delta_cache = got_opentemp();
431 if (delta_cache == NULL) {
432 err = got_error_from_errno("got_opentemp");
433 goto done;
436 if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_sendfds) == -1) {
437 err = got_error_from_errno("socketpair");
438 goto done;
441 sendpid = fork();
442 if (sendpid == -1) {
443 err = got_error_from_errno("fork");
444 goto done;
445 } else if (sendpid == 0){
446 got_privsep_exec_child(imsg_sendfds,
447 GOT_PATH_PROG_SEND_PACK, got_repo_get_path(repo));
450 if (close(imsg_sendfds[1]) == -1) {
451 err = got_error_from_errno("close");
452 goto done;
454 imsg_init(&sendibuf, imsg_sendfds[0]);
455 nsendfd = dup(sendfd);
456 if (nsendfd == -1) {
457 err = got_error_from_errno("dup");
458 goto done;
461 /*
462 * Convert reflist to pathlist since the privsep layer
463 * is linked into helper programs which lack reference.c.
464 */
465 TAILQ_FOREACH(re, &refs, entry) {
466 struct got_object_id *id;
467 err = got_ref_resolve(&id, repo, re->ref);
468 if (err)
469 goto done;
470 err = got_pathlist_append(&have_refs,
471 got_ref_get_name(re->ref), id);
472 if (err)
473 goto done;
474 /*
475 * Also prepare the array of our object IDs which
476 * will be needed for generating a pack file.
477 */
478 err = realloc_ids(&our_ids, &nalloc_ours, nours + 1);
479 if (err)
480 goto done;
481 our_ids[nours] = id;
482 nours++;
485 err = got_privsep_send_send_req(&sendibuf, nsendfd, &have_refs,
486 delete_branches, verbosity);
487 if (err)
488 goto done;
489 nsendfd = -1;
491 err = got_privsep_recv_send_remote_refs(&their_refs, &sendibuf);
492 if (err)
493 goto done;
495 /*
496 * Process references reported by the server.
497 * Push appropriate object IDs onto the "their IDs" array.
498 * This array will be used to exclude objects which already
499 * exist on the server from our pack file.
500 */
501 TAILQ_FOREACH(pe, &their_refs, entry) {
502 const char *refname = pe->path;
503 struct got_object_id *their_id = pe->data;
504 int have_their_id;
505 struct got_object *obj;
506 struct got_reference *my_ref = NULL;
507 int is_tag = 0;
509 /* Don't blindly trust the server to send us valid names. */
510 if (!got_ref_name_is_valid(refname))
511 continue;
513 if (strncmp(refname, "refs/tags/", 10) == 0)
514 is_tag = 1;
515 /*
516 * Find out whether this is a reference we want to upload.
517 * Otherwise we can still use this reference as a hint to
518 * avoid uploading any objects the server already has.
519 */
520 my_ref = find_ref(&refs, refname);
521 if (my_ref) {
522 struct got_object_id *my_id;
523 err = got_ref_resolve(&my_id, repo, my_ref);
524 if (err)
525 goto done;
526 if (got_object_id_cmp(my_id, their_id) != 0) {
527 if (!overwrite_refs && is_tag) {
528 err = got_error_fmt(
529 GOT_ERR_SEND_TAG_EXISTS,
530 "%s", refname);
531 free(my_id);
532 goto done;
534 refs_to_send++;
536 free(my_id);
539 /* Check if their object exists locally. */
540 err = got_object_open(&obj, repo, their_id);
541 if (err) {
542 if (err->code != GOT_ERR_NO_OBJ)
543 goto done;
544 if (!overwrite_refs && my_ref != NULL) {
545 err = got_error_fmt(GOT_ERR_SEND_ANCESTRY,
546 "%s", refname);
547 goto done;
549 have_their_id = 0;
550 } else {
551 got_object_close(obj);
552 have_their_id = 1;
555 err = realloc_ids(&their_ids, &nalloc_theirs, ntheirs + 1);
556 if (err)
557 goto done;
559 if (have_their_id) {
560 /* Enforce linear ancestry if required. */
561 if (!overwrite_refs && my_ref && !is_tag) {
562 struct got_object_id *my_id;
563 err = got_ref_resolve(&my_id, repo, my_ref);
564 if (err)
565 goto done;
566 err = check_common_ancestry(refname, my_id,
567 their_id, repo, cancel_cb, cancel_arg);
568 free(my_id);
569 my_id = NULL;
570 if (err)
571 goto done;
573 /* Exclude any objects reachable via their ID. */
574 their_ids[ntheirs] = got_object_id_dup(their_id);
575 if (their_ids[ntheirs] == NULL) {
576 err = got_error_from_errno("got_object_id_dup");
577 goto done;
579 ntheirs++;
580 } else if (!is_tag) {
581 char *remote_refname;
582 struct got_reference *ref;
583 /*
584 * Exclude any objects which exist on the server
585 * according to a locally cached remote reference.
586 */
587 err = get_remote_refname(&remote_refname,
588 remote_name, refname);
589 if (err)
590 goto done;
591 err = got_ref_open(&ref, repo, remote_refname, 0);
592 free(remote_refname);
593 if (err) {
594 if (err->code != GOT_ERR_NOT_REF)
595 goto done;
596 } else {
597 err = got_ref_resolve(&their_ids[ntheirs],
598 repo, ref);
599 got_ref_close(ref);
600 if (err)
601 goto done;
602 ntheirs++;
607 /* Account for any new references we are going to upload. */
608 TAILQ_FOREACH(re, &refs, entry) {
609 if (find_their_ref(&their_refs,
610 got_ref_get_name(re->ref)) == NULL)
611 refs_to_send++;
614 /* Account for any existing references we are going to delete. */
615 TAILQ_FOREACH(pe, delete_branches, entry) {
616 const char *branchname = pe->path;
617 if (find_their_ref(&their_refs, branchname))
618 refs_to_delete++;
621 if (refs_to_send == 0 && refs_to_delete == 0) {
622 got_privsep_send_stop(imsg_sendfds[0]);
623 goto done;
626 if (refs_to_send > 0) {
627 struct got_ratelimit rl;
628 got_ratelimit_init(&rl, 0, 500);
629 memset(&ppa, 0, sizeof(ppa));
630 ppa.progress_cb = progress_cb;
631 ppa.progress_arg = progress_arg;
632 err = got_pack_create(packsha1, packfd, delta_cache,
633 their_ids, ntheirs, our_ids, nours, repo, 0, 1, 0,
634 pack_progress, &ppa, &rl, cancel_cb, cancel_arg);
635 if (err)
636 goto done;
638 npackfd = dup(packfd);
639 if (npackfd == -1) {
640 err = got_error_from_errno("dup");
641 goto done;
643 err = got_privsep_send_packfd(&sendibuf, npackfd);
644 if (err != NULL)
645 goto done;
646 npackfd = -1;
647 } else {
648 err = got_privsep_send_packfd(&sendibuf, -1);
649 if (err != NULL)
650 goto done;
653 while (!done) {
654 int success = 0;
655 char *refname = NULL;
656 char *errmsg = NULL;
658 if (cancel_cb) {
659 err = (*cancel_cb)(cancel_arg);
660 if (err)
661 goto done;
663 err = got_privsep_recv_send_progress(&done, &bytes_sent,
664 &success, &refname, &errmsg, &sendibuf);
665 if (err)
666 goto done;
667 if (refname && got_ref_name_is_valid(refname) && success &&
668 strncmp(refname, "refs/tags/", 10) != 0) {
669 struct got_reference *my_ref;
670 /*
671 * The server has accepted our changes.
672 * Update our reference in refs/remotes/ accordingly.
673 */
674 my_ref = find_ref(&refs, refname);
675 if (my_ref) {
676 err = update_remote_ref(my_ref, remote_name,
677 repo);
678 if (err)
679 goto done;
682 if (refname != NULL ||
683 bytes_sent_cur != bytes_sent) {
684 err = progress_cb(progress_arg, ppa.ncolored,
685 ppa.nfound, ppa.ntrees, ppa.packfile_size,
686 ppa.ncommits, ppa.nobj_total, ppa.nobj_deltify,
687 ppa.nobj_written, bytes_sent,
688 refname, errmsg, success);
689 if (err) {
690 free(refname);
691 free(errmsg);
692 goto done;
694 bytes_sent_cur = bytes_sent;
696 free(refname);
697 free(errmsg);
699 done:
700 if (sendpid != -1) {
701 if (err)
702 got_privsep_send_stop(imsg_sendfds[0]);
703 if (waitpid(sendpid, &sendstatus, 0) == -1 && err == NULL)
704 err = got_error_from_errno("waitpid");
706 if (packfd != -1 && close(packfd) == -1 && err == NULL)
707 err = got_error_from_errno("close");
708 if (delta_cache && fclose(delta_cache) == EOF && err == NULL)
709 err = got_error_from_errno("fclose");
710 if (nsendfd != -1 && close(nsendfd) == -1 && err == NULL)
711 err = got_error_from_errno("close");
712 if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
713 err = got_error_from_errno("close");
715 got_ref_list_free(&refs);
716 got_pathlist_free(&have_refs, GOT_PATHLIST_FREE_NONE);
717 got_pathlist_free(&their_refs, GOT_PATHLIST_FREE_NONE);
718 for (i = 0; i < nours; i++)
719 free(our_ids[i]);
720 free(our_ids);
721 for (i = 0; i < ntheirs; i++)
722 free(their_ids[i]);
723 free(their_ids);
724 return err;