1 /*
2  * Copyright (c) 2018, 2019 Ori Bernstein <ori@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
21 #include <sys/socket.h>
22 #include <sys/wait.h>
23 #include <sys/resource.h>
24 #include <sys/socket.h>
25 
26 #include <endian.h>
27 #include <errno.h>
28 #include <err.h>
29 #include <fcntl.h>
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <stdint.h>
34 #include <sha1.h>
35 #include <unistd.h>
36 #include <zlib.h>
37 #include <ctype.h>
38 #include <limits.h>
39 #include <imsg.h>
40 #include <time.h>
41 #include <uuid.h>
42 
43 #include "got_error.h"
44 #include "got_reference.h"
45 #include "got_repository.h"
46 #include "got_path.h"
47 #include "got_cancel.h"
48 #include "got_worktree.h"
49 #include "got_object.h"
50 #include "got_opentemp.h"
51 #include "got_fetch.h"
52 
53 #include "got_lib_delta.h"
54 #include "got_lib_inflate.h"
55 #include "got_lib_object.h"
56 #include "got_lib_object_parse.h"
57 #include "got_lib_object_create.h"
58 #include "got_lib_pack.h"
59 #include "got_lib_sha1.h"
60 #include "got_lib_privsep.h"
61 #include "got_lib_object_cache.h"
62 #include "got_lib_repository.h"
63 #include "got_lib_dial.h"
64 #include "got_lib_pkt.h"
65 
66 #ifndef nitems
67 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
68 #endif
69 
70 #ifndef ssizeof
71 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
72 #endif
73 
74 #ifndef MIN
75 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
76 #endif
77 
78 const struct got_error *
79 got_fetch_connect(pid_t *fetchpid, int *fetchfd, const char *proto,
80     const char *host, const char *port, const char *server_path, int verbosity)
81 {
82 	const struct got_error *err = NULL;
83 
84 	*fetchpid = -1;
85 	*fetchfd = -1;
86 
87 	if (strcmp(proto, "ssh") == 0 || strcmp(proto, "git+ssh") == 0)
88 		err = got_dial_ssh(fetchpid, fetchfd, host, port,
89 		    server_path, GOT_DIAL_DIRECTION_FETCH, verbosity);
90 	else if (strcmp(proto, "git") == 0)
91 		err = got_dial_git(fetchfd, host, port, server_path,
92 		    GOT_DIAL_DIRECTION_FETCH);
93 	else if (strcmp(proto, "http") == 0 || strcmp(proto, "git+http") == 0)
94 		err = got_error_path(proto, GOT_ERR_NOT_IMPL);
95 	else
96 		err = got_error_path(proto, GOT_ERR_BAD_PROTO);
97 	return err;
98 }
99 
100 const struct got_error*
101 got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
102     struct got_pathlist_head *symrefs, const char *remote_name,
103     int mirror_references, int fetch_all_branches,
104     struct got_pathlist_head *wanted_branches,
105     struct got_pathlist_head *wanted_refs, int list_refs_only, int verbosity,
106     int fetchfd, struct got_repository *repo,
107     got_fetch_progress_cb progress_cb, void *progress_arg)
108 {
109 	size_t i;
110 	int imsg_fetchfds[2], imsg_idxfds[2];
111 	int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1;
112 	int tmpfds[3];
113 	int fetchstatus, idxstatus, done = 0;
114 	const struct got_error *err;
115 	struct imsgbuf fetchibuf, idxibuf;
116 	pid_t fetchpid, idxpid;
117 	char *tmppackpath = NULL, *tmpidxpath = NULL;
118 	char *packpath = NULL, *idxpath = NULL, *id_str = NULL;
119 	const char *repo_path = NULL;
120 	struct got_pathlist_head have_refs;
121 	struct got_pathlist_entry *pe;
122 	struct got_reflist_head my_refs;
123 	struct got_reflist_entry *re;
124 	off_t packfile_size = 0;
125 	struct got_packfile_hdr pack_hdr;
126 	uint32_t nobj = 0;
127 	char *ref_prefix = NULL;
128 	size_t ref_prefixlen = 0;
129 	char *path;
130 	char *progress = NULL;
131 
132 	*pack_hash = NULL;
133 
134 	/*
135 	 * Prevent fetching of references that won't make any
136 	 * sense outside of the remote repository's context.
137 	 */
138 	TAILQ_FOREACH(pe, wanted_refs, entry) {
139 		const char *refname = pe->path;
140 		if (strncmp(refname, "refs/got/", 9) == 0 ||
141 		    strncmp(refname, "got/", 4) == 0 ||
142 		    strncmp(refname, "refs/remotes/", 13) == 0 ||
143 		    strncmp(refname, "remotes/", 8) == 0)
144 			return got_error_path(refname, GOT_ERR_FETCH_BAD_REF);
145 	}
146 
147 	if (!list_refs_only)
148 		repo_path = got_repo_get_path_git_dir(repo);
149 
150 	for (i = 0; i < nitems(tmpfds); i++)
151 		tmpfds[i] = -1;
152 
153 	TAILQ_INIT(&have_refs);
154 	TAILQ_INIT(&my_refs);
155 
156 	if (!mirror_references) {
157 		if (asprintf(&ref_prefix, "refs/remotes/%s/",
158 		    remote_name) == -1)
159 			return got_error_from_errno("asprintf");
160 		ref_prefixlen = strlen(ref_prefix);
161 	}
162 
163 	if (!list_refs_only) {
164 		err = got_ref_list(&my_refs, repo, NULL,
165 		    got_ref_cmp_by_name, NULL);
166 		if (err)
167 			goto done;
168 	}
169 
170 	TAILQ_FOREACH(re, &my_refs, entry) {
171 		struct got_object_id *id;
172 		const char *refname;
173 
174 		if (got_ref_is_symbolic(re->ref))
175 			continue;
176 
177 		refname = got_ref_get_name(re->ref);
178 
179 		if (mirror_references) {
180 			char *name;
181 			err = got_ref_resolve(&id, repo, re->ref);
182 			if (err)
183 				goto done;
184 			name = strdup(refname);
185 			if (name == NULL) {
186 				err = got_error_from_errno("strdup");
187 				goto done;
188 			}
189 			err = got_pathlist_append(&have_refs, name, id);
190 			if (err)
191 				goto done;
192 			continue;
193 		}
194 
195 		if (strncmp("refs/tags/", refname, 10) == 0) {
196 			char *tagname;
197 
198 			err = got_ref_resolve(&id, repo, re->ref);
199 			if (err)
200 				goto done;
201 			tagname = strdup(refname);
202 			if (tagname == NULL) {
203 				err = got_error_from_errno("strdup");
204 				goto done;
205 			}
206 			err = got_pathlist_append(&have_refs, tagname, id);
207 			if (err) {
208 				free(tagname);
209 				goto done;
210 			}
211 		}
212 
213 		if (strncmp(ref_prefix, refname, ref_prefixlen) == 0) {
214 			char *branchname;
215 
216 			err = got_ref_resolve(&id, repo, re->ref);
217 			if (err)
218 				goto done;
219 
220 			if (asprintf(&branchname, "refs/heads/%s",
221 			    refname + ref_prefixlen) == -1) {
222 				err = got_error_from_errno("asprintf");
223 				goto done;
224 			}
225 			err = got_pathlist_append(&have_refs, branchname, id);
226 			if (err) {
227 				free(branchname);
228 				goto done;
229 			}
230 		}
231 	}
232 
233 	if (list_refs_only) {
234 		packfd = got_opentempfd();
235 		if (packfd == -1) {
236 			err = got_error_from_errno("got_opentempfd");
237 			goto done;
238 		}
239 	} else {
240 		if (asprintf(&path, "%s/%s/fetching.pack",
241 		    repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
242 			err = got_error_from_errno("asprintf");
243 			goto done;
244 		}
245 		err = got_opentemp_named_fd(&tmppackpath, &packfd, path);
246 		free(path);
247 		if (err)
248 			goto done;
249 		if (fchmod(packfd, GOT_DEFAULT_FILE_MODE) != 0) {
250 			err = got_error_from_errno2("fchmod", tmppackpath);
251 			goto done;
252 		}
253 	}
254 	if (list_refs_only) {
255 		idxfd = got_opentempfd();
256 		if (idxfd == -1) {
257 			err = got_error_from_errno("got_opentempfd");
258 			goto done;
259 		}
260 	} else {
261 		if (asprintf(&path, "%s/%s/fetching.idx",
262 		    repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
263 			err = got_error_from_errno("asprintf");
264 			goto done;
265 		}
266 		err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path);
267 		free(path);
268 		if (err)
269 			goto done;
270 		if (fchmod(idxfd, GOT_DEFAULT_FILE_MODE) != 0) {
271 			err = got_error_from_errno2("fchmod", tmpidxpath);
272 			goto done;
273 		}
274 	}
275 	nidxfd = dup(idxfd);
276 	if (nidxfd == -1) {
277 		err = got_error_from_errno("dup");
278 		goto done;
279 	}
280 
281 	for (i = 0; i < nitems(tmpfds); i++) {
282 		tmpfds[i] = got_opentempfd();
283 		if (tmpfds[i] == -1) {
284 			err = got_error_from_errno("got_opentempfd");
285 			goto done;
286 		}
287 	}
288 
289 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fetchfds) == -1) {
290 		err = got_error_from_errno("socketpair");
291 		goto done;
292 	}
293 
294 	fetchpid = fork();
295 	if (fetchpid == -1) {
296 		err = got_error_from_errno("fork");
297 		goto done;
298 	} else if (fetchpid == 0){
299 		got_privsep_exec_child(imsg_fetchfds,
300 		    GOT_PATH_PROG_FETCH_PACK, tmppackpath);
301 	}
302 
303 	if (close(imsg_fetchfds[1]) == -1) {
304 		err = got_error_from_errno("close");
305 		goto done;
306 	}
307 	imsg_init(&fetchibuf, imsg_fetchfds[0]);
308 	nfetchfd = dup(fetchfd);
309 	if (nfetchfd == -1) {
310 		err = got_error_from_errno("dup");
311 		goto done;
312 	}
313 	err = got_privsep_send_fetch_req(&fetchibuf, nfetchfd, &have_refs,
314 	    fetch_all_branches, wanted_branches, wanted_refs,
315 	    list_refs_only, verbosity);
316 	if (err != NULL)
317 		goto done;
318 	nfetchfd = -1;
319 	npackfd = dup(packfd);
320 	if (npackfd == -1) {
321 		err = got_error_from_errno("dup");
322 		goto done;
323 	}
324 	err = got_privsep_send_fetch_outfd(&fetchibuf, npackfd);
325 	if (err != NULL)
326 		goto done;
327 	npackfd = -1;
328 
329 	packfile_size = 0;
330 	progress = calloc(GOT_PKT_MAX, 1);
331 	if (progress == NULL) {
332 		err = got_error_from_errno("calloc");
333 		goto done;
334 	}
335 
336 	*pack_hash = calloc(1, sizeof(**pack_hash));
337 	if (*pack_hash == NULL) {
338 		err = got_error_from_errno("calloc");
339 		goto done;
340 	}
341 
342 	while (!done) {
343 		struct got_object_id *id = NULL;
344 		char *refname = NULL;
345 		char *server_progress = NULL;
346 		off_t packfile_size_cur = 0;
347 
348 		err = got_privsep_recv_fetch_progress(&done,
349 		    &id, &refname, symrefs, &server_progress,
350 		    &packfile_size_cur, (*pack_hash)->sha1, &fetchibuf);
351 		if (err != NULL)
352 			goto done;
353 		if (!done && refname && id) {
354 			err = got_pathlist_insert(NULL, refs, refname, id);
355 			if (err)
356 				goto done;
357 		} else if (!done && server_progress) {
358 			char *p;
359 			/*
360 			 * XXX git-daemon tends to send batched output with
361 			 * lines spanning separate packets. Buffer progress
362 			 * output until we see a CR or LF to avoid giving
363 			 * partial lines of progress output to the callback.
364 			 */
365 			if (strlcat(progress, server_progress,
366 			    GOT_PKT_MAX) >= GOT_PKT_MAX) {
367 				progress[0] = '\0'; /* discard */
368 				continue;
369 			}
370 			while ((p = strchr(progress, '\r')) != NULL ||
371 			    (p = strchr(progress, '\n')) != NULL) {
372 				char *s;
373 				size_t n;
374 				char c = *p;
375 				*p = '\0';
376 				if (asprintf(&s, "%s%s", progress,
377 				    c == '\n' ? "\n" : "") == -1) {
378 					err = got_error_from_errno("asprintf");
379 					goto done;
380 				}
381 				err = progress_cb(progress_arg, s,
382 				    packfile_size_cur, 0, 0, 0, 0);
383 				free(s);
384 				if (err)
385 					break;
386 				n = strlen(progress);
387 				if (n < GOT_PKT_MAX - 1) {
388 					memmove(progress, &progress[n + 1],
389 					    GOT_PKT_MAX - n - 1);
390 				} else
391 					progress[0] = '\0';
392 			}
393 			free(server_progress);
394 			if (err)
395 				goto done;
396 		} else if (!done && packfile_size_cur != packfile_size) {
397 			err = progress_cb(progress_arg, NULL,
398 			    packfile_size_cur, 0, 0, 0, 0);
399 			if (err)
400 				break;
401 			packfile_size = packfile_size_cur;
402 		}
403 	}
404 	if (waitpid(fetchpid, &fetchstatus, 0) == -1) {
405 		err = got_error_from_errno("waitpid");
406 		goto done;
407 	}
408 
409 	if (lseek(packfd, 0, SEEK_SET) == -1) {
410 		err = got_error_from_errno("lseek");
411 		goto done;
412 	}
413 
414 	/* If zero data was fetched without error we are already up-to-date. */
415 	if (packfile_size == 0) {
416 		free(*pack_hash);
417 		*pack_hash = NULL;
418 		goto done;
419 	} else if (packfile_size < ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
420 		err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
421 		goto done;
422 	} else {
423 		ssize_t n;
424 
425 		n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
426 		if (n == -1) {
427 			err = got_error_from_errno("read");
428 			goto done;
429 		}
430 		if (n != ssizeof(pack_hdr)) {
431 			err = got_error(GOT_ERR_IO);
432 			goto done;
433 		}
434 		if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
435 			err = got_error_msg(GOT_ERR_BAD_PACKFILE,
436 			    "bad pack file signature");
437 			goto done;
438 		}
439 		if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
440 			err = got_error_msg(GOT_ERR_BAD_PACKFILE,
441 			    "bad pack file version");
442 			goto done;
443 		}
444 		nobj = be32toh(pack_hdr.nobjects);
445 		if (nobj == 0 &&
446 		    packfile_size > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH)
447 			return got_error_msg(GOT_ERR_BAD_PACKFILE,
448 			    "bad pack file with zero objects");
449 		if (nobj != 0 &&
450 		    packfile_size <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH)
451 			return got_error_msg(GOT_ERR_BAD_PACKFILE,
452 			    "empty pack file with non-zero object count");
453 	}
454 
455 	/*
456 	 * If the pack file contains no objects, we may only need to update
457 	 * references in our repository. The caller will take care of that.
458 	 */
459 	if (nobj == 0)
460 		goto done;
461 
462 	if (lseek(packfd, 0, SEEK_SET) == -1) {
463 		err = got_error_from_errno("lseek");
464 		goto done;
465 	}
466  
467 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
468 		err = got_error_from_errno("socketpair");
469 		goto done;
470 	}
471 	idxpid = fork();
472 	if (idxpid == -1) {
473 		err= got_error_from_errno("fork");
474 		goto done;
475 	} else if (idxpid == 0)
476 		got_privsep_exec_child(imsg_idxfds,
477 		    GOT_PATH_PROG_INDEX_PACK, tmppackpath);
478 	if (close(imsg_idxfds[1]) == -1) {
479 		err = got_error_from_errno("close");
480 		goto done;
481 	}
482 	imsg_init(&idxibuf, imsg_idxfds[0]);
483 
484 	npackfd = dup(packfd);
485 	if (npackfd == -1) {
486 		err = got_error_from_errno("dup");
487 		goto done;
488 	}
489 	err = got_privsep_send_index_pack_req(&idxibuf, (*pack_hash)->sha1,
490 	    npackfd);
491 	if (err != NULL)
492 		goto done;
493 	npackfd = -1;
494 	err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
495 	if (err != NULL)
496 		goto done;
497 	nidxfd = -1;
498 	for (i = 0; i < nitems(tmpfds); i++) {
499 		err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
500 		if (err != NULL)
501 			goto done;
502 		tmpfds[i] = -1;
503 	}
504 	done = 0;
505 	while (!done) {
506 		int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
507 
508 		err = got_privsep_recv_index_progress(&done, &nobj_total,
509 		    &nobj_indexed, &nobj_loose, &nobj_resolved,
510 		    &idxibuf);
511 		if (err != NULL)
512 			goto done;
513 		if (nobj_indexed != 0) {
514 			err = progress_cb(progress_arg, NULL,
515 			    packfile_size, nobj_total,
516 			    nobj_indexed, nobj_loose, nobj_resolved);
517 			if (err)
518 				break;
519 		}
520 		imsg_clear(&idxibuf);
521 	}
522 	if (close(imsg_idxfds[0]) == -1) {
523 		err = got_error_from_errno("close");
524 		goto done;
525 	}
526 	if (waitpid(idxpid, &idxstatus, 0) == -1) {
527 		err = got_error_from_errno("waitpid");
528 		goto done;
529 	}
530 
531 	err = got_object_id_str(&id_str, *pack_hash);
532 	if (err)
533 		goto done;
534 	if (asprintf(&packpath, "%s/%s/pack-%s.pack",
535 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
536 		err = got_error_from_errno("asprintf");
537 		goto done;
538 	}
539 
540 	if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
541 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
542 		err = got_error_from_errno("asprintf");
543 		goto done;
544 	}
545 
546 	if (rename(tmppackpath, packpath) == -1) {
547 		err = got_error_from_errno3("rename", tmppackpath, packpath);
548 		goto done;
549 	}
550 	free(tmppackpath);
551 	tmppackpath = NULL;
552 	if (rename(tmpidxpath, idxpath) == -1) {
553 		err = got_error_from_errno3("rename", tmpidxpath, idxpath);
554 		goto done;
555 	}
556 	free(tmpidxpath);
557 	tmpidxpath = NULL;
558 
559 done:
560 	if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
561 		err = got_error_from_errno2("unlink", tmppackpath);
562 	if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
563 		err = got_error_from_errno2("unlink", tmpidxpath);
564 	if (nfetchfd != -1 && close(nfetchfd) == -1 && err == NULL)
565 		err = got_error_from_errno("close");
566 	if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
567 		err = got_error_from_errno("close");
568 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
569 		err = got_error_from_errno("close");
570 	if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
571 		err = got_error_from_errno("close");
572 	for (i = 0; i < nitems(tmpfds); i++) {
573 		if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
574 			err = got_error_from_errno("close");
575 	}
576 	free(tmppackpath);
577 	free(tmpidxpath);
578 	free(idxpath);
579 	free(packpath);
580 	free(ref_prefix);
581 	free(progress);
582 
583 	TAILQ_FOREACH(pe, &have_refs, entry) {
584 		free((char *)pe->path);
585 		free(pe->data);
586 	}
587 	got_pathlist_free(&have_refs);
588 	got_ref_list_free(&my_refs);
589 	if (err) {
590 		free(*pack_hash);
591 		*pack_hash = NULL;
592 		TAILQ_FOREACH(pe, refs, entry) {
593 			free((void *)pe->path);
594 			free(pe->data);
595 		}
596 		got_pathlist_free(refs);
597 		TAILQ_FOREACH(pe, symrefs, entry) {
598 			free((void *)pe->path);
599 			free(pe->data);
600 		}
601 		got_pathlist_free(symrefs);
602 	}
603 	return err;
604 }