Gotweb

Blob

Date:: Mon Jul 10 20:45:36 2023 UTC
Message:: portable: enable cvg
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2023 Omar Polo <op@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include "got_compat.h"
18 
19 #include <sys/queue.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/time.h>
23 #include <sys/types.h>
24 #include <sys/uio.h>
25 #include <sys/wait.h>
26 
27 #include <limits.h>
28 #include <stdint.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <imsg.h>
34 
35 #include "got_error.h"
36 #include "got_cancel.h"
37 #include "got_object.h"
38 #include "got_opentemp.h"
39 #include "got_path.h"
40 #include "got_reference.h"
41 #include "got_repository.h"
42 #include "got_repository_load.h"
43 
44 #include "got_lib_delta.h"
45 #include "got_lib_hash.h"
46 #include "got_lib_object.h"
47 #include "got_lib_object_cache.h"
48 #include "got_lib_pack.h"
49 #include "got_lib_ratelimit.h"
50 #include "got_lib_repository.h"
51 #include "got_lib_privsep.h"
52 
53 #define GIT_BUNDLE_SIGNATURE_V2 "# v2 git bundle\n"
54 
55 #ifndef nitems
56 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
57 #endif
58 
59 #ifndef ssizeof
60 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
61 #endif
62 
63 static const struct got_error *
64 temp_file(int *fd, char **path, const char *ext, struct got_repository *repo)
65 {
66 	const struct got_error *err;
67 	char p[PATH_MAX];
68 	int r;
69 
70 	*path = NULL;
71 
72 	r = snprintf(p, sizeof(p), "%s/%s/loading",
73 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR);
74 	if (r < 0 || (size_t)r >= sizeof(p))
75 		return got_error_from_errno("snprintf");
76 
77 	err = got_opentemp_named_fd(path, fd, p, ext);
78 	if (err)
79 		return err;
80 
81 	if (fchmod(*fd, GOT_DEFAULT_FILE_MODE) == -1)
82 		return got_error_from_errno("fchmod");
83 
84 	return NULL;
85 }
86 
87 static const struct got_error *
88 load_report_progress(got_load_progress_cb progress_cb, void *progress_arg,
89     struct got_ratelimit *rl, off_t packsiz, int nobj_total,
90     int nobj_indexed, int nobj_loose, int nobj_resolved)
91 {
92 	const struct got_error *err;
93 	int elapsed;
94 
95 	if (progress_cb == NULL)
96 		return NULL;
97 
98 	err = got_ratelimit_check(&elapsed, rl);
99 	if (err || !elapsed)
100 		return err;
101 
102 	return progress_cb(progress_arg, packsiz, nobj_total, nobj_indexed,
103 	    nobj_loose, nobj_resolved);
104 }
105 
106 static const struct got_error *
107 copypack(FILE *in, int outfd, off_t *tot,
108     struct got_object_id *id, struct got_ratelimit *rl,
109     got_load_progress_cb progress_cb, void *progress_arg,
110     got_cancel_cb cancel_cb, void *cancel_arg)
111 {
112 	const struct got_error *err;
113 	struct got_hash hash;
114 	struct got_object_id expected_id;
115 	char buf[BUFSIZ], sha1buf[SHA1_DIGEST_LENGTH];
116 	size_t r, sha1buflen = 0;
117 
118 	*tot = 0;
119 	got_hash_init(&hash, GOT_HASH_SHA1);
120 
121 	for (;;) {
122 		err = cancel_cb(cancel_arg);
123 		if (err)
124 			return err;
125 
126 		r = fread(buf, 1, sizeof(buf), in);
127 		if (r == 0)
128 			break;
129 
130 		/*
131 		 * An expected SHA1 checksum sits at the end of the
132 		 * pack file.  Since we don't know the file size ahead
133 		 * of time we have to keep SHA1_DIGEST_LENGTH bytes
134 		 * buffered and avoid mixing those bytes int our hash
135 		 * computation until we know for sure that additional
136 		 * pack file data bytes follow.
137 		 *
138 		 * We can assume that BUFSIZE is greater than
139 		 * SHA1_DIGEST_LENGTH and that a short read means that
140 		 * we've reached EOF.
141 		 */
142 
143 		if (r >= sizeof(sha1buf)) {
144 			*tot += sha1buflen;
145 			got_hash_update(&hash, sha1buf, sha1buflen);
146 			if (write(outfd, sha1buf, sha1buflen) == -1)
147 				return got_error_from_errno("write");
148 
149 			r -= sizeof(sha1buf);
150 			memcpy(sha1buf, &buf[r], sizeof(sha1buf));
151 			sha1buflen = sizeof(sha1buf);
152 
153 			*tot += r;
154 			got_hash_update(&hash, buf, r);
155 			if (write(outfd, buf, r) == -1)
156 				return got_error_from_errno("write");
157 
158 			err = load_report_progress(progress_cb, progress_arg,
159 			    rl, *tot, 0, 0, 0, 0);
160 			if (err)
161 				return err;
162 
163 			continue;
164 		}
165 
166 		if (sha1buflen == 0)
167 			return got_error(GOT_ERR_BAD_PACKFILE);
168 
169 		/* short read, we've reached EOF */
170 		*tot += r;
171 		got_hash_update(&hash, sha1buf, r);
172 		if (write(outfd, sha1buf, r) == -1)
173 			return got_error_from_errno("write");
174 
175 		memmove(&sha1buf[0], &sha1buf[r], sizeof(sha1buf) - r);
176 		memcpy(&sha1buf[sizeof(sha1buf) - r], buf, r);
177 		break;
178 	}
179 
180 	if (sha1buflen == 0)
181 		return got_error(GOT_ERR_BAD_PACKFILE);
182 
183 	got_hash_final_object_id(&hash, id);
184 
185 	/* XXX SHA256 */
186 	memset(&expected_id, 0, sizeof(expected_id));
187 	memcpy(&expected_id.sha1, sha1buf, sizeof(expected_id.sha1));
188 
189 	if (got_object_id_cmp(id, &expected_id) != 0)
190 		return got_error(GOT_ERR_PACKIDX_CSUM);
191 
192 	/* re-add the expected hash at the end of the pack */
193 	if (write(outfd, sha1buf, sizeof(sha1buf)) == -1)
194 		return got_error_from_errno("write");
195 
196 	*tot += sizeof(sha1buf);
197 	err = progress_cb(progress_arg, *tot, 0, 0, 0, 0);
198 	if (err)
199 		return err;
200 
201 	return NULL;
202 }
203 
204 const struct got_error *
205 got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
206     struct got_repository *repo, int list_refs_only, int noop,
207     got_load_progress_cb progress_cb, void *progress_arg,
208     got_cancel_cb cancel_cb, void *cancel_arg)
209 {
210 	const struct got_error *err = NULL;
211 	struct got_object_id id;
212 	struct got_object *obj;
213 	struct got_packfile_hdr pack_hdr;
214 	struct got_ratelimit rl;
215 	struct imsgbuf idxibuf;
216 	const char *repo_path;
217 	char *packpath = NULL, *idxpath = NULL;
218 	char *tmppackpath = NULL, *tmpidxpath = NULL;
219 	int packfd = -1, idxfd = -1;
220 	char *spc, *refname, *id_str = NULL;
221 	char *line = NULL;
222 	size_t linesize = 0;
223 	ssize_t linelen;
224 	size_t i;
225 	ssize_t n;
226 	off_t packsiz;
227 	int tmpfds[3] = {-1, -1, -1};
228 	int imsg_idxfds[2] = {-1, -1};
229 	int ch, done, nobj, idxstatus;
230 	pid_t idxpid;
231 
232 	got_ratelimit_init(&rl, 0, 500);
233 
234 	repo_path = got_repo_get_path_git_dir(repo);
235 
236 	linelen = getline(&line, &linesize, in);
237 	if (linelen == -1) {
238 		err = got_ferror(in, GOT_ERR_IO);
239 		goto done;
240 	}
241 
242 	if (strcmp(line, GIT_BUNDLE_SIGNATURE_V2) != 0) {
243 		err = got_error(GOT_ERR_BUNDLE_FORMAT);
244 		goto done;
245 	}
246 
247 	/* Parse the prerequisite */
248 	for (;;) {
249 		ch = fgetc(in);
250 		if (ch != '-') {
251 			if (ch != EOF)
252 				ungetc(ch, in);
253 			break;
254 		}
255 
256 		linelen = getline(&line, &linesize, in);
257 		if (linelen == -1) {
258 			err = got_ferror(in, GOT_ERR_IO);
259 			goto done;
260 		}
261 
262 		if (line[linelen - 1] == '\n')
263 			line[linelen - 1] = '\0';
264 
265 		if (!got_parse_object_id(&id, line, GOT_HASH_SHA1)) {
266 			err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR);
267 			goto done;
268 		}
269 
270 		err = got_object_open(&obj, repo, &id);
271 		if (err)
272 			goto done;
273 		got_object_close(obj);
274 	}
275 
276 	/* Read references */
277 	for (;;) {
278 		struct got_object_id *id;
279 		char *dup;
280 
281 		linelen = getline(&line, &linesize, in);
282 		if (linelen == -1) {
283 			err = got_ferror(in, GOT_ERR_IO);
284 			goto done;
285 		}
286 		if (line[linelen - 1] == '\n')
287 			line[linelen - 1] = '\0';
288 		if (*line == '\0')
289 			break;
290 
291 		spc = strchr(line, ' ');
292 		if (spc == NULL) {
293 			err = got_error(GOT_ERR_IO);
294 			goto done;
295 		}
296 		*spc = '\0';
297 
298 		refname = spc + 1;
299 		if (!got_ref_name_is_valid(refname)) {
300 			err = got_error(GOT_ERR_BAD_REF_DATA);
301 			goto done;
302 		}
303 
304 		id = malloc(sizeof(*id));
305 		if (id == NULL) {
306 			err = got_error_from_errno("malloc");
307 			goto done;
308 		}
309 
310 		if (!got_parse_object_id(id, line, GOT_HASH_SHA1)) {
311 			free(id);
312 			err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
313 			goto done;
314 		}
315 
316 		dup = strdup(refname);
317 		if (dup == NULL) {
318 			free(id);
319 			err = got_error_from_errno("strdup");
320 			goto done;
321 		}
322 
323 		err = got_pathlist_append(refs_found, dup, id);
324 		if (err) {
325 			free(id);
326 			free(dup);
327 			goto done;
328 		}
329 	}
330 
331 	if (list_refs_only)
332 		goto done;
333 
334 	err = temp_file(&packfd, &tmppackpath, ".pack", repo);
335 	if (err)
336 		goto done;
337 
338 	err = temp_file(&idxfd, &tmpidxpath, ".idx", repo);
339 	if (err)
340 		goto done;
341 
342 	err = copypack(in, packfd, &packsiz, &id, &rl,
343 	    progress_cb, progress_arg, cancel_cb, cancel_arg);
344 	if (err)
345 		goto done;
346 
347 	if (lseek(packfd, 0, SEEK_SET) == -1) {
348 		err = got_error_from_errno("lseek");
349 		goto done;
350 	}
351 
352 	/* Safety checks on the pack' content. */
353 	if (packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
354 		err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
355 		goto done;
356 	}
357 
358 	n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
359 	if (n == -1) {
360 		err = got_error_from_errno("read");
361 		goto done;
362 	}
363 	if (n != ssizeof(pack_hdr)) {
364 		err = got_error(GOT_ERR_IO);
365 		goto done;
366 	}
367 	if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
368 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
369 		    "bad pack file signature");
370 		goto done;
371 	}
372 	if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
373 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
374 		    "bad pack file version");
375 		goto done;
376 	}
377 	nobj = be32toh(pack_hdr.nobjects);
378 	if (nobj == 0 &&
379 	    packsiz > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
380 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
381 		    "bad pack file with zero objects");
382 		goto done;
383 	}
384 	if (nobj != 0 &&
385 	    packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
386 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
387 		    "empty pack file with non-zero object count");
388 		goto done;
389 	}
390 
391 	/* nothing to do if there are no objects. */
392 	if (nobj == 0)
393 		goto done;
394 
395 	for (i = 0; i < nitems(tmpfds); i++) {
396 		tmpfds[i] = got_opentempfd();
397 		if (tmpfds[i] == -1) {
398 			err = got_error_from_errno("got_opentempfd");
399 			goto done;
400 		}
401 	}
402 
403 	if (lseek(packfd, 0, SEEK_SET) == -1) {
404 		err = got_error_from_errno("lseek");
405 		goto done;
406 	}
407 
408 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
409 		err = got_error_from_errno("socketpair");
410 		goto done;
411 	}
412 	idxpid = fork();
413 	if (idxpid == -1) {
414 		err= got_error_from_errno("fork");
415 		goto done;
416 	} else if (idxpid == 0)
417 		got_privsep_exec_child(imsg_idxfds,
418 		    GOT_PATH_PROG_INDEX_PACK, tmppackpath);
419 	if (close(imsg_idxfds[1]) == -1) {
420 		err = got_error_from_errno("close");
421 		goto done;
422 	}
423 	imsg_idxfds[1] = -1;
424 	imsg_init(&idxibuf, imsg_idxfds[0]);
425 
426 	err = got_privsep_send_index_pack_req(&idxibuf, id.sha1, packfd);
427 	if (err)
428 		goto done;
429 	packfd = -1;
430 
431 	err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
432 	if (err)
433 		goto done;
434 	idxfd = -1;
435 
436 	for (i = 0; i < nitems(tmpfds); i++) {
437 		err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
438 		if (err != NULL)
439 			goto done;
440 		tmpfds[i] = -1;
441 	}
442 
443 	done = 0;
444 	while (!done) {
445 		int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
446 
447 		err = got_privsep_recv_index_progress(&done, &nobj_total,
448 		    &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
449 		if (err)
450 			goto done;
451 		if (nobj_indexed != 0) {
452 			err = load_report_progress(progress_cb, progress_arg,
453 			    &rl, packsiz, nobj_total, nobj_indexed,
454 			    nobj_loose, nobj_resolved);
455 			if (err)
456 				goto done;
457 		}
458 	}
459 	if (close(imsg_idxfds[0]) == -1) {
460 		err = got_error_from_errno("close");
461 		goto done;
462 	}
463 	imsg_idxfds[0] = -1;
464 	if (waitpid(idxpid, &idxstatus, 0) == -1) {
465 		err = got_error_from_errno("waitpid");
466 		goto done;
467 	}
468 
469 	if (noop)
470 		goto done;
471 
472 	err = got_object_id_str(&id_str, &id);
473 	if (err)
474 		goto done;
475 
476 	if (asprintf(&packpath, "%s/%s/pack-%s.pack", repo_path,
477 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
478 		err = got_error_from_errno("asprintf");
479 		goto done;
480 	}
481 
482 	if (asprintf(&idxpath, "%s/%s/pack-%s.idx", repo_path,
483 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
484 		err = got_error_from_errno("asprintf");
485 		goto done;
486 	}
487 
488 	if (rename(tmppackpath, packpath) == -1) {
489 		err = got_error_from_errno3("rename", tmppackpath, packpath);
490 		goto done;
491 	}
492 	free(tmppackpath);
493 	tmppackpath = NULL;
494 
495 	if (rename(tmpidxpath, idxpath) == -1) {
496 		err = got_error_from_errno3("rename", tmpidxpath, idxpath);
497 		goto done;
498 	}
499 	free(tmpidxpath);
500 	tmpidxpath = NULL;
501 
502  done:
503 	free(line);
504 	free(packpath);
505 	free(idxpath);
506 	free(id_str);
507 
508 	if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
509 		err = got_error_from_errno2("unlink", tmppackpath);
510 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
511 		err = got_error_from_errno("close");
512 	free(tmppackpath);
513 
514 	if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
515 		err = got_error_from_errno2("unlink", tmpidxpath);
516 	if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
517 		err = got_error_from_errno("close");
518 	free(tmpidxpath);
519 
520 	if (imsg_idxfds[0] != -1 && close(imsg_idxfds[0]) == -1 && err == NULL)
521 		err = got_error_from_errno("close");
522 	if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
523 		err = got_error_from_errno("close");
524 
525 	for (i = 0; i < nitems(tmpfds); ++i)
526 		if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
527 			err = got_error_from_errno("close");
528 
529 	return err;
530 }