Gotweb

Blob

Date:: Sat Sep 15 18:05:22 2018 UTC
Message:: fix non-mmap read mode in got-read-pack
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/limits.h>
22 #include <sys/syslimits.h>
23 #include <sys/mman.h>
24 
25 #include <limits.h>
26 #include <stdint.h>
27 #include <imsg.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <sha1.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 
37 #include "got_lib_delta.h"
38 #include "got_lib_inflate.h"
39 #include "got_lib_object.h"
40 #include "got_lib_object_cache.h"
41 #include "got_lib_object_parse.h"
42 #include "got_lib_privsep.h"
43 #include "got_lib_pack.h"
44 
45 static const struct got_error *
46 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
47     struct got_packidx *packidx, struct got_object_cache *objcache)
48 {
49 	const struct got_error *err = NULL;
50 	struct got_imsg_packed_object iobj;
51 	struct got_object *obj;
52 	size_t datalen;
53 
54 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
55 	if (datalen != sizeof(iobj))
56 		return got_error(GOT_ERR_PRIVSEP_LEN);
57 	memcpy(&iobj, imsg->data, sizeof(iobj));
58 
59 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, NULL);
60 	if (err)
61 		return err;
62 	obj->refcnt++;
63 
64 	err = got_object_cache_add(objcache, &obj->id, obj);
65 	if (err)
66 		goto done;
67 	obj->refcnt++;
68 
69 	err = got_privsep_send_obj(ibuf, obj);
70 done:
71 	got_object_close(obj);
72 	return err;
73 }
74 
75 static const struct got_error *
76 get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf,
77     struct got_pack *pack, struct got_packidx *packidx,
78     struct got_object_cache *objcache, int type)
79 {
80 	const struct got_error *err = NULL;
81 	struct got_object *iobj;
82 
83 	err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf);
84 	if (err)
85 		return err;
86 
87 	if (iobj->type != type) {
88 		err = got_error(GOT_ERR_OBJ_TYPE);
89 		goto done;
90 	}
91 
92 	if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0)
93 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
94 
95 	*obj = got_object_cache_get(objcache, &iobj->id);
96 	if (*obj == NULL) {
97 		err = got_packfile_open_object(obj, pack, packidx,
98 		    iobj->pack_idx, &iobj->id);
99 		if (err)
100 			goto done;
101 	}
102 	(*obj)->refcnt++;
103 done:
104 	got_object_close(iobj);
105 	return err;
106 }
107 
108 static const struct got_error *
109 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
110     struct got_packidx *packidx, struct got_object_cache *objcache)
111 {
112 	const struct got_error *err = NULL;
113 	struct got_object *obj = NULL;
114 	struct got_commit_object *commit = NULL;
115 	uint8_t *buf;
116 	size_t len;
117 
118 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
119 	    GOT_OBJ_TYPE_COMMIT);
120 	if (err)
121 		return err;
122 
123 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
124 	if (err)
125 		return err;
126 
127 	obj->size = len;
128 	err = got_object_parse_commit(&commit, buf, len);
129 	free(buf);
130 
131 	err = got_privsep_send_commit(ibuf, commit);
132 	if (obj)
133 		got_object_close(obj);
134 	got_object_commit_close(commit);
135 	if (err) {
136 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
137 			err = NULL;
138 		else
139 			got_privsep_send_error(ibuf, err);
140 	}
141 
142 	return err;
143 }
144 
145 static const struct got_error *
146 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
147     struct got_packidx *packidx, struct got_object_cache *objcache)
148 {
149 	const struct got_error *err = NULL;
150 	struct got_object *obj = NULL;
151 	struct got_tree_object *tree = NULL;
152 	uint8_t *buf;
153 	size_t len;
154 
155 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
156 	    GOT_OBJ_TYPE_TREE);
157 	if (err)
158 		return err;
159 
160 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
161 	if (err)
162 		return err;
163 
164 	obj->size = len;
165 	err = got_object_parse_tree(&tree, buf, len);
166 	free(buf);
167 
168 	err = got_privsep_send_tree(ibuf, tree);
169 	if (obj)
170 		got_object_close(obj);
171 	got_object_tree_close(tree);
172 	if (err) {
173 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
174 			err = NULL;
175 		else
176 			got_privsep_send_error(ibuf, err);
177 	}
178 
179 	return err;
180 }
181 
182 static const struct got_error *
183 receive_file(FILE **f, struct imsgbuf *ibuf, int imsg_code)
184 {
185 	const struct got_error *err;
186 	struct imsg imsg;
187 	size_t datalen;
188 
189 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
190 	if (err)
191 		return err;
192 
193 	if (imsg.hdr.type != imsg_code) {
194 		err = got_error(GOT_ERR_PRIVSEP_MSG);
195 		goto done;
196 	}
197 
198 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
199 	if (datalen != 0) {
200 		err = got_error(GOT_ERR_PRIVSEP_LEN);
201 		goto done;
202 	}
203 	if (imsg.fd == -1) {
204 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
205 		goto done;
206 	}
207 
208 	*f = fdopen(imsg.fd, "w+");
209 	if (*f == NULL) {
210 		close(imsg.fd);
211 		err = got_error_from_errno();
212 		goto done;
213 	}
214 done:
215 	imsg_free(&imsg);
216 	return err;
217 }
218 
219 static const struct got_error *
220 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
221     struct got_packidx *packidx, struct got_object_cache *objcache)
222 {
223 	const struct got_error *err = NULL;
224 	struct got_object *obj = NULL;
225 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
226 
227 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
228 	    GOT_OBJ_TYPE_BLOB);
229 	if (err)
230 		return err;
231 
232 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
233 	if (err)
234 		return err;
235 	err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD);
236 	if (err)
237 		return err;
238 	err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD);
239 	if (err)
240 		return err;
241 
242 	err = got_packfile_extract_object(pack, obj, outfile, basefile,
243 	    accumfile);
244 	if (err)
245 		goto done;
246 
247 	err = got_privsep_send_blob(ibuf, obj->size);
248 done:
249 	if (outfile)
250 		fclose(outfile);
251 	if (basefile)
252 		fclose(basefile);
253 	if (accumfile)
254 		fclose(accumfile);
255 	if (obj)
256 		got_object_close(obj);
257 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
258 		got_privsep_send_error(ibuf, err);
259 
260 	return err;
261 }
262 
263 static const struct got_error *
264 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
265 {
266 	const struct got_error *err = NULL;
267 	struct imsg imsg;
268 	struct got_imsg_packidx ipackidx;
269 	size_t datalen;
270 	struct got_packidx *p;
271 
272 	*packidx = NULL;
273 
274 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
275 	if (err)
276 		return err;
277 
278 	p = calloc(1, sizeof(*p));
279 	if (p == NULL) {
280 		err = got_error_from_errno();
281 		goto done;
282 	}
283 
284 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
285 		err = got_error(GOT_ERR_PRIVSEP_MSG);
286 		goto done;
287 	}
288 
289 	if (imsg.fd == -1) {
290 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
291 		goto done;
292 	}
293 
294 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
295 	if (datalen != sizeof(ipackidx)) {
296 		err = got_error(GOT_ERR_PRIVSEP_LEN);
297 		goto done;
298 	}
299 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
300 
301 	p->len = ipackidx.len;
302 	p->fd = dup(imsg.fd);
303 	if (p->fd == -1) {
304 		err = got_error_from_errno();
305 		goto done;
306 	}
307 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
308 		err = got_error_from_errno();
309 		goto done;
310 	}
311 
312 #ifndef GOT_PACK_NO_MMAP
313 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
314 	if (p->map == MAP_FAILED)
315 		p->map = NULL; /* fall back to read(2) */
316 #endif
317 	err = got_packidx_init_hdr(p, 1);
318 done:
319 	if (err) {
320 		if (imsg.fd != -1)
321 			close(imsg.fd);
322 		got_packidx_close(p);
323 	} else
324 		*packidx = p;
325 	imsg_free(&imsg);
326 	return err;
327 }
328 
329 static const struct got_error *
330 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
331 {
332 	const struct got_error *err = NULL;
333 	struct imsg imsg;
334 	struct got_imsg_pack ipack;
335 	size_t datalen;
336 	struct got_pack *pack;
337 
338 	*packp = NULL;
339 
340 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
341 	if (err)
342 		return err;
343 
344 	pack = calloc(1, sizeof(*pack));
345 	if (pack == NULL) {
346 		err = got_error_from_errno();
347 		goto done;
348 	}
349 
350 	if (imsg.hdr.type != GOT_IMSG_PACK) {
351 		err = got_error(GOT_ERR_PRIVSEP_MSG);
352 		goto done;
353 	}
354 
355 	if (imsg.fd == -1) {
356 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
357 		goto done;
358 	}
359 
360 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
361 	if (datalen != sizeof(ipack)) {
362 		err = got_error(GOT_ERR_PRIVSEP_LEN);
363 		goto done;
364 	}
365 	memcpy(&ipack, imsg.data, sizeof(ipack));
366 
367 	pack->filesize = ipack.filesize;
368 	pack->fd = dup(imsg.fd);
369 	if (pack->fd == -1) {
370 		err = got_error_from_errno();
371 		goto done;
372 	}
373 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
374 		err = got_error_from_errno();
375 		goto done;
376 	}
377 	pack->path_packfile = strdup(ipack.path_packfile);
378 	if (pack->path_packfile == NULL) {
379 		err = got_error_from_errno();
380 		goto done;
381 	}
382 
383 #ifndef GOT_PACK_NO_MMAP
384 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
385 	    pack->fd, 0);
386 	if (pack->map == MAP_FAILED)
387 		pack->map = NULL; /* fall back to read(2) */
388 #endif
389 done:
390 	if (err) {
391 		if (imsg.fd != -1)
392 			close(imsg.fd);
393 		free(pack);
394 	} else
395 		*packp = pack;
396 	imsg_free(&imsg);
397 	return err;
398 }
399 
400 int
401 main(int argc, char *argv[])
402 {
403 	const struct got_error *err = NULL;
404 	struct imsgbuf ibuf;
405 	struct imsg imsg;
406 	struct got_packidx *packidx = NULL;
407 	struct got_pack *pack = NULL;
408 	struct got_object_cache objcache;
409 
410 	//static int attached;
411 	//while (!attached) sleep(1);
412 
413 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
414 
415 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
416 	if (err) {
417 		err = got_error_from_errno();
418 		got_privsep_send_error(&ibuf, err);
419 		return 1;
420 	}
421 
422 #ifndef PROFILE
423 	/* revoke access to most system calls */
424 	if (pledge("stdio recvfd", NULL) == -1) {
425 		err = got_error_from_errno();
426 		got_privsep_send_error(&ibuf, err);
427 		return 1;
428 	}
429 #endif
430 
431 	err = receive_packidx(&packidx, &ibuf);
432 	if (err) {
433 		got_privsep_send_error(&ibuf, err);
434 		return 1;
435 	}
436 
437 	err = receive_pack(&pack, &ibuf);
438 	if (err) {
439 		got_privsep_send_error(&ibuf, err);
440 		return 1;
441 	}
442 
443 	while (1) {
444 		imsg.fd = -1;
445 
446 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
447 		if (err) {
448 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
449 				err = NULL;
450 			break;
451 		}
452 
453 		if (imsg.hdr.type == GOT_IMSG_STOP)
454 			break;
455 
456 		switch (imsg.hdr.type) {
457 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
458 			err = object_request(&imsg, &ibuf, pack, packidx,
459 			    &objcache);
460 			break;
461 		case GOT_IMSG_COMMIT_REQUEST:
462 			err = commit_request(&imsg, &ibuf, pack, packidx,
463 			    &objcache);
464 			break;
465 		case GOT_IMSG_TREE_REQUEST:
466 			err = tree_request(&imsg, &ibuf, pack, packidx,
467 			   &objcache);
468 			break;
469 		case GOT_IMSG_BLOB_REQUEST:
470 			err = blob_request(&imsg, &ibuf, pack, packidx,
471 			   &objcache);
472 			break;
473 		default:
474 			err = got_error(GOT_ERR_PRIVSEP_MSG);
475 			break;
476 		}
477 
478 		if (imsg.fd != -1)
479 			close(imsg.fd);
480 		imsg_free(&imsg);
481 		if (err) {
482 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
483 				err = NULL;
484 			else
485 				got_privsep_send_error(&ibuf, err);
486 			break;
487 		}
488 	}
489 
490 	if (packidx)
491 		got_packidx_close(packidx);
492 	if (pack)
493 		got_pack_close(pack);
494 	imsg_clear(&ibuf);
495 	if (err)
496 		fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
497 	close(GOT_IMSG_FD_CHILD);
498 	return err ? 1 : 0;
499 }