Gotweb

Blob

Date:: Sat Sep 15 18:57:10 2018 UTC
Message:: make object caching actually work in got-read-pack
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/limits.h>
22 #include <sys/syslimits.h>
23 #include <sys/mman.h>
24 
25 #include <limits.h>
26 #include <stdint.h>
27 #include <imsg.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <sha1.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 
37 #include "got_lib_delta.h"
38 #include "got_lib_inflate.h"
39 #include "got_lib_object.h"
40 #include "got_lib_object_cache.h"
41 #include "got_lib_object_parse.h"
42 #include "got_lib_privsep.h"
43 #include "got_lib_pack.h"
44 
45 static const struct got_error *
46 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
47     struct got_packidx *packidx, struct got_object_cache *objcache)
48 {
49 	const struct got_error *err = NULL;
50 	struct got_imsg_packed_object iobj;
51 	struct got_object *obj;
52 	struct got_object_id id;
53 	size_t datalen;
54 
55 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
56 	if (datalen != sizeof(iobj))
57 		return got_error(GOT_ERR_PRIVSEP_LEN);
58 	memcpy(&iobj, imsg->data, sizeof(iobj));
59 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
60 
61 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
62 	if (err)
63 		return err;
64 	obj->refcnt++;
65 
66 	err = got_object_cache_add(objcache, &obj->id, obj);
67 	if (err)
68 		goto done;
69 	obj->refcnt++;
70 
71 	err = got_privsep_send_obj(ibuf, obj);
72 done:
73 	got_object_close(obj);
74 	return err;
75 }
76 
77 static const struct got_error *
78 get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf,
79     struct got_pack *pack, struct got_packidx *packidx,
80     struct got_object_cache *objcache, int type)
81 {
82 	const struct got_error *err = NULL;
83 	struct got_object *iobj;
84 
85 	err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf);
86 	if (err)
87 		return err;
88 
89 	if (iobj->type != type) {
90 		err = got_error(GOT_ERR_OBJ_TYPE);
91 		goto done;
92 	}
93 
94 	if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0)
95 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
96 
97 	*obj = got_object_cache_get(objcache, &iobj->id);
98 	if (*obj == NULL) {
99 		err = got_packfile_open_object(obj, pack, packidx,
100 		    iobj->pack_idx, &iobj->id);
101 		if (err)
102 			goto done;
103 	}
104 	(*obj)->refcnt++;
105 done:
106 	got_object_close(iobj);
107 	return err;
108 }
109 
110 static const struct got_error *
111 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
112     struct got_packidx *packidx, struct got_object_cache *objcache)
113 {
114 	const struct got_error *err = NULL;
115 	struct got_object *obj = NULL;
116 	struct got_commit_object *commit = NULL;
117 	uint8_t *buf;
118 	size_t len;
119 
120 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
121 	    GOT_OBJ_TYPE_COMMIT);
122 	if (err)
123 		return err;
124 
125 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
126 	if (err)
127 		return err;
128 
129 	obj->size = len;
130 	err = got_object_parse_commit(&commit, buf, len);
131 	free(buf);
132 
133 	err = got_privsep_send_commit(ibuf, commit);
134 	if (obj)
135 		got_object_close(obj);
136 	got_object_commit_close(commit);
137 	if (err) {
138 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
139 			err = NULL;
140 		else
141 			got_privsep_send_error(ibuf, err);
142 	}
143 
144 	return err;
145 }
146 
147 static const struct got_error *
148 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
149     struct got_packidx *packidx, struct got_object_cache *objcache)
150 {
151 	const struct got_error *err = NULL;
152 	struct got_object *obj = NULL;
153 	struct got_tree_object *tree = NULL;
154 	uint8_t *buf;
155 	size_t len;
156 
157 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
158 	    GOT_OBJ_TYPE_TREE);
159 	if (err)
160 		return err;
161 
162 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
163 	if (err)
164 		return err;
165 
166 	obj->size = len;
167 	err = got_object_parse_tree(&tree, buf, len);
168 	free(buf);
169 
170 	err = got_privsep_send_tree(ibuf, tree);
171 	if (obj)
172 		got_object_close(obj);
173 	got_object_tree_close(tree);
174 	if (err) {
175 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
176 			err = NULL;
177 		else
178 			got_privsep_send_error(ibuf, err);
179 	}
180 
181 	return err;
182 }
183 
184 static const struct got_error *
185 receive_file(FILE **f, struct imsgbuf *ibuf, int imsg_code)
186 {
187 	const struct got_error *err;
188 	struct imsg imsg;
189 	size_t datalen;
190 
191 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
192 	if (err)
193 		return err;
194 
195 	if (imsg.hdr.type != imsg_code) {
196 		err = got_error(GOT_ERR_PRIVSEP_MSG);
197 		goto done;
198 	}
199 
200 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
201 	if (datalen != 0) {
202 		err = got_error(GOT_ERR_PRIVSEP_LEN);
203 		goto done;
204 	}
205 	if (imsg.fd == -1) {
206 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
207 		goto done;
208 	}
209 
210 	*f = fdopen(imsg.fd, "w+");
211 	if (*f == NULL) {
212 		close(imsg.fd);
213 		err = got_error_from_errno();
214 		goto done;
215 	}
216 done:
217 	imsg_free(&imsg);
218 	return err;
219 }
220 
221 static const struct got_error *
222 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
223     struct got_packidx *packidx, struct got_object_cache *objcache)
224 {
225 	const struct got_error *err = NULL;
226 	struct got_object *obj = NULL;
227 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
228 
229 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
230 	    GOT_OBJ_TYPE_BLOB);
231 	if (err)
232 		return err;
233 
234 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
235 	if (err)
236 		return err;
237 	err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD);
238 	if (err)
239 		return err;
240 	err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD);
241 	if (err)
242 		return err;
243 
244 	err = got_packfile_extract_object(pack, obj, outfile, basefile,
245 	    accumfile);
246 	if (err)
247 		goto done;
248 
249 	err = got_privsep_send_blob(ibuf, obj->size);
250 done:
251 	if (outfile)
252 		fclose(outfile);
253 	if (basefile)
254 		fclose(basefile);
255 	if (accumfile)
256 		fclose(accumfile);
257 	if (obj)
258 		got_object_close(obj);
259 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
260 		got_privsep_send_error(ibuf, err);
261 
262 	return err;
263 }
264 
265 static const struct got_error *
266 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
267 {
268 	const struct got_error *err = NULL;
269 	struct imsg imsg;
270 	struct got_imsg_packidx ipackidx;
271 	size_t datalen;
272 	struct got_packidx *p;
273 
274 	*packidx = NULL;
275 
276 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
277 	if (err)
278 		return err;
279 
280 	p = calloc(1, sizeof(*p));
281 	if (p == NULL) {
282 		err = got_error_from_errno();
283 		goto done;
284 	}
285 
286 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
287 		err = got_error(GOT_ERR_PRIVSEP_MSG);
288 		goto done;
289 	}
290 
291 	if (imsg.fd == -1) {
292 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
293 		goto done;
294 	}
295 
296 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
297 	if (datalen != sizeof(ipackidx)) {
298 		err = got_error(GOT_ERR_PRIVSEP_LEN);
299 		goto done;
300 	}
301 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
302 
303 	p->len = ipackidx.len;
304 	p->fd = dup(imsg.fd);
305 	if (p->fd == -1) {
306 		err = got_error_from_errno();
307 		goto done;
308 	}
309 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
310 		err = got_error_from_errno();
311 		goto done;
312 	}
313 
314 #ifndef GOT_PACK_NO_MMAP
315 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
316 	if (p->map == MAP_FAILED)
317 		p->map = NULL; /* fall back to read(2) */
318 #endif
319 	err = got_packidx_init_hdr(p, 1);
320 done:
321 	if (err) {
322 		if (imsg.fd != -1)
323 			close(imsg.fd);
324 		got_packidx_close(p);
325 	} else
326 		*packidx = p;
327 	imsg_free(&imsg);
328 	return err;
329 }
330 
331 static const struct got_error *
332 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
333 {
334 	const struct got_error *err = NULL;
335 	struct imsg imsg;
336 	struct got_imsg_pack ipack;
337 	size_t datalen;
338 	struct got_pack *pack;
339 
340 	*packp = NULL;
341 
342 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
343 	if (err)
344 		return err;
345 
346 	pack = calloc(1, sizeof(*pack));
347 	if (pack == NULL) {
348 		err = got_error_from_errno();
349 		goto done;
350 	}
351 
352 	if (imsg.hdr.type != GOT_IMSG_PACK) {
353 		err = got_error(GOT_ERR_PRIVSEP_MSG);
354 		goto done;
355 	}
356 
357 	if (imsg.fd == -1) {
358 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
359 		goto done;
360 	}
361 
362 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
363 	if (datalen != sizeof(ipack)) {
364 		err = got_error(GOT_ERR_PRIVSEP_LEN);
365 		goto done;
366 	}
367 	memcpy(&ipack, imsg.data, sizeof(ipack));
368 
369 	pack->filesize = ipack.filesize;
370 	pack->fd = dup(imsg.fd);
371 	if (pack->fd == -1) {
372 		err = got_error_from_errno();
373 		goto done;
374 	}
375 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
376 		err = got_error_from_errno();
377 		goto done;
378 	}
379 	pack->path_packfile = strdup(ipack.path_packfile);
380 	if (pack->path_packfile == NULL) {
381 		err = got_error_from_errno();
382 		goto done;
383 	}
384 
385 #ifndef GOT_PACK_NO_MMAP
386 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
387 	    pack->fd, 0);
388 	if (pack->map == MAP_FAILED)
389 		pack->map = NULL; /* fall back to read(2) */
390 #endif
391 done:
392 	if (err) {
393 		if (imsg.fd != -1)
394 			close(imsg.fd);
395 		free(pack);
396 	} else
397 		*packp = pack;
398 	imsg_free(&imsg);
399 	return err;
400 }
401 
402 int
403 main(int argc, char *argv[])
404 {
405 	const struct got_error *err = NULL;
406 	struct imsgbuf ibuf;
407 	struct imsg imsg;
408 	struct got_packidx *packidx = NULL;
409 	struct got_pack *pack = NULL;
410 	struct got_object_cache objcache;
411 
412 	//static int attached;
413 	//while (!attached) sleep(1);
414 
415 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
416 
417 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
418 	if (err) {
419 		err = got_error_from_errno();
420 		got_privsep_send_error(&ibuf, err);
421 		return 1;
422 	}
423 
424 #ifndef PROFILE
425 	/* revoke access to most system calls */
426 	if (pledge("stdio recvfd", NULL) == -1) {
427 		err = got_error_from_errno();
428 		got_privsep_send_error(&ibuf, err);
429 		return 1;
430 	}
431 #endif
432 
433 	err = receive_packidx(&packidx, &ibuf);
434 	if (err) {
435 		got_privsep_send_error(&ibuf, err);
436 		return 1;
437 	}
438 
439 	err = receive_pack(&pack, &ibuf);
440 	if (err) {
441 		got_privsep_send_error(&ibuf, err);
442 		return 1;
443 	}
444 
445 	while (1) {
446 		imsg.fd = -1;
447 
448 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
449 		if (err) {
450 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
451 				err = NULL;
452 			break;
453 		}
454 
455 		if (imsg.hdr.type == GOT_IMSG_STOP)
456 			break;
457 
458 		switch (imsg.hdr.type) {
459 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
460 			err = object_request(&imsg, &ibuf, pack, packidx,
461 			    &objcache);
462 			break;
463 		case GOT_IMSG_COMMIT_REQUEST:
464 			err = commit_request(&imsg, &ibuf, pack, packidx,
465 			    &objcache);
466 			break;
467 		case GOT_IMSG_TREE_REQUEST:
468 			err = tree_request(&imsg, &ibuf, pack, packidx,
469 			   &objcache);
470 			break;
471 		case GOT_IMSG_BLOB_REQUEST:
472 			err = blob_request(&imsg, &ibuf, pack, packidx,
473 			   &objcache);
474 			break;
475 		default:
476 			err = got_error(GOT_ERR_PRIVSEP_MSG);
477 			break;
478 		}
479 
480 		if (imsg.fd != -1)
481 			close(imsg.fd);
482 		imsg_free(&imsg);
483 		if (err) {
484 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
485 				err = NULL;
486 			else
487 				got_privsep_send_error(&ibuf, err);
488 			break;
489 		}
490 	}
491 
492 	if (packidx)
493 		got_packidx_close(packidx);
494 	if (pack)
495 		got_pack_close(pack);
496 	got_object_cache_close(&objcache);
497 	imsg_clear(&ibuf);
498 	if (err)
499 		fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
500 	close(GOT_IMSG_FD_CHILD);
501 	return err ? 1 : 0;
502 }