Gotweb

Blob

Date:: Tue Sep 11 10:29:46 2018 UTC
Message:: stop sending deltas in imsg; cache deltas in got-read-pack
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/limits.h>
22 #include <sys/syslimits.h>
23 #include <sys/mman.h>
24 
25 #include <limits.h>
26 #include <stdint.h>
27 #include <imsg.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <sha1.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 
37 #include "got_lib_delta.h"
38 #include "got_lib_inflate.h"
39 #include "got_lib_object.h"
40 #include "got_lib_object_cache.h"
41 #include "got_lib_object_parse.h"
42 #include "got_lib_privsep.h"
43 #include "got_lib_pack.h"
44 
45 static const struct got_error *
46 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
47     struct got_packidx *packidx, struct got_object_cache *objcache)
48 {
49 	const struct got_error *err = NULL;
50 	struct got_imsg_packed_object iobj;
51 	struct got_object *obj;
52 	size_t datalen;
53 
54 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
55 	if (datalen != sizeof(iobj))
56 		return got_error(GOT_ERR_PRIVSEP_LEN);
57 	memcpy(&iobj, imsg->data, sizeof(iobj));
58 
59 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, NULL);
60 	if (err)
61 		return err;
62 	obj->refcnt++;
63 
64 	err = got_object_cache_add(objcache, &obj->id, obj);
65 	if (err)
66 		goto done;
67 	obj->refcnt++;
68 
69 	err = got_privsep_send_obj(ibuf, obj);
70 done:
71 	got_object_close(obj);
72 	return err;
73 }
74 
75 static const struct got_error *
76 get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf,
77     struct got_pack *pack, struct got_packidx *packidx,
78     struct got_object_cache *objcache, int type)
79 {
80 	const struct got_error *err = NULL;
81 	struct got_object *iobj;
82 
83 	err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf);
84 	if (err)
85 		return err;
86 
87 	if (iobj->type != type) {
88 		err = got_error(GOT_ERR_OBJ_TYPE);
89 		goto done;
90 	}
91 
92 	if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0)
93 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
94 
95 	*obj = got_object_cache_get(objcache, &iobj->id);
96 	if (*obj == NULL) {
97 		err = got_packfile_open_object(obj, pack, packidx,
98 		    iobj->pack_idx, &iobj->id);
99 		if (err)
100 			goto done;
101 	}
102 	(*obj)->refcnt++;
103 done:
104 	got_object_close(iobj);
105 	return err;
106 }
107 
108 static const struct got_error *
109 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
110     struct got_packidx *packidx, struct got_object_cache *objcache)
111 {
112 	const struct got_error *err = NULL;
113 	struct got_object *obj = NULL;
114 	struct got_commit_object *commit = NULL;
115 	uint8_t *buf;
116 	size_t len;
117 
118 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
119 	    GOT_OBJ_TYPE_COMMIT);
120 	if (err)
121 		return err;
122 
123 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
124 	if (err)
125 		return err;
126 
127 	obj->size = len;
128 	err = got_object_parse_commit(&commit, buf, len);
129 	free(buf);
130 
131 	err = got_privsep_send_commit(ibuf, commit);
132 	if (obj)
133 		got_object_close(obj);
134 	got_object_commit_close(commit);
135 	if (err) {
136 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
137 			err = NULL;
138 		else
139 			got_privsep_send_error(ibuf, err);
140 	}
141 
142 	return err;
143 }
144 
145 static const struct got_error *
146 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
147     struct got_packidx *packidx, struct got_object_cache *objcache)
148 {
149 	const struct got_error *err = NULL;
150 	struct got_object *obj = NULL;
151 	struct got_tree_object *tree = NULL;
152 	uint8_t *buf;
153 	size_t len;
154 
155 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
156 	    GOT_OBJ_TYPE_TREE);
157 	if (err)
158 		return err;
159 
160 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
161 	if (err)
162 		return err;
163 
164 	obj->size = len;
165 	err = got_object_parse_tree(&tree, buf, len);
166 	free(buf);
167 
168 	err = got_privsep_send_tree(ibuf, tree);
169 	if (obj)
170 		got_object_close(obj);
171 	got_object_tree_close(tree);
172 	if (err) {
173 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
174 			err = NULL;
175 		else
176 			got_privsep_send_error(ibuf, err);
177 	}
178 
179 	return err;
180 }
181 
182 static const struct got_error *
183 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
184     struct got_packidx *packidx, struct got_object_cache *objcache)
185 {
186 	const struct got_error *err = NULL;
187 	struct got_object *obj = NULL;
188 	FILE *f = NULL;
189 	struct imsg imsg_outfd;
190 	size_t datalen;
191 
192 	memset(&imsg_outfd, 0, sizeof(imsg_outfd));
193 	imsg_outfd.fd = -1;
194 
195 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
196 	    GOT_OBJ_TYPE_BLOB);
197 	if (err)
198 		return err;
199 
200 	err = got_privsep_recv_imsg(&imsg_outfd, ibuf, 0);
201 	if (err)
202 		return err;
203 
204 	if (imsg_outfd.hdr.type != GOT_IMSG_BLOB_OUTFD) {
205 		err = got_error(GOT_ERR_PRIVSEP_MSG);
206 		goto done;
207 	}
208 
209 	datalen = imsg_outfd.hdr.len - IMSG_HEADER_SIZE;
210 	if (datalen != 0) {
211 		err = got_error(GOT_ERR_PRIVSEP_LEN);
212 		goto done;
213 	}
214 	if (imsg_outfd.fd == -1) {
215 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
216 		goto done;
217 	}
218 
219 	f = fdopen(imsg_outfd.fd, "w+");
220 	if (f == NULL) {
221 		err = got_error_from_errno();
222 		goto done;
223 	}
224 
225 	err = got_packfile_extract_object(pack, obj, f);
226 	if (err)
227 		goto done;
228 
229 	err = got_privsep_send_blob(ibuf, obj->size);
230 done:
231 	if (f)
232 		fclose(f);
233 	else if (imsg_outfd.fd != -1)
234 		close(imsg_outfd.fd);
235 	imsg_free(&imsg_outfd);
236 	if (obj)
237 		got_object_close(obj);
238 	if (err) {
239 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
240 			err = NULL;
241 		else
242 			got_privsep_send_error(ibuf, err);
243 	}
244 
245 	return err;
246 }
247 
248 static const struct got_error *
249 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
250 {
251 	const struct got_error *err = NULL;
252 	struct imsg imsg;
253 	struct got_imsg_packidx ipackidx;
254 	size_t datalen;
255 	struct got_packidx *p;
256 
257 	*packidx = NULL;
258 
259 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
260 	if (err)
261 		return err;
262 
263 	p = calloc(1, sizeof(*p));
264 	if (p == NULL) {
265 		err = got_error_from_errno();
266 		goto done;
267 	}
268 
269 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
270 		err = got_error(GOT_ERR_PRIVSEP_MSG);
271 		goto done;
272 	}
273 
274 	if (imsg.fd == -1) {
275 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
276 		goto done;
277 	}
278 
279 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
280 	if (datalen != sizeof(ipackidx)) {
281 		err = got_error(GOT_ERR_PRIVSEP_LEN);
282 		goto done;
283 	}
284 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
285 
286 	p->len = ipackidx.len;
287 	p->fd = dup(imsg.fd);
288 	if (p->fd == -1) {
289 		err = got_error_from_errno();
290 		goto done;
291 	}
292 
293 #ifndef GOT_PACK_NO_MMAP
294 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
295 	if (p->map == MAP_FAILED)
296 		p->map = NULL; /* fall back to read(2) */
297 #endif
298 	err = got_packidx_init_hdr(p, 1);
299 done:
300 	if (err) {
301 		if (imsg.fd != -1)
302 			close(imsg.fd);
303 		got_packidx_close(p);
304 	} else
305 		*packidx = p;
306 	imsg_free(&imsg);
307 	return err;
308 }
309 
310 static const struct got_error *
311 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
312 {
313 	const struct got_error *err = NULL;
314 	struct imsg imsg;
315 	struct got_imsg_pack ipack;
316 	size_t datalen;
317 	struct got_pack *pack;
318 
319 	*packp = NULL;
320 
321 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
322 	if (err)
323 		return err;
324 
325 	pack = calloc(1, sizeof(*pack));
326 	if (pack == NULL) {
327 		err = got_error_from_errno();
328 		goto done;
329 	}
330 
331 	if (imsg.hdr.type != GOT_IMSG_PACK) {
332 		err = got_error(GOT_ERR_PRIVSEP_MSG);
333 		goto done;
334 	}
335 
336 	if (imsg.fd == -1) {
337 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
338 		goto done;
339 	}
340 
341 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
342 	if (datalen != sizeof(ipack)) {
343 		err = got_error(GOT_ERR_PRIVSEP_LEN);
344 		goto done;
345 	}
346 	memcpy(&ipack, imsg.data, sizeof(ipack));
347 
348 	pack->filesize = ipack.filesize;
349 	pack->fd = dup(imsg.fd);
350 	if (pack->fd == -1) {
351 		err = got_error_from_errno();
352 		goto done;
353 	}
354 	pack->path_packfile = strdup(ipack.path_packfile);
355 	if (pack->path_packfile == NULL) {
356 		err = got_error_from_errno();
357 		goto done;
358 	}
359 
360 #ifndef GOT_PACK_NO_MMAP
361 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
362 	    pack->fd, 0);
363 	if (pack->map == MAP_FAILED)
364 		pack->map = NULL; /* fall back to read(2) */
365 #endif
366 done:
367 	if (err) {
368 		if (imsg.fd != -1)
369 			close(imsg.fd);
370 		free(pack);
371 	} else
372 		*packp = pack;
373 	imsg_free(&imsg);
374 	return err;
375 }
376 
377 int
378 main(int argc, char *argv[])
379 {
380 	const struct got_error *err = NULL;
381 	struct imsgbuf ibuf;
382 	struct imsg imsg;
383 	struct got_packidx *packidx = NULL;
384 	struct got_pack *pack = NULL;
385 	struct got_object_cache objcache;
386 
387 	//static int attached;
388 	//while (!attached) sleep(1);
389 
390 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
391 
392 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
393 	if (err) {
394 		err = got_error_from_errno();
395 		got_privsep_send_error(&ibuf, err);
396 		return 1;
397 	}
398 
399 	/* revoke access to most system calls */
400 	if (pledge("stdio recvfd", NULL) == -1) {
401 		err = got_error_from_errno();
402 		got_privsep_send_error(&ibuf, err);
403 		return 1;
404 	}
405 
406 	err = receive_packidx(&packidx, &ibuf);
407 	if (err) {
408 		got_privsep_send_error(&ibuf, err);
409 		return 1;
410 	}
411 
412 	err = receive_pack(&pack, &ibuf);
413 	if (err) {
414 		got_privsep_send_error(&ibuf, err);
415 		return 1;
416 	}
417 
418 	while (1) {
419 		imsg.fd = -1;
420 
421 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
422 		if (err) {
423 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
424 				err = NULL;
425 			break;
426 		}
427 
428 		if (imsg.hdr.type == GOT_IMSG_STOP)
429 			break;
430 
431 		switch (imsg.hdr.type) {
432 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
433 			err = object_request(&imsg, &ibuf, pack, packidx,
434 			    &objcache);
435 			break;
436 		case GOT_IMSG_COMMIT_REQUEST:
437 			err = commit_request(&imsg, &ibuf, pack, packidx,
438 			    &objcache);
439 			break;
440 		case GOT_IMSG_TREE_REQUEST:
441 			err = tree_request(&imsg, &ibuf, pack, packidx,
442 			   &objcache);
443 			break;
444 		case GOT_IMSG_BLOB_REQUEST:
445 			err = blob_request(&imsg, &ibuf, pack, packidx,
446 			   &objcache);
447 			break;
448 		default:
449 			err = got_error(GOT_ERR_PRIVSEP_MSG);
450 			break;
451 		}
452 
453 		if (imsg.fd != -1)
454 			close(imsg.fd);
455 		imsg_free(&imsg);
456 		if (err) {
457 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
458 				err = NULL;
459 			else
460 				got_privsep_send_error(&ibuf, err);
461 			break;
462 		}
463 	}
464 
465 	if (packidx)
466 		got_packidx_close(packidx);
467 	if (pack)
468 		got_pack_close(pack);
469 	imsg_clear(&ibuf);
470 	if (err)
471 		fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
472 	close(GOT_IMSG_FD_CHILD);
473 	return err ? 1 : 0;
474 }