Gotweb

Blob

Date:: Sun Apr 1 13:57:43 2018 UTC
Message:: look up deltas in the correct pack file
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22 #include <zlib.h>
23 #include <sha1.h>
24 
25 #include "got_error.h"
26 #include "got_repository.h"
27 #include "got_object.h"
28 
29 #include "got_lib_delta.h"
30 #include "got_lib_path.h"
31 #include "got_lib_zbuf.h"
32 
33 #ifndef MIN
34 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
35 #endif
36 
37 struct got_delta *
38 got_delta_open(const char *path_packfile, off_t offset, size_t tslen,
39     int type, size_t size, off_t data_offset, uint8_t *delta_buf,
40     size_t delta_len)
41 {
42 	struct got_delta *delta;
43 
44 	delta = calloc(1, sizeof(*delta));
45 	if (delta == NULL)
46 		return NULL;
47 
48 	delta->path_packfile = strdup(path_packfile);
49 	if (delta->path_packfile == NULL) {
50 		free(delta);
51 		return NULL;
52 	}
53 	delta->type = type;
54 	delta->offset = offset;
55 	delta->tslen = tslen;
56 	delta->size = size;
57 	delta->data_offset = data_offset;
58 	delta->delta_buf = delta_buf;
59 	delta->delta_len = delta_len;
60 	return delta;
61 }
62 
63 void
64 got_delta_close(struct got_delta *delta)
65 {
66 	free(delta->path_packfile);
67 	free(delta->delta_buf);
68 	free(delta);
69 }
70 
71 const struct got_error *
72 got_delta_chain_get_base_type(int *type, struct got_delta_chain *deltas)
73 {
74 	struct got_delta *delta;
75 
76 	/* The first delta in the chain should represent the base object. */
77 	delta = SIMPLEQ_FIRST(&deltas->entries);
78 	if (delta->type == GOT_OBJ_TYPE_COMMIT ||
79 	    delta->type == GOT_OBJ_TYPE_TREE ||
80 	    delta->type == GOT_OBJ_TYPE_BLOB ||
81 	    delta->type == GOT_OBJ_TYPE_TAG) {
82 		*type = delta->type;
83 		return NULL;
84 	}
85 
86 	return got_error(GOT_ERR_BAD_DELTA_CHAIN);
87 }
88 
89 /* Fetch another (required) byte from the delta stream. */
90 static const struct got_error *
91 next_delta_byte(const uint8_t **p, size_t *remain)
92 {
93 	if (--(*remain) == 0)
94 		return got_error(GOT_ERR_BAD_DELTA);
95 	(*p)++;
96 	return NULL;
97 }
98 
99 static const struct got_error *
100 parse_size(uint64_t *size, const uint8_t **p, size_t *remain)
101 {
102 	const struct got_error *err = NULL;
103 	int i = 0;
104 
105 	*size = 0;
106 	do {
107 		/* We do not support size values which don't fit in 64 bit. */
108 		if (i > 9)
109 			return got_error(GOT_ERR_NO_SPACE);
110 
111 		if (i == 0)
112 			*size = ((**p) & GOT_DELTA_SIZE_VAL_MASK);
113 		else {
114 			size_t shift = GOT_DELTA_SIZE_SHIFT * i;
115 			*size |= (((**p) & GOT_DELTA_SIZE_VAL_MASK) << shift);
116 		}
117 
118 		if (((**p) & GOT_DELTA_SIZE_MORE) == 0)
119 			break;
120 		i++;
121 		err = next_delta_byte(p, remain);
122 	} while (err == NULL);
123 
124 	return err;
125 }
126 
127 static const struct got_error *
128 parse_opcode(off_t *offset, size_t *len, const uint8_t **p, size_t *remain)
129 {
130 	const struct got_error *err = NULL;
131 	off_t o = 0;
132 	size_t l = 0;
133 	uint8_t opcode = **p;
134 
135 	if (opcode & GOT_DELTA_COPY_OFF1) {
136 		err = next_delta_byte(p, remain);
137 		if (err)
138 			return err;
139 		o = (off_t)(**p);
140 	}
141 	if (opcode & GOT_DELTA_COPY_OFF2) {
142 		err = next_delta_byte(p, remain);
143 		if (err)
144 			return err;
145 		o |= ((off_t)(**p)) << 8;
146 	}
147 	if (opcode & GOT_DELTA_COPY_OFF3) {
148 		err = next_delta_byte(p, remain);
149 		if (err)
150 			return err;
151 		o |= ((off_t)(**p)) << 16;
152 	}
153 	if (opcode & GOT_DELTA_COPY_OFF4) {
154 		err = next_delta_byte(p, remain);
155 		if (err)
156 			return err;
157 		o |= ((off_t)(**p)) << 24;
158 	}
159 
160 	if (opcode & GOT_DELTA_COPY_LEN1) {
161 		err = next_delta_byte(p, remain);
162 		if (err)
163 			return err;
164 		l = (off_t)(**p);
165 	}
166 	if (opcode & GOT_DELTA_COPY_LEN2) {
167 		err = next_delta_byte(p, remain);
168 		if (err)
169 			return err;
170 		l |= ((off_t)(**p)) << 8;
171 	}
172 	if (opcode & GOT_DELTA_COPY_LEN3) {
173 		err = next_delta_byte(p, remain);
174 		if (err)
175 			return err;
176 		l |= ((off_t)(**p)) << 16;
177 	}
178 
179 	if (o == 0)
180 		o = GOT_DELTA_COPY_DEFAULT_OFF;
181 	if (l == 0)
182 		l = GOT_DELTA_COPY_DEFAULT_LEN;
183 
184 	*offset = o;
185 	*len = l;
186 	return NULL;
187 }
188 
189 static const struct got_error *
190 copy_from_base(FILE *base_file, off_t offset, size_t size, FILE *outfile)
191 {
192 	if (fseeko(base_file, offset, SEEK_SET) != 0)
193 		return got_error_from_errno();
194 
195 	while (size > 0) {
196 		uint8_t data[2048];
197 		size_t len = MIN(size, sizeof(data));
198 		size_t n;
199 
200 		n = fread(data, len, 1, base_file);
201 		if (n != 1)
202 			return got_ferror(base_file, GOT_ERR_IO);
203 
204 		n = fwrite(data, len, 1, outfile);
205 		if (n != 1)
206 			return got_ferror(outfile, GOT_ERR_IO);
207 
208 		size -= len;
209 	}
210 
211 	return NULL;
212 }
213 
214 static const struct got_error *
215 copy_from_delta(const uint8_t **p, size_t *remain, size_t len, FILE *outfile)
216 {
217 	size_t n;
218 
219 	if (*remain < len)
220 		return got_error(GOT_ERR_BAD_DELTA);
221 
222 	n = fwrite(*p, len, 1, outfile);
223 	if (n != 1)
224 		return got_ferror(outfile, GOT_ERR_IO);
225 
226 	*p += len;
227 	*remain -= len;
228 	return NULL;
229 }
230 
231 static const struct got_error *
232 parse_delta_sizes(uint64_t *base_size, uint64_t *result_size,
233     const uint8_t **p, size_t *remain)
234 {
235 	const struct got_error *err;
236 
237 	/* Read the two size fields at the beginning of the stream. */
238 	err = parse_size(base_size, p, remain);
239 	if (err)
240 		return err;
241 	err = next_delta_byte(p, remain);
242 	if (err)
243 		return err;
244 	err = parse_size(result_size, p, remain);
245 	if (err)
246 		return err;
247 
248 	return NULL;
249 }
250 
251 const struct got_error *
252 got_delta_get_sizes(uint64_t *base_size, uint64_t *result_size,
253     const uint8_t *delta_buf, size_t delta_len)
254 {
255 	size_t remain;
256 	const uint8_t *p;
257 
258 	if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
259 		return got_error(GOT_ERR_BAD_DELTA);
260 
261 	p = delta_buf;
262 	remain = delta_len;
263 	return parse_delta_sizes(base_size, result_size, &p, &remain);
264 }
265 
266 const struct got_error *
267 got_delta_apply_in_mem(uint8_t *base_buf, const uint8_t *delta_buf,
268     size_t delta_len, uint8_t *outbuf, size_t *outsize)
269 {
270 	const struct got_error *err = NULL;
271 	uint64_t base_size, result_size;
272 	size_t remain;
273 	const uint8_t *p;
274 
275 	*outsize= 0;
276 
277 	if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
278 		return got_error(GOT_ERR_BAD_DELTA);
279 
280 	p = delta_buf;
281 	remain = delta_len;
282 	err = parse_delta_sizes(&base_size, &result_size, &p, &remain);
283 	if (err)
284 		return err;
285 
286 	/* Decode and execute copy instructions from the delta stream. */
287 	err = next_delta_byte(&p, &remain);
288 	while (err == NULL && remain > 0) {
289 		if (*p & GOT_DELTA_BASE_COPY) {
290 			off_t offset = 0;
291 			size_t len = 0;
292 			err = parse_opcode(&offset, &len, &p, &remain);
293 			if (err)
294 				break;
295 			memcpy(outbuf + *outsize, base_buf + offset, len);
296 			if (err == NULL) {
297 				*outsize += len;
298 				if (remain > 0) {
299 					p++;
300 					remain--;
301 				}
302 			}
303 		} else {
304 			size_t len = (size_t)*p;
305 			if (len == 0) {
306 				err = got_error(GOT_ERR_BAD_DELTA);
307 				break;
308 			}
309 			err = next_delta_byte(&p, &remain);
310 			if (err)
311 				break;
312 			if (remain < len)
313 				return got_error(GOT_ERR_BAD_DELTA);
314 			memcpy(outbuf + *outsize, p, len);
315 			p += len;
316 			remain -= len;
317 			*outsize += len;
318 		}
319 	}
320 
321 	if (*outsize != result_size)
322 		err = got_error(GOT_ERR_BAD_DELTA);
323 	return err;
324 }
325 
326 const struct got_error *
327 got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
328     size_t delta_len, FILE *outfile, size_t *outsize)
329 {
330 	const struct got_error *err = NULL;
331 	uint64_t base_size, result_size;
332 	size_t remain = 0;
333 	const uint8_t *p;
334 	FILE *memstream = NULL;
335 	char *memstream_buf = NULL;
336 	size_t memstream_size = 0;
337 
338 	*outsize = 0;
339 
340 	if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
341 		return got_error(GOT_ERR_BAD_DELTA);
342 
343 	p = delta_buf;
344 	remain = delta_len;
345 	err = parse_delta_sizes(&base_size, &result_size, &p, &remain);
346 	if (err)
347 		return err;
348 
349 	if (result_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
350 		memstream = open_memstream(&memstream_buf, &memstream_size);
351 
352 	/* Decode and execute copy instructions from the delta stream. */
353 	err = next_delta_byte(&p, &remain);
354 	while (err == NULL && remain > 0) {
355 		if (*p & GOT_DELTA_BASE_COPY) {
356 			off_t offset = 0;
357 			size_t len = 0;
358 			err = parse_opcode(&offset, &len, &p, &remain);
359 			if (err)
360 				break;
361 			err = copy_from_base(base_file, offset, len,
362 			    memstream ? memstream : outfile);
363 			if (err == NULL) {
364 				*outsize += len;
365 				if (remain > 0) {
366 					p++;
367 					remain--;
368 				}
369 			}
370 		} else {
371 			size_t len = (size_t)*p;
372 			if (len == 0) {
373 				err = got_error(GOT_ERR_BAD_DELTA);
374 				break;
375 			}
376 			err = next_delta_byte(&p, &remain);
377 			if (err)
378 				break;
379 			err = copy_from_delta(&p, &remain, len,
380 			    memstream ? memstream : outfile);
381 			if (err == NULL)
382 				*outsize += len;
383 		}
384 	}
385 
386 	if (*outsize != result_size)
387 		err = got_error(GOT_ERR_BAD_DELTA);
388 
389 	if (memstream != NULL) {
390 		fclose(memstream);
391 		if (err == NULL) {
392 			size_t n;
393 			n = fwrite(memstream_buf, 1, memstream_size, outfile);
394 			if (n != memstream_size)
395 				err = got_ferror(outfile, GOT_ERR_IO);
396 		}
397 		free(memstream_buf);
398 	}
399 	if (err == NULL)
400 		rewind(outfile);
401 	return err;
402 }