2 efd2a263 2018-01-19 stsp * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
4 efd2a263 2018-01-19 stsp * Permission to use, copy, modify, and distribute this software for any
5 efd2a263 2018-01-19 stsp * purpose with or without fee is hereby granted, provided that the above
6 efd2a263 2018-01-19 stsp * copyright notice and this permission notice appear in all copies.
8 efd2a263 2018-01-19 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 efd2a263 2018-01-19 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 efd2a263 2018-01-19 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 efd2a263 2018-01-19 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 efd2a263 2018-01-19 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 efd2a263 2018-01-19 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 efd2a263 2018-01-19 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 8b925c6c 2022-07-16 thomas #include <sys/queue.h>
19 efd2a263 2018-01-19 stsp #include <stdio.h>
20 96f5e8b3 2018-01-23 stsp #include <stdlib.h>
21 b2f7af54 2018-11-11 stsp #include <stdint.h>
22 96f5e8b3 2018-01-23 stsp #include <string.h>
23 56e0773d 2019-11-28 stsp #include <limits.h>
24 efd2a263 2018-01-19 stsp #include <zlib.h>
25 788c352e 2018-06-16 stsp #include <time.h>
26 15a94983 2018-12-23 stsp #include <zlib.h>
28 efd2a263 2018-01-19 stsp #include "got_error.h"
29 efd2a263 2018-01-19 stsp #include "got_repository.h"
30 efd2a263 2018-01-19 stsp #include "got_object.h"
31 324d37e7 2019-05-11 stsp #include "got_path.h"
33 718b3ab0 2018-03-17 stsp #include "got_lib_delta.h"
34 63581804 2018-07-09 stsp #include "got_lib_inflate.h"
35 15a94983 2018-12-23 stsp #include "got_lib_object.h"
38 885d3e02 2018-01-27 stsp #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
41 c3703302 2018-01-23 stsp struct got_delta *
42 c336f889 2018-07-23 stsp got_delta_open(off_t offset, size_t tslen, int type, size_t size,
43 42c69117 2019-11-10 stsp off_t data_offset)
45 c3703302 2018-01-23 stsp struct got_delta *delta;
47 14118581 2018-11-05 stsp delta = malloc(sizeof(*delta));
48 c3703302 2018-01-23 stsp if (delta == NULL)
49 96f5e8b3 2018-01-23 stsp return NULL;
51 c3703302 2018-01-23 stsp delta->type = type;
52 c3703302 2018-01-23 stsp delta->offset = offset;
53 0e22967e 2018-02-11 stsp delta->tslen = tslen;
54 c3703302 2018-01-23 stsp delta->size = size;
55 bdd2fbb3 2018-02-11 stsp delta->data_offset = data_offset;
56 c3703302 2018-01-23 stsp return delta;
59 efd2a263 2018-01-19 stsp const struct got_error *
60 96f5e8b3 2018-01-23 stsp got_delta_chain_get_base_type(int *type, struct got_delta_chain *deltas)
62 c3703302 2018-01-23 stsp struct got_delta *delta;
64 6691714a 2018-01-23 stsp /* The first delta in the chain should represent the base object. */
65 dbdddfee 2021-06-23 naddy delta = STAILQ_FIRST(&deltas->entries);
66 6691714a 2018-01-23 stsp if (delta->type == GOT_OBJ_TYPE_COMMIT ||
67 6691714a 2018-01-23 stsp delta->type == GOT_OBJ_TYPE_TREE ||
68 6691714a 2018-01-23 stsp delta->type == GOT_OBJ_TYPE_BLOB ||
69 6691714a 2018-01-23 stsp delta->type == GOT_OBJ_TYPE_TAG) {
70 6691714a 2018-01-23 stsp *type = delta->type;
71 6691714a 2018-01-23 stsp return NULL;
74 96f5e8b3 2018-01-23 stsp return got_error(GOT_ERR_BAD_DELTA_CHAIN);
77 885d3e02 2018-01-27 stsp /* Fetch another (required) byte from the delta stream. */
78 885d3e02 2018-01-27 stsp static const struct got_error *
79 885d3e02 2018-01-27 stsp next_delta_byte(const uint8_t **p, size_t *remain)
81 885d3e02 2018-01-27 stsp if (--(*remain) == 0)
82 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA,
83 9069347b 2021-05-20 stsp "delta data truncated");
85 885d3e02 2018-01-27 stsp return NULL;
88 885d3e02 2018-01-27 stsp static const struct got_error *
89 885d3e02 2018-01-27 stsp parse_size(uint64_t *size, const uint8_t **p, size_t *remain)
91 885d3e02 2018-01-27 stsp const struct got_error *err = NULL;
96 885d3e02 2018-01-27 stsp /* We do not support size values which don't fit in 64 bit. */
98 885d3e02 2018-01-27 stsp return got_error(GOT_ERR_NO_SPACE);
100 885d3e02 2018-01-27 stsp if (i == 0)
101 885d3e02 2018-01-27 stsp *size = ((**p) & GOT_DELTA_SIZE_VAL_MASK);
103 885d3e02 2018-01-27 stsp size_t shift = GOT_DELTA_SIZE_SHIFT * i;
104 885d3e02 2018-01-27 stsp *size |= (((**p) & GOT_DELTA_SIZE_VAL_MASK) << shift);
107 885d3e02 2018-01-27 stsp if (((**p) & GOT_DELTA_SIZE_MORE) == 0)
110 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
111 885d3e02 2018-01-27 stsp } while (err == NULL);
113 885d3e02 2018-01-27 stsp return err;
116 885d3e02 2018-01-27 stsp static const struct got_error *
117 885d3e02 2018-01-27 stsp parse_opcode(off_t *offset, size_t *len, const uint8_t **p, size_t *remain)
119 885d3e02 2018-01-27 stsp const struct got_error *err = NULL;
120 885d3e02 2018-01-27 stsp off_t o = 0;
121 885d3e02 2018-01-27 stsp size_t l = 0;
122 885d3e02 2018-01-27 stsp uint8_t opcode = **p;
124 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_OFF1) {
125 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
127 885d3e02 2018-01-27 stsp return err;
128 885d3e02 2018-01-27 stsp o = (off_t)(**p);
130 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_OFF2) {
131 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
133 885d3e02 2018-01-27 stsp return err;
134 885d3e02 2018-01-27 stsp o |= ((off_t)(**p)) << 8;
136 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_OFF3) {
137 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
139 885d3e02 2018-01-27 stsp return err;
140 885d3e02 2018-01-27 stsp o |= ((off_t)(**p)) << 16;
142 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_OFF4) {
143 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
145 885d3e02 2018-01-27 stsp return err;
146 885d3e02 2018-01-27 stsp o |= ((off_t)(**p)) << 24;
149 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_LEN1) {
150 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
152 885d3e02 2018-01-27 stsp return err;
153 885d3e02 2018-01-27 stsp l = (off_t)(**p);
155 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_LEN2) {
156 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
158 885d3e02 2018-01-27 stsp return err;
159 885d3e02 2018-01-27 stsp l |= ((off_t)(**p)) << 8;
161 885d3e02 2018-01-27 stsp if (opcode & GOT_DELTA_COPY_LEN3) {
162 885d3e02 2018-01-27 stsp err = next_delta_byte(p, remain);
164 885d3e02 2018-01-27 stsp return err;
165 885d3e02 2018-01-27 stsp l |= ((off_t)(**p)) << 16;
168 885d3e02 2018-01-27 stsp if (o == 0)
169 885d3e02 2018-01-27 stsp o = GOT_DELTA_COPY_DEFAULT_OFF;
170 885d3e02 2018-01-27 stsp if (l == 0)
171 885d3e02 2018-01-27 stsp l = GOT_DELTA_COPY_DEFAULT_LEN;
173 885d3e02 2018-01-27 stsp *offset = o;
175 885d3e02 2018-01-27 stsp return NULL;
178 885d3e02 2018-01-27 stsp static const struct got_error *
179 885d3e02 2018-01-27 stsp copy_from_base(FILE *base_file, off_t offset, size_t size, FILE *outfile)
181 885d3e02 2018-01-27 stsp if (fseeko(base_file, offset, SEEK_SET) != 0)
182 638f9024 2019-05-13 stsp return got_error_from_errno("fseeko");
184 885d3e02 2018-01-27 stsp while (size > 0) {
185 885d3e02 2018-01-27 stsp uint8_t data[2048];
186 885d3e02 2018-01-27 stsp size_t len = MIN(size, sizeof(data));
189 885d3e02 2018-01-27 stsp n = fread(data, len, 1, base_file);
190 885d3e02 2018-01-27 stsp if (n != 1)
191 885d3e02 2018-01-27 stsp return got_ferror(base_file, GOT_ERR_IO);
193 885d3e02 2018-01-27 stsp n = fwrite(data, len, 1, outfile);
194 885d3e02 2018-01-27 stsp if (n != 1)
195 885d3e02 2018-01-27 stsp return got_ferror(outfile, GOT_ERR_IO);
197 885d3e02 2018-01-27 stsp size -= len;
200 885d3e02 2018-01-27 stsp return NULL;
203 885d3e02 2018-01-27 stsp static const struct got_error *
204 885d3e02 2018-01-27 stsp copy_from_delta(const uint8_t **p, size_t *remain, size_t len, FILE *outfile)
208 885d3e02 2018-01-27 stsp if (*remain < len)
209 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA,
210 9069347b 2021-05-20 stsp "copy from beyond end of delta data");
212 885d3e02 2018-01-27 stsp n = fwrite(*p, len, 1, outfile);
213 885d3e02 2018-01-27 stsp if (n != 1)
214 885d3e02 2018-01-27 stsp return got_ferror(outfile, GOT_ERR_IO);
217 885d3e02 2018-01-27 stsp *remain -= len;
218 885d3e02 2018-01-27 stsp return NULL;
221 22484865 2018-03-13 stsp static const struct got_error *
222 22484865 2018-03-13 stsp parse_delta_sizes(uint64_t *base_size, uint64_t *result_size,
223 22484865 2018-03-13 stsp const uint8_t **p, size_t *remain)
225 22484865 2018-03-13 stsp const struct got_error *err;
227 22484865 2018-03-13 stsp /* Read the two size fields at the beginning of the stream. */
228 22484865 2018-03-13 stsp err = parse_size(base_size, p, remain);
230 22484865 2018-03-13 stsp return err;
231 22484865 2018-03-13 stsp err = next_delta_byte(p, remain);
233 22484865 2018-03-13 stsp return err;
234 22484865 2018-03-13 stsp err = parse_size(result_size, p, remain);
236 22484865 2018-03-13 stsp return err;
238 22484865 2018-03-13 stsp return NULL;
241 885d3e02 2018-01-27 stsp const struct got_error *
242 22484865 2018-03-13 stsp got_delta_get_sizes(uint64_t *base_size, uint64_t *result_size,
243 22484865 2018-03-13 stsp const uint8_t *delta_buf, size_t delta_len)
245 22484865 2018-03-13 stsp size_t remain;
246 22484865 2018-03-13 stsp const uint8_t *p;
248 22484865 2018-03-13 stsp if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
249 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA, "delta too small");
251 22484865 2018-03-13 stsp p = delta_buf;
252 22484865 2018-03-13 stsp remain = delta_len;
253 22484865 2018-03-13 stsp return parse_delta_sizes(base_size, result_size, &p, &remain);
256 8628c62d 2018-03-15 stsp const struct got_error *
257 34fca9c3 2018-11-11 stsp got_delta_apply_in_mem(uint8_t *base_buf, size_t base_bufsz,
258 34fca9c3 2018-11-11 stsp const uint8_t *delta_buf, size_t delta_len, uint8_t *outbuf,
259 34fca9c3 2018-11-11 stsp size_t *outsize, size_t maxoutsize)
261 8628c62d 2018-03-15 stsp const struct got_error *err = NULL;
262 8628c62d 2018-03-15 stsp uint64_t base_size, result_size;
263 8628c62d 2018-03-15 stsp size_t remain;
264 8628c62d 2018-03-15 stsp const uint8_t *p;
266 8628c62d 2018-03-15 stsp *outsize= 0;
268 8628c62d 2018-03-15 stsp if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
269 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA, "delta too small");
271 8628c62d 2018-03-15 stsp p = delta_buf;
272 8628c62d 2018-03-15 stsp remain = delta_len;
273 8628c62d 2018-03-15 stsp err = parse_delta_sizes(&base_size, &result_size, &p, &remain);
275 8628c62d 2018-03-15 stsp return err;
277 8628c62d 2018-03-15 stsp /* Decode and execute copy instructions from the delta stream. */
278 8628c62d 2018-03-15 stsp err = next_delta_byte(&p, &remain);
279 8628c62d 2018-03-15 stsp while (err == NULL && remain > 0) {
280 8628c62d 2018-03-15 stsp if (*p & GOT_DELTA_BASE_COPY) {
281 8628c62d 2018-03-15 stsp off_t offset = 0;
282 8628c62d 2018-03-15 stsp size_t len = 0;
283 8628c62d 2018-03-15 stsp err = parse_opcode(&offset, &len, &p, &remain);
286 b2f7af54 2018-11-11 stsp if (SIZE_MAX - offset < len || offset + len < 0 ||
287 b2f7af54 2018-11-11 stsp base_bufsz < offset + len ||
288 34fca9c3 2018-11-11 stsp *outsize + len > maxoutsize)
289 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA,
290 9069347b 2021-05-20 stsp "bad delta copy length");
291 8628c62d 2018-03-15 stsp memcpy(outbuf + *outsize, base_buf + offset, len);
292 8628c62d 2018-03-15 stsp if (err == NULL) {
293 8628c62d 2018-03-15 stsp *outsize += len;
294 8628c62d 2018-03-15 stsp if (remain > 0) {
300 8628c62d 2018-03-15 stsp size_t len = (size_t)*p;
301 8628c62d 2018-03-15 stsp if (len == 0) {
302 9069347b 2021-05-20 stsp err = got_error_msg(GOT_ERR_BAD_DELTA,
303 9069347b 2021-05-20 stsp "zero length delta");
306 8628c62d 2018-03-15 stsp err = next_delta_byte(&p, &remain);
309 b2f7af54 2018-11-11 stsp if (remain < len || SIZE_MAX - *outsize < len ||
310 b2f7af54 2018-11-11 stsp *outsize + len > maxoutsize)
311 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA,
312 9069347b 2021-05-20 stsp "bad delta copy length");
313 8628c62d 2018-03-15 stsp memcpy(outbuf + *outsize, p, len);
315 8628c62d 2018-03-15 stsp remain -= len;
316 8628c62d 2018-03-15 stsp *outsize += len;
320 8628c62d 2018-03-15 stsp if (*outsize != result_size)
321 9069347b 2021-05-20 stsp err = got_error_msg(GOT_ERR_BAD_DELTA,
322 9069347b 2021-05-20 stsp "delta application result size mismatch");
323 8628c62d 2018-03-15 stsp return err;
326 22484865 2018-03-13 stsp const struct got_error *
327 0e22967e 2018-02-11 stsp got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
328 b29656e2 2018-03-16 stsp size_t delta_len, FILE *outfile, size_t *outsize)
330 885d3e02 2018-01-27 stsp const struct got_error *err = NULL;
331 885d3e02 2018-01-27 stsp uint64_t base_size, result_size;
332 b29656e2 2018-03-16 stsp size_t remain = 0;
333 885d3e02 2018-01-27 stsp const uint8_t *p;
334 39ff877f 2018-03-13 stsp FILE *memstream = NULL;
335 39ff877f 2018-03-13 stsp char *memstream_buf = NULL;
336 39ff877f 2018-03-13 stsp size_t memstream_size = 0;
338 b29656e2 2018-03-16 stsp *outsize = 0;
340 885d3e02 2018-01-27 stsp if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
341 9069347b 2021-05-20 stsp return got_error_msg(GOT_ERR_BAD_DELTA, "delta too small");
343 885d3e02 2018-01-27 stsp p = delta_buf;
344 885d3e02 2018-01-27 stsp remain = delta_len;
345 22484865 2018-03-13 stsp err = parse_delta_sizes(&base_size, &result_size, &p, &remain);
347 885d3e02 2018-01-27 stsp return err;
349 39ff877f 2018-03-13 stsp if (result_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
350 39ff877f 2018-03-13 stsp memstream = open_memstream(&memstream_buf, &memstream_size);
352 885d3e02 2018-01-27 stsp /* Decode and execute copy instructions from the delta stream. */
353 885d3e02 2018-01-27 stsp err = next_delta_byte(&p, &remain);
354 06e5fc98 2018-02-11 stsp while (err == NULL && remain > 0) {
355 824801e7 2018-01-27 stsp if (*p & GOT_DELTA_BASE_COPY) {
356 885d3e02 2018-01-27 stsp off_t offset = 0;
357 885d3e02 2018-01-27 stsp size_t len = 0;
358 885d3e02 2018-01-27 stsp err = parse_opcode(&offset, &len, &p, &remain);
361 39ff877f 2018-03-13 stsp err = copy_from_base(base_file, offset, len,
362 39ff877f 2018-03-13 stsp memstream ? memstream : outfile);
363 06e5fc98 2018-02-11 stsp if (err == NULL) {
364 b29656e2 2018-03-16 stsp *outsize += len;
365 06e5fc98 2018-02-11 stsp if (remain > 0) {
371 885d3e02 2018-01-27 stsp size_t len = (size_t)*p;
372 885d3e02 2018-01-27 stsp if (len == 0) {
373 9069347b 2021-05-20 stsp err = got_error_msg(GOT_ERR_BAD_DELTA,
374 9069347b 2021-05-20 stsp "zero length delta");
377 885d3e02 2018-01-27 stsp err = next_delta_byte(&p, &remain);
380 39ff877f 2018-03-13 stsp err = copy_from_delta(&p, &remain, len,
381 39ff877f 2018-03-13 stsp memstream ? memstream : outfile);
382 885d3e02 2018-01-27 stsp if (err == NULL)
383 b29656e2 2018-03-16 stsp *outsize += len;
387 b29656e2 2018-03-16 stsp if (*outsize != result_size)
388 9069347b 2021-05-20 stsp err = got_error_msg(GOT_ERR_BAD_DELTA,
389 9069347b 2021-05-20 stsp "delta application result size mismatch");
391 39ff877f 2018-03-13 stsp if (memstream != NULL) {
392 56b63ca4 2021-01-22 stsp if (fclose(memstream) == EOF)
393 638f9024 2019-05-13 stsp err = got_error_from_errno("fclose");
394 39ff877f 2018-03-13 stsp if (err == NULL) {
396 39ff877f 2018-03-13 stsp n = fwrite(memstream_buf, 1, memstream_size, outfile);
397 39ff877f 2018-03-13 stsp if (n != memstream_size)
398 39ff877f 2018-03-13 stsp err = got_ferror(outfile, GOT_ERR_IO);
400 39ff877f 2018-03-13 stsp free(memstream_buf);
402 885d3e02 2018-01-27 stsp if (err == NULL)
403 885d3e02 2018-01-27 stsp rewind(outfile);
404 885d3e02 2018-01-27 stsp return err;
