commit 8628c62d73b92bfe39c6a8ec4d5a85f8a6fb662d from: Stefan Sperling date: Thu Mar 15 00:21:57 2018 UTC process small deltas in memory; unfortunately it is not faster... commit - 1cc8e7f9595db4aa8d3e79361376d1a0e7064635 commit + 8628c62d73b92bfe39c6a8ec4d5a85f8a6fb662d blob - 8b8243244f2258d9a028e56b322c67023310b69e blob + c187dc31fb4d77d7908a8334c2dc1a939230168d --- lib/delta.c +++ lib/delta.c @@ -258,6 +258,66 @@ got_delta_get_sizes(uint64_t *base_size, uint64_t *res p = delta_buf; remain = delta_len; return parse_delta_sizes(base_size, result_size, &p, &remain); +} + +const struct got_error * +got_delta_apply_in_mem(uint8_t *base_buf, const uint8_t *delta_buf, + size_t delta_len, uint8_t *outbuf, size_t *outsize) +{ + const struct got_error *err = NULL; + uint64_t base_size, result_size; + size_t remain; + const uint8_t *p; + + *outsize= 0; + + if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN) + return got_error(GOT_ERR_BAD_DELTA); + + p = delta_buf; + remain = delta_len; + err = parse_delta_sizes(&base_size, &result_size, &p, &remain); + if (err) + return err; + + /* Decode and execute copy instructions from the delta stream. */ + err = next_delta_byte(&p, &remain); + while (err == NULL && remain > 0) { + if (*p & GOT_DELTA_BASE_COPY) { + off_t offset = 0; + size_t len = 0; + err = parse_opcode(&offset, &len, &p, &remain); + if (err) + break; + memcpy(outbuf + *outsize, base_buf + offset, len); + if (err == NULL) { + *outsize += len; + if (remain > 0) { + p++; + remain--; + } + } + } else { + size_t len = (size_t)*p; + if (len == 0) { + err = got_error(GOT_ERR_BAD_DELTA); + break; + } + err = next_delta_byte(&p, &remain); + if (err) + break; + if (remain < len) + return got_error(GOT_ERR_BAD_DELTA); + memcpy(outbuf + *outsize, p, len); + p += len; + remain -= len; + *outsize += len; + } + } + + if (*outsize != result_size) + err = got_error(GOT_ERR_BAD_DELTA); + return err; } const struct got_error * blob - 3c7bb235cb37bcd85135050f703b4191e33fc84c blob + 27f892a1d1f51a54cd4519bf471c326285fd47ae --- lib/got_delta_lib.h +++ lib/got_delta_lib.h @@ -36,6 +36,8 @@ const struct got_error *got_delta_chain_get_base_type( struct got_delta_chain *); const struct got_error *got_delta_get_sizes(uint64_t *, uint64_t *, const uint8_t *, size_t); +const struct got_error *got_delta_apply_in_mem(uint8_t *, const uint8_t *, + size_t, uint8_t *, size_t *); const struct got_error *got_delta_apply(FILE *, const uint8_t *, size_t, FILE *); blob - 06e97009123ad8a83250cae46a1f7e0e4bde4d00 blob + 569429d80e967730cfa186780dfdf22d2ed164f6 --- lib/pack.c +++ lib/pack.c @@ -1093,31 +1093,33 @@ dump_delta_chain(struct got_delta_chain *deltas, FILE const struct got_error *err = NULL; struct got_delta *delta; FILE *base_file = NULL, *accum_file = NULL; + uint8_t *base_buf = NULL, *accum_buf = NULL; + size_t accum_size; uint64_t max_size; int n = 0; if (SIMPLEQ_EMPTY(&deltas->entries)) return got_error(GOT_ERR_BAD_DELTA_CHAIN); + /* We process small enough files entirely in memory for speed. */ err = get_delta_chain_max_size(&max_size, deltas); if (err) return err; - - if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) - base_file = fmemopen(NULL, max_size, "w+"); - else + if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) { + accum_buf = malloc(max_size); + if (accum_buf == NULL) + return got_error(GOT_ERR_NO_MEM); + } else { base_file = got_opentemp(); - if (base_file == NULL) - return got_error_from_errno(); + if (base_file == NULL) + return got_error_from_errno(); - if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) - accum_file = fmemopen(NULL, max_size, "w+"); - else accum_file = got_opentemp(); - if (accum_file == NULL) { - err = got_error_from_errno(); - fclose(base_file); - return err; + if (accum_file == NULL) { + err = got_error_from_errno(); + fclose(base_file); + return err; + } } /* Deltas are ordered in ascending order. */ @@ -1127,6 +1129,7 @@ dump_delta_chain(struct got_delta_chain *deltas, FILE if (n == 0) { FILE *delta_file; + size_t base_len; /* Plain object types are the delta base. */ if (delta->type != GOT_OBJ_TYPE_COMMIT && @@ -1149,13 +1152,28 @@ dump_delta_chain(struct got_delta_chain *deltas, FILE err = got_error_from_errno(); goto done; } - err = got_inflate_to_file(&delta_len, delta_file, - base_file); + if (base_file) + err = got_inflate_to_file(&delta_len, + delta_file, base_file); + else { + err = got_inflate_to_mem(&base_buf, &base_len, + delta_file); + if (base_len < max_size) { + uint8_t *p; + p = reallocarray(base_buf, 1, max_size); + if (p == NULL) { + err = got_error(GOT_ERR_NO_MEM); + goto done; + } + base_buf = p; + } + } fclose(delta_file); if (err) goto done; n++; - rewind(base_file); + if (base_file) + rewind(base_file); continue; } @@ -1188,25 +1206,46 @@ dump_delta_chain(struct got_delta_chain *deltas, FILE } /* delta_buf is now cached */ - err = got_delta_apply(base_file, delta_buf, delta_len, - /* Final delta application writes to the output file. */ - ++n < deltas->nentries ? accum_file : outfile); + if (base_buf) { + err = got_delta_apply_in_mem(base_buf, delta_buf, + delta_len, accum_buf, &accum_size); + n++; + } else { + err = got_delta_apply(base_file, delta_buf, delta_len, + /* Final delta application writes to output file. */ + ++n < deltas->nentries ? accum_file : outfile); + } if (err) goto done; if (n < deltas->nentries) { /* Accumulated delta becomes the new base. */ - FILE *tmp = accum_file; - accum_file = base_file; - base_file = tmp; - rewind(base_file); - rewind(accum_file); + if (base_buf) { + uint8_t *tmp = accum_buf; + accum_buf = base_buf; + base_buf = tmp; + } else { + FILE *tmp = accum_file; + accum_file = base_file; + base_file = tmp; + rewind(base_file); + rewind(accum_file); + } } } done: - fclose(base_file); - fclose(accum_file); + free(base_buf); + if (accum_buf) { + size_t len = fwrite(accum_buf, 1, accum_size, outfile); + free(accum_buf); + if (len != accum_size) + return got_ferror(outfile, GOT_ERR_IO); + } + if (base_file) + fclose(base_file); + if (accum_file) + fclose(accum_file); rewind(outfile); return err; }