Blame


1 32cb896c 2018-03-11 stsp /*
2 32cb896c 2018-03-11 stsp * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3 32cb896c 2018-03-11 stsp *
4 32cb896c 2018-03-11 stsp * Permission to use, copy, modify, and distribute this software for any
5 32cb896c 2018-03-11 stsp * purpose with or without fee is hereby granted, provided that the above
6 32cb896c 2018-03-11 stsp * copyright notice and this permission notice appear in all copies.
7 32cb896c 2018-03-11 stsp *
8 32cb896c 2018-03-11 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 32cb896c 2018-03-11 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 32cb896c 2018-03-11 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 32cb896c 2018-03-11 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 32cb896c 2018-03-11 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 32cb896c 2018-03-11 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 32cb896c 2018-03-11 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 32cb896c 2018-03-11 stsp */
16 32cb896c 2018-03-11 stsp
17 7e656b93 2018-03-17 stsp /* A pack file segment mapped with mmap(2). */
18 7e656b93 2018-03-17 stsp struct got_pack_mapping {
19 7e656b93 2018-03-17 stsp TAILQ_ENTRY(got_pack_mapping) entry;
20 7e656b93 2018-03-17 stsp int fd;
21 7e656b93 2018-03-17 stsp uint8_t *addr;
22 7e656b93 2018-03-17 stsp off_t offset;
23 7e656b93 2018-03-17 stsp size_t len;
24 7e656b93 2018-03-17 stsp };
25 7e656b93 2018-03-17 stsp
26 7e656b93 2018-03-17 stsp #define GOT_PACK_MAX_OPEN_MAPPINGS 512
27 7e656b93 2018-03-17 stsp #define GOT_PACK_MAPPING_MIN_SIZE 8192
28 7e656b93 2018-03-17 stsp #define GOT_PACK_MAPPING_MAX_SIZE (2048 * GOT_PACK_MAPPING_MIN_SIZE)
29 7e656b93 2018-03-17 stsp
30 7e656b93 2018-03-17 stsp /* An open pack file. */
31 7e656b93 2018-03-17 stsp struct got_pack {
32 7e656b93 2018-03-17 stsp char *path_packfile;
33 7e656b93 2018-03-17 stsp FILE *packfile;
34 7e656b93 2018-03-17 stsp size_t filesize;
35 7e656b93 2018-03-17 stsp int nmappings;
36 f7e127f3 2018-03-17 stsp
37 7e656b93 2018-03-17 stsp TAILQ_HEAD(, got_pack_mapping) mappings;
38 7e656b93 2018-03-17 stsp };
39 7e656b93 2018-03-17 stsp
40 7e656b93 2018-03-17 stsp const struct got_error *got_pack_close(struct got_pack *);
41 7e656b93 2018-03-17 stsp
42 32cb896c 2018-03-11 stsp /* See Documentation/technical/pack-format.txt in Git. */
43 32cb896c 2018-03-11 stsp
44 32cb896c 2018-03-11 stsp struct got_packidx_trailer {
45 32cb896c 2018-03-11 stsp u_int8_t packfile_sha1[SHA1_DIGEST_LENGTH];
46 32cb896c 2018-03-11 stsp u_int8_t packidx_sha1[SHA1_DIGEST_LENGTH];
47 32cb896c 2018-03-11 stsp } __attribute__((__packed__));
48 32cb896c 2018-03-11 stsp
49 32cb896c 2018-03-11 stsp /* Ignore pack index version 1 which is no longer written by Git. */
50 32cb896c 2018-03-11 stsp #define GOT_PACKIDX_VERSION 2
51 32cb896c 2018-03-11 stsp
52 32cb896c 2018-03-11 stsp struct got_packidx_v2_hdr {
53 32cb896c 2018-03-11 stsp uint32_t magic; /* big endian */
54 32cb896c 2018-03-11 stsp #define GOT_PACKIDX_V2_MAGIC 0xff744f63 /* "\377t0c" */
55 32cb896c 2018-03-11 stsp uint32_t version;
56 32cb896c 2018-03-11 stsp
57 32cb896c 2018-03-11 stsp /*
58 32cb896c 2018-03-11 stsp * Each entry N in the fanout table contains the number of objects in
59 32cb896c 2018-03-11 stsp * the packfile whose SHA1 begins with a byte less than or equal to N.
60 32cb896c 2018-03-11 stsp * The last entry (index 255) contains the number of objects in the
61 32cb896c 2018-03-11 stsp * pack file whose first SHA1 byte is <= 0xff, and thus records the
62 32cb896c 2018-03-11 stsp * total number of objects in the pack file. All pointer variables
63 32cb896c 2018-03-11 stsp * below point to tables with a corresponding number of entries.
64 32cb896c 2018-03-11 stsp */
65 32cb896c 2018-03-11 stsp uint32_t fanout_table[0xff + 1]; /* values are big endian */
66 32cb896c 2018-03-11 stsp
67 32cb896c 2018-03-11 stsp /* Sorted SHA1 checksums for each object in the pack file. */
68 32cb896c 2018-03-11 stsp struct got_object_id *sorted_ids;
69 32cb896c 2018-03-11 stsp
70 32cb896c 2018-03-11 stsp /* CRC32 of the packed representation of each object. */
71 32cb896c 2018-03-11 stsp uint32_t *crc32;
72 32cb896c 2018-03-11 stsp
73 32cb896c 2018-03-11 stsp /* Offset into the pack file for each object. */
74 32cb896c 2018-03-11 stsp uint32_t *offsets; /* values are big endian */
75 32cb896c 2018-03-11 stsp #define GOT_PACKIDX_OFFSET_VAL_MASK 0x7fffffff
76 32cb896c 2018-03-11 stsp #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX 0x80000000
77 32cb896c 2018-03-11 stsp
78 32cb896c 2018-03-11 stsp /* Large offsets table is empty for pack files < 2 GB. */
79 32cb896c 2018-03-11 stsp uint64_t *large_offsets; /* values are big endian */
80 32cb896c 2018-03-11 stsp
81 32cb896c 2018-03-11 stsp struct got_packidx_trailer trailer;
82 32cb896c 2018-03-11 stsp };
83 32cb896c 2018-03-11 stsp
84 32cb896c 2018-03-11 stsp struct got_packfile_hdr {
85 32cb896c 2018-03-11 stsp uint32_t signature;
86 32cb896c 2018-03-11 stsp #define GOT_PACKFILE_SIGNATURE 0x5041434b /* 'P' 'A' 'C' 'K' */
87 32cb896c 2018-03-11 stsp uint32_t version; /* big endian */
88 32cb896c 2018-03-11 stsp #define GOT_PACKFILE_VERSION 2
89 32cb896c 2018-03-11 stsp uint32_t nobjects; /* big endian */
90 32cb896c 2018-03-11 stsp };
91 32cb896c 2018-03-11 stsp
92 32cb896c 2018-03-11 stsp struct got_packfile_obj_hdr {
93 32cb896c 2018-03-11 stsp /*
94 32cb896c 2018-03-11 stsp * The object size field uses a variable length encoding:
95 32cb896c 2018-03-11 stsp * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
96 32cb896c 2018-03-11 stsp * least significant part and sizeN is the most significant part.
97 32cb896c 2018-03-11 stsp * If the MSB of a size byte is set, an additional size byte follows.
98 32cb896c 2018-03-11 stsp * Of the 7 remaining bits of size0, the first 3 bits indicate the
99 32cb896c 2018-03-11 stsp * object's type, and the remaining 4 bits contribute to the size.
100 32cb896c 2018-03-11 stsp */
101 32cb896c 2018-03-11 stsp uint8_t *size; /* variable length */
102 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_SIZE_MORE 0x80
103 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_SIZE0_TYPE_MASK 0x70 /* See struct got_object->type */
104 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT 4
105 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_SIZE0_VAL_MASK 0x0f
106 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_SIZE_VAL_MASK 0x7f
107 32cb896c 2018-03-11 stsp };
108 32cb896c 2018-03-11 stsp
109 32cb896c 2018-03-11 stsp /* If object is not a DELTA type. */
110 32cb896c 2018-03-11 stsp struct got_packfile_object_data {
111 32cb896c 2018-03-11 stsp uint8_t *data; /* compressed */
112 32cb896c 2018-03-11 stsp };
113 32cb896c 2018-03-11 stsp
114 32cb896c 2018-03-11 stsp /* If object is of type GOT_OBJ_TYPE_REF_DELTA. */
115 32cb896c 2018-03-11 stsp struct got_packfile_object_data_ref_delta {
116 32cb896c 2018-03-11 stsp uint8_t sha1[SHA1_DIGEST_LENGTH];
117 32cb896c 2018-03-11 stsp uint8_t *delta_data; /* compressed */
118 32cb896c 2018-03-11 stsp };
119 32cb896c 2018-03-11 stsp
120 32cb896c 2018-03-11 stsp /* If object is of type GOT_OBJ_TYPE_OFFSET_DELTA. */
121 32cb896c 2018-03-11 stsp struct got_packfile_object_data_offset_delta {
122 32cb896c 2018-03-11 stsp /*
123 32cb896c 2018-03-11 stsp * This offset is interpreted as a negative offset from
124 32cb896c 2018-03-11 stsp * the got_packfile_obj_hdr corresponding to this object.
125 32cb896c 2018-03-11 stsp * The size provided in the header specifies the amount
126 32cb896c 2018-03-11 stsp * of compressed delta data that follows.
127 32cb896c 2018-03-11 stsp *
128 32cb896c 2018-03-11 stsp * This field uses a variable length encoding of N bytes,
129 32cb896c 2018-03-11 stsp * where the MSB is always set except for the last byte.
130 32cb896c 2018-03-11 stsp * The value is encoded as a series of N 7 bit integers,
131 32cb896c 2018-03-11 stsp * which are concatenated, and if N > 1 the value 2^7 +
132 32cb896c 2018-03-11 stsp * 2^14 + ... + 2^(7 * (n-1)) is added to the result.
133 32cb896c 2018-03-11 stsp */
134 32cb896c 2018-03-11 stsp uint8_t *offset; /* variable length */
135 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_DELTA_OFF_MORE 0x80
136 32cb896c 2018-03-11 stsp #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK 0x7f
137 32cb896c 2018-03-11 stsp uint8_t *delta_data; /* compressed */
138 32cb896c 2018-03-11 stsp };
139 32cb896c 2018-03-11 stsp
140 32cb896c 2018-03-11 stsp struct got_packfile_obj_data {
141 32cb896c 2018-03-11 stsp union {
142 32cb896c 2018-03-11 stsp struct got_packfile_object_data data;
143 32cb896c 2018-03-11 stsp struct got_packfile_object_data_ref_delta ref_delta;
144 32cb896c 2018-03-11 stsp struct got_packfile_object_data_offset_delta offset_delta;
145 32cb896c 2018-03-11 stsp } __attribute__((__packed__));
146 32cb896c 2018-03-11 stsp } __attribute__((__packed__));
147 32cb896c 2018-03-11 stsp
148 32cb896c 2018-03-11 stsp const struct got_error *got_packidx_open(struct got_packidx_v2_hdr **,
149 32cb896c 2018-03-11 stsp const char *);
150 32cb896c 2018-03-11 stsp void got_packidx_close(struct got_packidx_v2_hdr *);
151 32cb896c 2018-03-11 stsp
152 32cb896c 2018-03-11 stsp const struct got_error *got_packfile_open_object(struct got_object **,
153 32cb896c 2018-03-11 stsp struct got_object_id *, struct got_repository *);
154 32cb896c 2018-03-11 stsp const struct got_error *got_packfile_extract_object(FILE **,
155 32cb896c 2018-03-11 stsp struct got_object *, struct got_repository *);
156 e0ab43e7 2018-03-16 stsp const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
157 e0ab43e7 2018-03-16 stsp struct got_object *, struct got_repository *);