2 93658fb9 2020-03-18 stsp * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
4 93658fb9 2020-03-18 stsp * Permission to use, copy, modify, and distribute this software for any
5 93658fb9 2020-03-18 stsp * purpose with or without fee is hereby granted, provided that the above
6 93658fb9 2020-03-18 stsp * copyright notice and this permission notice appear in all copies.
8 93658fb9 2020-03-18 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 93658fb9 2020-03-18 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 93658fb9 2020-03-18 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 93658fb9 2020-03-18 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 93658fb9 2020-03-18 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 93658fb9 2020-03-18 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 93658fb9 2020-03-18 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 93658fb9 2020-03-18 stsp #include <sys/queue.h>
18 93658fb9 2020-03-18 stsp #include <sys/stat.h>
19 93658fb9 2020-03-18 stsp #include <sys/syslimits.h>
20 93658fb9 2020-03-18 stsp #include <sys/time.h>
21 93658fb9 2020-03-18 stsp #include <sys/types.h>
22 93658fb9 2020-03-18 stsp #include <sys/uio.h>
24 93658fb9 2020-03-18 stsp #include <stdint.h>
25 93658fb9 2020-03-18 stsp #include <errno.h>
26 93658fb9 2020-03-18 stsp #include <imsg.h>
27 93658fb9 2020-03-18 stsp #include <limits.h>
28 93658fb9 2020-03-18 stsp #include <signal.h>
29 93658fb9 2020-03-18 stsp #include <stdio.h>
30 93658fb9 2020-03-18 stsp #include <stdlib.h>
31 93658fb9 2020-03-18 stsp #include <string.h>
32 93658fb9 2020-03-18 stsp #include <ctype.h>
33 93658fb9 2020-03-18 stsp #include <sha1.h>
34 93658fb9 2020-03-18 stsp #include <fcntl.h>
35 93658fb9 2020-03-18 stsp #include <zlib.h>
36 93658fb9 2020-03-18 stsp #include <err.h>
37 93658fb9 2020-03-18 stsp #include <assert.h>
38 93658fb9 2020-03-18 stsp #include <dirent.h>
40 93658fb9 2020-03-18 stsp #include "got_error.h"
41 93658fb9 2020-03-18 stsp #include "got_object.h"
43 93658fb9 2020-03-18 stsp #include "got_lib_sha1.h"
44 93658fb9 2020-03-18 stsp #include "got_lib_delta.h"
45 93658fb9 2020-03-18 stsp #include "got_lib_inflate.h"
46 93658fb9 2020-03-18 stsp #include "got_lib_object.h"
47 93658fb9 2020-03-18 stsp #include "got_lib_object_parse.h"
48 93658fb9 2020-03-18 stsp #include "got_lib_object_idset.h"
49 93658fb9 2020-03-18 stsp #include "got_lib_privsep.h"
51 93658fb9 2020-03-18 stsp typedef struct Cinfo Cinfo;
52 93658fb9 2020-03-18 stsp typedef struct Tinfo Tinfo;
53 93658fb9 2020-03-18 stsp typedef struct Object Object;
54 93658fb9 2020-03-18 stsp typedef struct Pack Pack;
55 93658fb9 2020-03-18 stsp typedef struct Buf Buf;
56 93658fb9 2020-03-18 stsp typedef struct Dirent Dirent;
57 93658fb9 2020-03-18 stsp typedef struct Idxent Idxent;
58 93658fb9 2020-03-18 stsp typedef struct Ols Ols;
61 93658fb9 2020-03-18 stsp /* 5k objects should be enough */
62 93658fb9 2020-03-18 stsp Cachemax = 5*1024,
63 93658fb9 2020-03-18 stsp Pathmax = 512,
64 93658fb9 2020-03-18 stsp Hashsz = 20,
65 93658fb9 2020-03-18 stsp Pktmax = 65536,
67 93658fb9 2020-03-18 stsp Nproto = 16,
69 93658fb9 2020-03-18 stsp Nhost = 256,
70 93658fb9 2020-03-18 stsp Npath = 128,
72 93658fb9 2020-03-18 stsp Nbranch = 32,
75 93658fb9 2020-03-18 stsp typedef enum Type {
77 93658fb9 2020-03-18 stsp GCommit = 1,
81 93658fb9 2020-03-18 stsp GOdelta = 6,
82 93658fb9 2020-03-18 stsp GRdelta = 7,
86 93658fb9 2020-03-18 stsp Cloaded = 1 << 0,
87 93658fb9 2020-03-18 stsp Cidx = 1 << 1,
88 93658fb9 2020-03-18 stsp Ccache = 1 << 2,
89 93658fb9 2020-03-18 stsp Cexist = 1 << 3,
90 93658fb9 2020-03-18 stsp Cparsed = 1 << 5,
93 93658fb9 2020-03-18 stsp struct Dirent {
97 93658fb9 2020-03-18 stsp struct got_object_id h;
100 93658fb9 2020-03-18 stsp struct Object {
101 93658fb9 2020-03-18 stsp /* Git data */
102 93658fb9 2020-03-18 stsp struct got_object_id hash;
105 93658fb9 2020-03-18 stsp /* Cache */
109 93658fb9 2020-03-18 stsp Object *next;
110 93658fb9 2020-03-18 stsp Object *prev;
112 93658fb9 2020-03-18 stsp /* For indexing */
115 93658fb9 2020-03-18 stsp uint32_t crc;
117 93658fb9 2020-03-18 stsp /* Everything below here gets cleared */
119 93658fb9 2020-03-18 stsp char *data;
120 93658fb9 2020-03-18 stsp /* size excludes header */
121 93658fb9 2020-03-18 stsp off_t size;
124 93658fb9 2020-03-18 stsp Cinfo *commit;
125 93658fb9 2020-03-18 stsp Tinfo *tree;
129 93658fb9 2020-03-18 stsp struct Tinfo {
131 93658fb9 2020-03-18 stsp Dirent *ent;
135 93658fb9 2020-03-18 stsp struct Cinfo {
136 93658fb9 2020-03-18 stsp /* Commit */
137 93658fb9 2020-03-18 stsp struct got_object_id *parent;
138 93658fb9 2020-03-18 stsp int nparent;
139 93658fb9 2020-03-18 stsp struct got_object_id tree;
140 93658fb9 2020-03-18 stsp char *author;
141 93658fb9 2020-03-18 stsp char *committer;
144 93658fb9 2020-03-18 stsp off_t ctime;
145 93658fb9 2020-03-18 stsp off_t mtime;
148 93658fb9 2020-03-18 stsp typedef struct Buf Buf;
150 93658fb9 2020-03-18 stsp struct Buf {
153 93658fb9 2020-03-18 stsp char *data;
156 93658fb9 2020-03-18 stsp static int readpacked(FILE *, Object *, int);
157 93658fb9 2020-03-18 stsp static Object *readidxobject(FILE *, struct got_object_id, int);
159 93658fb9 2020-03-18 stsp struct got_object_idset *objcache;
160 93658fb9 2020-03-18 stsp int next_object_id;
161 93658fb9 2020-03-18 stsp Object *lruhead;
162 93658fb9 2020-03-18 stsp Object *lrutail;
163 93658fb9 2020-03-18 stsp int ncache;
165 93658fb9 2020-03-18 stsp #define GETBE16(b)\
166 93658fb9 2020-03-18 stsp ((((b)[0] & 0xFFul) << 8) | \
167 93658fb9 2020-03-18 stsp (((b)[1] & 0xFFul) << 0))
169 93658fb9 2020-03-18 stsp #define GETBE32(b)\
170 93658fb9 2020-03-18 stsp ((((b)[0] & 0xFFul) << 24) | \
171 93658fb9 2020-03-18 stsp (((b)[1] & 0xFFul) << 16) | \
172 93658fb9 2020-03-18 stsp (((b)[2] & 0xFFul) << 8) | \
173 93658fb9 2020-03-18 stsp (((b)[3] & 0xFFul) << 0))
174 93658fb9 2020-03-18 stsp #define GETBE64(b)\
175 93658fb9 2020-03-18 stsp ((((b)[0] & 0xFFull) << 56) | \
176 93658fb9 2020-03-18 stsp (((b)[1] & 0xFFull) << 48) | \
177 93658fb9 2020-03-18 stsp (((b)[2] & 0xFFull) << 40) | \
178 93658fb9 2020-03-18 stsp (((b)[3] & 0xFFull) << 32) | \
179 93658fb9 2020-03-18 stsp (((b)[4] & 0xFFull) << 24) | \
180 93658fb9 2020-03-18 stsp (((b)[5] & 0xFFull) << 16) | \
181 93658fb9 2020-03-18 stsp (((b)[6] & 0xFFull) << 8) | \
182 93658fb9 2020-03-18 stsp (((b)[7] & 0xFFull) << 0))
184 93658fb9 2020-03-18 stsp #define PUTBE16(b, n)\
186 93658fb9 2020-03-18 stsp (b)[0] = (n) >> 8; \
187 93658fb9 2020-03-18 stsp (b)[1] = (n) >> 0; \
190 93658fb9 2020-03-18 stsp #define PUTBE32(b, n)\
192 93658fb9 2020-03-18 stsp (b)[0] = (n) >> 24; \
193 93658fb9 2020-03-18 stsp (b)[1] = (n) >> 16; \
194 93658fb9 2020-03-18 stsp (b)[2] = (n) >> 8; \
195 93658fb9 2020-03-18 stsp (b)[3] = (n) >> 0; \
198 93658fb9 2020-03-18 stsp #define PUTBE64(b, n)\
200 93658fb9 2020-03-18 stsp (b)[0] = (n) >> 56; \
201 93658fb9 2020-03-18 stsp (b)[1] = (n) >> 48; \
202 93658fb9 2020-03-18 stsp (b)[2] = (n) >> 40; \
203 93658fb9 2020-03-18 stsp (b)[3] = (n) >> 32; \
204 93658fb9 2020-03-18 stsp (b)[4] = (n) >> 24; \
205 93658fb9 2020-03-18 stsp (b)[5] = (n) >> 16; \
206 93658fb9 2020-03-18 stsp (b)[6] = (n) >> 8; \
207 93658fb9 2020-03-18 stsp (b)[7] = (n) >> 0; \
211 93658fb9 2020-03-18 stsp charval(int c, int *err)
213 93658fb9 2020-03-18 stsp if(c >= '0' && c <= '9')
214 93658fb9 2020-03-18 stsp return c - '0';
215 93658fb9 2020-03-18 stsp if(c >= 'a' && c <= 'f')
216 93658fb9 2020-03-18 stsp return c - 'a' + 10;
217 93658fb9 2020-03-18 stsp if(c >= 'A' && c <= 'F')
218 93658fb9 2020-03-18 stsp return c - 'A' + 10;
224 93658fb9 2020-03-18 stsp hparse(struct got_object_id *h, char *b)
226 93658fb9 2020-03-18 stsp int i, err;
229 93658fb9 2020-03-18 stsp for(i = 0; i < sizeof(h->sha1); i++){
231 93658fb9 2020-03-18 stsp h->sha1[i] = 0;
232 93658fb9 2020-03-18 stsp h->sha1[i] |= ((charval(b[2*i], &err) & 0xf) << 4);
233 93658fb9 2020-03-18 stsp h->sha1[i] |= ((charval(b[2*i+1], &err)& 0xf) << 0);
240 93658fb9 2020-03-18 stsp static void *
241 93658fb9 2020-03-18 stsp emalloc(size_t n)
245 93658fb9 2020-03-18 stsp v = calloc(n, 1);
246 93658fb9 2020-03-18 stsp if(v == NULL)
247 93658fb9 2020-03-18 stsp err(1, "malloc:");
251 93658fb9 2020-03-18 stsp static void *
252 93658fb9 2020-03-18 stsp erealloc(void *p, ulong n)
256 93658fb9 2020-03-18 stsp v = realloc(p, n);
257 93658fb9 2020-03-18 stsp if(v == NULL)
258 93658fb9 2020-03-18 stsp err(1, "realloc:");
259 93658fb9 2020-03-18 stsp memset(v, 0, n);
264 93658fb9 2020-03-18 stsp hasheq(struct got_object_id *a, struct got_object_id *b)
266 93658fb9 2020-03-18 stsp return memcmp(a->sha1, b->sha1, sizeof(a->sha1)) == 0;
269 93658fb9 2020-03-18 stsp static char *
270 93658fb9 2020-03-18 stsp typestr(int t)
272 93658fb9 2020-03-18 stsp char *types[] = {
281 93658fb9 2020-03-18 stsp if (t < 0 || t >= sizeof(types)/sizeof(types[0]))
283 93658fb9 2020-03-18 stsp return types[t];
286 93658fb9 2020-03-18 stsp static char *
287 93658fb9 2020-03-18 stsp hashfmt(char *out, size_t nout, struct got_object_id *h)
289 93658fb9 2020-03-18 stsp int i, n, c0, c1;
292 93658fb9 2020-03-18 stsp if (nout < 2*sizeof(h->sha1) + 1)
293 93658fb9 2020-03-18 stsp return NULL;
295 93658fb9 2020-03-18 stsp for(i = 0; i < sizeof(h->sha1); i++){
296 93658fb9 2020-03-18 stsp n = (h->sha1[i] >> 4) & 0xf;
297 93658fb9 2020-03-18 stsp c0 = (n >= 10) ? n-10 + 'a' : n + '0';
298 93658fb9 2020-03-18 stsp n = h->sha1[i] & 0xf;
299 93658fb9 2020-03-18 stsp c1 = (n >= 10) ? n-10 + 'a' : n + '0';
304 93658fb9 2020-03-18 stsp return out;
307 93658fb9 2020-03-18 stsp static void
308 93658fb9 2020-03-18 stsp clear(Object *o)
313 93658fb9 2020-03-18 stsp assert(o->refs == 0);
314 93658fb9 2020-03-18 stsp assert((o->flag & Ccache) == 0);
315 93658fb9 2020-03-18 stsp assert(o->flag & Cloaded);
316 93658fb9 2020-03-18 stsp switch(o->type){
317 93658fb9 2020-03-18 stsp case GCommit:
318 93658fb9 2020-03-18 stsp if(!o->commit)
320 93658fb9 2020-03-18 stsp free(o->commit->parent);
321 93658fb9 2020-03-18 stsp free(o->commit->author);
322 93658fb9 2020-03-18 stsp free(o->commit->committer);
323 93658fb9 2020-03-18 stsp free(o->commit);
324 93658fb9 2020-03-18 stsp o->commit = NULL;
326 93658fb9 2020-03-18 stsp case GTree:
327 93658fb9 2020-03-18 stsp if(!o->tree)
329 93658fb9 2020-03-18 stsp free(o->tree->ent);
330 93658fb9 2020-03-18 stsp free(o->tree);
331 93658fb9 2020-03-18 stsp o->tree = NULL;
337 93658fb9 2020-03-18 stsp free(o->all);
338 93658fb9 2020-03-18 stsp o->all = NULL;
339 93658fb9 2020-03-18 stsp o->data = NULL;
340 93658fb9 2020-03-18 stsp o->flag &= ~Cloaded;
343 93658fb9 2020-03-18 stsp static void
344 93658fb9 2020-03-18 stsp unref(Object *o)
349 93658fb9 2020-03-18 stsp if(!o->refs)
353 93658fb9 2020-03-18 stsp static Object*
354 93658fb9 2020-03-18 stsp ref(Object *o)
360 93658fb9 2020-03-18 stsp static void
361 93658fb9 2020-03-18 stsp cache(Object *o)
363 93658fb9 2020-03-18 stsp char buf[41];
366 93658fb9 2020-03-18 stsp hashfmt(buf, sizeof(buf), &o->hash);
367 93658fb9 2020-03-18 stsp if(o == lruhead)
369 93658fb9 2020-03-18 stsp if(o == lrutail)
370 93658fb9 2020-03-18 stsp lrutail = lrutail->prev;
371 93658fb9 2020-03-18 stsp if(!(o->flag & Cexist)){
372 93658fb9 2020-03-18 stsp got_object_idset_add(objcache, &o->hash, o);
373 93658fb9 2020-03-18 stsp o->id = next_object_id++;
374 93658fb9 2020-03-18 stsp o->flag |= Cexist;
376 93658fb9 2020-03-18 stsp if(o->prev)
377 93658fb9 2020-03-18 stsp o->prev->next = o->next;
378 93658fb9 2020-03-18 stsp if(o->next)
379 93658fb9 2020-03-18 stsp o->next->prev = o->prev;
380 93658fb9 2020-03-18 stsp if(lrutail == o){
381 93658fb9 2020-03-18 stsp lrutail = o->prev;
382 93658fb9 2020-03-18 stsp lrutail->next = NULL;
383 93658fb9 2020-03-18 stsp }else if(!lrutail)
384 93658fb9 2020-03-18 stsp lrutail = o;
385 93658fb9 2020-03-18 stsp if(lruhead)
386 93658fb9 2020-03-18 stsp lruhead->prev = o;
387 93658fb9 2020-03-18 stsp o->next = lruhead;
388 93658fb9 2020-03-18 stsp o->prev = NULL;
389 93658fb9 2020-03-18 stsp lruhead = o;
391 93658fb9 2020-03-18 stsp if(!(o->flag & Ccache)){
392 93658fb9 2020-03-18 stsp o->flag |= Ccache;
396 93658fb9 2020-03-18 stsp while(ncache > Cachemax){
397 93658fb9 2020-03-18 stsp p = lrutail;
398 93658fb9 2020-03-18 stsp lrutail = p->prev;
399 93658fb9 2020-03-18 stsp lrutail->next = NULL;
400 93658fb9 2020-03-18 stsp p->flag &= ~Ccache;
401 93658fb9 2020-03-18 stsp p->prev = NULL;
402 93658fb9 2020-03-18 stsp p->next = NULL;
409 93658fb9 2020-03-18 stsp preadbe32(FILE *b, int *v, off_t off)
411 93658fb9 2020-03-18 stsp char buf[4];
413 93658fb9 2020-03-18 stsp if(fseek(b, off, 0) == -1)
415 93658fb9 2020-03-18 stsp if(fread(buf, 1, sizeof(buf), b) == -1)
417 93658fb9 2020-03-18 stsp *v = GETBE32(buf);
422 93658fb9 2020-03-18 stsp preadbe64(FILE *b, off_t *v, off_t off)
424 93658fb9 2020-03-18 stsp char buf[8];
426 93658fb9 2020-03-18 stsp if(fseek(b, off, 0) == -1)
428 93658fb9 2020-03-18 stsp if(fread(buf, 1, sizeof(buf), b) == -1)
430 93658fb9 2020-03-18 stsp *v = GETBE64(buf);
435 93658fb9 2020-03-18 stsp readvint(char *p, char **pp)
437 93658fb9 2020-03-18 stsp int i, n, c;
443 93658fb9 2020-03-18 stsp n |= (c & 0x7f) << i;
445 93658fb9 2020-03-18 stsp } while (c & 0x80);
452 93658fb9 2020-03-18 stsp applydelta(Object *dst, Object *base, char *d, int nd)
454 93658fb9 2020-03-18 stsp char *r, *b, *ed, *er;
455 93658fb9 2020-03-18 stsp int n, nr, c;
456 93658fb9 2020-03-18 stsp off_t o, l;
458 93658fb9 2020-03-18 stsp ed = d + nd;
459 93658fb9 2020-03-18 stsp b = base->data;
460 93658fb9 2020-03-18 stsp n = readvint(d, &d);
461 93658fb9 2020-03-18 stsp if(n != base->size){
462 93658fb9 2020-03-18 stsp fprintf(stderr, "mismatched source size");
466 93658fb9 2020-03-18 stsp nr = readvint(d, &d);
467 93658fb9 2020-03-18 stsp r = emalloc(nr + 64);
468 93658fb9 2020-03-18 stsp n = snprintf(r, 64, "%s %d", typestr(base->type), nr) + 1;
469 93658fb9 2020-03-18 stsp dst->all = r;
470 93658fb9 2020-03-18 stsp dst->type = base->type;
471 93658fb9 2020-03-18 stsp dst->data = r + n;
472 93658fb9 2020-03-18 stsp dst->size = nr;
473 93658fb9 2020-03-18 stsp er = dst->data + nr;
474 93658fb9 2020-03-18 stsp r = dst->data;
477 93658fb9 2020-03-18 stsp if(d == ed)
481 93658fb9 2020-03-18 stsp fprintf(stderr, "bad delta encoding");
484 93658fb9 2020-03-18 stsp /* copy from base */
485 93658fb9 2020-03-18 stsp if(c & 0x80){
488 93658fb9 2020-03-18 stsp /* Offset in base */
489 93658fb9 2020-03-18 stsp if(c & 0x01 && d != ed) o |= (*d++ << 0) & 0x000000ff;
490 93658fb9 2020-03-18 stsp if(c & 0x02 && d != ed) o |= (*d++ << 8) & 0x0000ff00;
491 93658fb9 2020-03-18 stsp if(c & 0x04 && d != ed) o |= (*d++ << 16) & 0x00ff0000;
492 93658fb9 2020-03-18 stsp if(c & 0x08 && d != ed) o |= (*d++ << 24) & 0xff000000;
494 93658fb9 2020-03-18 stsp /* Length to copy */
495 93658fb9 2020-03-18 stsp if(c & 0x10 && d != ed) l |= (*d++ << 0) & 0x0000ff;
496 93658fb9 2020-03-18 stsp if(c & 0x20 && d != ed) l |= (*d++ << 8) & 0x00ff00;
497 93658fb9 2020-03-18 stsp if(c & 0x40 && d != ed) l |= (*d++ << 16) & 0xff0000;
498 93658fb9 2020-03-18 stsp if(l == 0) l = 0x10000;
500 93658fb9 2020-03-18 stsp assert(o + l <= base->size);
501 93658fb9 2020-03-18 stsp memmove(r, b + o, l);
503 93658fb9 2020-03-18 stsp /* inline data */
505 93658fb9 2020-03-18 stsp memmove(r, d, c);
511 93658fb9 2020-03-18 stsp if(r != er){
512 93658fb9 2020-03-18 stsp fprintf(stderr, "truncated delta (%zd)", er - r);
520 93658fb9 2020-03-18 stsp readrdelta(FILE *f, Object *o, int nd, int flag)
522 93658fb9 2020-03-18 stsp const struct got_error *e;
523 93658fb9 2020-03-18 stsp struct got_object_id h;
525 93658fb9 2020-03-18 stsp uint8_t *d;
529 93658fb9 2020-03-18 stsp if(fread(h.sha1, 1, sizeof(h.sha1), f) != sizeof(h.sha1))
530 93658fb9 2020-03-18 stsp goto error;
531 93658fb9 2020-03-18 stsp if(hasheq(&o->hash, &h))
532 93658fb9 2020-03-18 stsp goto error;
533 93658fb9 2020-03-18 stsp if ((e = got_inflate_to_mem(&d, &n, f)) != NULL)
534 93658fb9 2020-03-18 stsp goto error;
535 93658fb9 2020-03-18 stsp o->len = ftello(f) - o->off;
536 93658fb9 2020-03-18 stsp if(d == NULL || n != nd)
537 93658fb9 2020-03-18 stsp goto error;
538 93658fb9 2020-03-18 stsp if((b = readidxobject(f, h, flag)) == NULL)
539 93658fb9 2020-03-18 stsp goto error;
540 93658fb9 2020-03-18 stsp if(applydelta(o, b, d, n) == -1)
541 93658fb9 2020-03-18 stsp goto error;
550 93658fb9 2020-03-18 stsp readodelta(FILE *f, Object *o, off_t nd, off_t p, int flag)
553 93658fb9 2020-03-18 stsp uint8_t *d;
561 93658fb9 2020-03-18 stsp if((c = fgetc(f)) == -1)
562 93658fb9 2020-03-18 stsp goto error;
563 93658fb9 2020-03-18 stsp r |= c & 0x7f;
564 93658fb9 2020-03-18 stsp if (!(c & 0x80))
568 93658fb9 2020-03-18 stsp }while(c & 0x80);
571 93658fb9 2020-03-18 stsp fprintf(stderr, "junk offset -%lld (from %lld)", r, p);
572 93658fb9 2020-03-18 stsp goto error;
575 93658fb9 2020-03-18 stsp if (got_inflate_to_mem(&d, &n, f) == NULL)
576 93658fb9 2020-03-18 stsp goto error;
577 93658fb9 2020-03-18 stsp o->len = ftello(f) - o->off;
578 93658fb9 2020-03-18 stsp if(d == NULL || n != nd)
579 93658fb9 2020-03-18 stsp goto error;
580 93658fb9 2020-03-18 stsp if(fseek(f, p - r, 0) == -1)
581 93658fb9 2020-03-18 stsp goto error;
582 93658fb9 2020-03-18 stsp if(readpacked(f, &b, flag) == -1)
583 93658fb9 2020-03-18 stsp goto error;
584 93658fb9 2020-03-18 stsp if(applydelta(o, &b, d, nd) == -1)
585 93658fb9 2020-03-18 stsp goto error;
594 93658fb9 2020-03-18 stsp readpacked(FILE *f, Object *o, int flag)
596 93658fb9 2020-03-18 stsp const struct got_error *e;
597 93658fb9 2020-03-18 stsp int c, s, n;
598 93658fb9 2020-03-18 stsp off_t l, p;
599 93658fb9 2020-03-18 stsp size_t ndata;
600 93658fb9 2020-03-18 stsp uint8_t *data;
604 93658fb9 2020-03-18 stsp p = ftello(f);
605 93658fb9 2020-03-18 stsp c = fgetc(f);
606 93658fb9 2020-03-18 stsp if(c == -1)
608 93658fb9 2020-03-18 stsp l = c & 0xf;
610 93658fb9 2020-03-18 stsp t = (c >> 4) & 0x7;
612 93658fb9 2020-03-18 stsp fprintf(stderr, "unknown type for byte %x", c);
615 93658fb9 2020-03-18 stsp while(c & 0x80){
616 93658fb9 2020-03-18 stsp if((c = fgetc(f)) == -1)
618 93658fb9 2020-03-18 stsp l |= (c & 0x7f) << s;
624 93658fb9 2020-03-18 stsp fprintf(stderr, "invalid object at %lld", ftello(f));
626 93658fb9 2020-03-18 stsp case GCommit:
627 93658fb9 2020-03-18 stsp case GTree:
629 93658fb9 2020-03-18 stsp case GBlob:
630 93658fb9 2020-03-18 stsp b.sz = 64 + l;
632 93658fb9 2020-03-18 stsp b.data = emalloc(b.sz);
633 93658fb9 2020-03-18 stsp n = snprintf(b.data, 64, "%s %lld", typestr(t), l) + 1;
635 93658fb9 2020-03-18 stsp e = got_inflate_to_mem(&data, &ndata, f);
636 93658fb9 2020-03-18 stsp if (e != NULL || n + ndata >= b.sz) {
637 93658fb9 2020-03-18 stsp free(b.data);
640 93658fb9 2020-03-18 stsp memcpy(b.data + n, data, ndata);
641 93658fb9 2020-03-18 stsp o->len = ftello(f) - o->off;
642 93658fb9 2020-03-18 stsp o->type = t;
643 93658fb9 2020-03-18 stsp o->all = b.data;
644 93658fb9 2020-03-18 stsp o->data = b.data + n;
645 93658fb9 2020-03-18 stsp o->size = ndata;
646 93658fb9 2020-03-18 stsp free(data);
648 93658fb9 2020-03-18 stsp case GOdelta:
649 93658fb9 2020-03-18 stsp if(readodelta(f, o, l, p, flag) == -1)
652 93658fb9 2020-03-18 stsp case GRdelta:
653 93658fb9 2020-03-18 stsp if(readrdelta(f, o, l, flag) == -1)
657 93658fb9 2020-03-18 stsp o->flag |= Cloaded|flag;
662 93658fb9 2020-03-18 stsp readloose(FILE *f, Object *o, int flag)
664 93658fb9 2020-03-18 stsp struct { char *tag; int type; } *p, types[] = {
665 93658fb9 2020-03-18 stsp {"blob", GBlob},
666 93658fb9 2020-03-18 stsp {"tree", GTree},
667 93658fb9 2020-03-18 stsp {"commit", GCommit},
668 93658fb9 2020-03-18 stsp {"tag", GTag},
671 93658fb9 2020-03-18 stsp char *s, *e;
672 93658fb9 2020-03-18 stsp uint8_t *d;
677 93658fb9 2020-03-18 stsp if (got_inflate_to_mem(&d, &n, f) != NULL)
680 93658fb9 2020-03-18 stsp s = (char *)d;
681 93658fb9 2020-03-18 stsp o->type = GNone;
682 93658fb9 2020-03-18 stsp for(p = types; p->tag; p++){
683 93658fb9 2020-03-18 stsp l = strlen(p->tag);
684 93658fb9 2020-03-18 stsp if(strncmp(s, p->tag, l) == 0){
686 93658fb9 2020-03-18 stsp o->type = p->type;
687 93658fb9 2020-03-18 stsp while(!isspace(*s))
692 93658fb9 2020-03-18 stsp if(o->type == GNone){
693 93658fb9 2020-03-18 stsp free(o->data);
696 93658fb9 2020-03-18 stsp sz = strtol(s, &e, 0);
697 93658fb9 2020-03-18 stsp if(e == s || *e++ != 0){
698 93658fb9 2020-03-18 stsp fprintf(stderr, "malformed object header");
699 93658fb9 2020-03-18 stsp goto error;
701 93658fb9 2020-03-18 stsp if(sz != n - (e - (char *)d)){
702 93658fb9 2020-03-18 stsp fprintf(stderr, "mismatched sizes");
703 93658fb9 2020-03-18 stsp goto error;
705 93658fb9 2020-03-18 stsp o->size = sz;
706 93658fb9 2020-03-18 stsp o->data = e;
707 93658fb9 2020-03-18 stsp o->all = d;
708 93658fb9 2020-03-18 stsp o->flag |= Cloaded|flag;
716 93658fb9 2020-03-18 stsp static off_t
717 93658fb9 2020-03-18 stsp searchindex(FILE *f, struct got_object_id h)
719 93658fb9 2020-03-18 stsp int lo, hi, idx, i, nent;
720 93658fb9 2020-03-18 stsp off_t o, oo;
721 93658fb9 2020-03-18 stsp struct got_object_id hh;
725 93658fb9 2020-03-18 stsp * Read the fanout table. The fanout table
726 93658fb9 2020-03-18 stsp * contains 256 entries, corresponsding to
727 93658fb9 2020-03-18 stsp * the first byte of the hash. Each entry
728 93658fb9 2020-03-18 stsp * is a 4 byte big endian integer, containing
729 93658fb9 2020-03-18 stsp * the total number of entries with a leading
730 93658fb9 2020-03-18 stsp * byte <= the table index, allowing us to
731 93658fb9 2020-03-18 stsp * rapidly do a binary search on them.
733 93658fb9 2020-03-18 stsp if (h.sha1[0] == 0){
735 93658fb9 2020-03-18 stsp if(preadbe32(f, &hi, o) == -1)
738 93658fb9 2020-03-18 stsp o += h.sha1[0]*4 - 4;
739 93658fb9 2020-03-18 stsp if(preadbe32(f, &lo, o + 0) == -1)
741 93658fb9 2020-03-18 stsp if(preadbe32(f, &hi, o + 4) == -1)
744 93658fb9 2020-03-18 stsp if(hi == lo)
745 93658fb9 2020-03-18 stsp goto notfound;
746 93658fb9 2020-03-18 stsp if(preadbe32(f, &nent, 8 + 255*4) == -1)
750 93658fb9 2020-03-18 stsp * Now that we know the range of hashes that the
751 93658fb9 2020-03-18 stsp * entry may exist in, read them in so we can do
752 93658fb9 2020-03-18 stsp * a bsearch.
755 93658fb9 2020-03-18 stsp fseek(f, Hashsz*lo + 8 + 256*4, 0);
756 93658fb9 2020-03-18 stsp for(i = 0; i < hi - lo; i++){
757 93658fb9 2020-03-18 stsp if(fread(hh.sha1, 1, sizeof(hh.sha1), f) == -1)
759 93658fb9 2020-03-18 stsp if(hasheq(&hh, &h))
760 93658fb9 2020-03-18 stsp idx = lo + i;
762 93658fb9 2020-03-18 stsp if(idx == -1)
763 93658fb9 2020-03-18 stsp goto notfound;
767 93658fb9 2020-03-18 stsp * We found the entry. If it's 32 bits, then we
768 93658fb9 2020-03-18 stsp * can just return the oset, otherwise the 32
769 93658fb9 2020-03-18 stsp * bit entry contains the oset to the 64 bit
772 93658fb9 2020-03-18 stsp oo = 8; /* Header */
773 93658fb9 2020-03-18 stsp oo += 256*4; /* Fanout table */
774 93658fb9 2020-03-18 stsp oo += Hashsz*nent; /* Hashes */
775 93658fb9 2020-03-18 stsp oo += 4*nent; /* Checksums */
776 93658fb9 2020-03-18 stsp oo += 4*idx; /* Offset offset */
777 93658fb9 2020-03-18 stsp if(preadbe32(f, &i, oo) == -1)
779 93658fb9 2020-03-18 stsp o = i & 0xffffffff;
780 93658fb9 2020-03-18 stsp if(o & (1ull << 31)){
781 93658fb9 2020-03-18 stsp o &= 0x7fffffff;
782 93658fb9 2020-03-18 stsp if(preadbe64(f, &o, o) == -1)
788 93658fb9 2020-03-18 stsp fprintf(stderr, "unable to read packfile\n");
792 93658fb9 2020-03-18 stsp char hstr[41];
793 93658fb9 2020-03-18 stsp hashfmt(hstr, sizeof(hstr), &h);
794 93658fb9 2020-03-18 stsp fprintf(stdout, "could not find object %s\n", hstr);
800 93658fb9 2020-03-18 stsp * Scans for non-empty word, copying it into buf.
801 93658fb9 2020-03-18 stsp * Strips off word, leading, and trailing space
802 93658fb9 2020-03-18 stsp * from input.
804 93658fb9 2020-03-18 stsp * Returns -1 on empty string or error, leaving
805 93658fb9 2020-03-18 stsp * input unmodified.
808 93658fb9 2020-03-18 stsp scanword(char **str, int *nstr, char *buf, int nbuf)
816 93658fb9 2020-03-18 stsp while(n && isblank(*p)){
821 93658fb9 2020-03-18 stsp for(; n && *p && !isspace(*p); p++, n--){
823 93658fb9 2020-03-18 stsp *buf++ = *p;
825 93658fb9 2020-03-18 stsp if(nbuf == 0)
828 93658fb9 2020-03-18 stsp while(n && isblank(*p)){
838 93658fb9 2020-03-18 stsp static void
839 93658fb9 2020-03-18 stsp nextline(char **str, int *nstr)
843 93658fb9 2020-03-18 stsp if((s = strchr(*str, '\n')) != NULL){
844 93658fb9 2020-03-18 stsp *nstr -= s - *str + 1;
845 93658fb9 2020-03-18 stsp *str = s + 1;
850 93658fb9 2020-03-18 stsp parseauthor(char **str, int *nstr, char **name, off_t *time)
855 93658fb9 2020-03-18 stsp static void
856 93658fb9 2020-03-18 stsp parsecommit(Object *o)
858 93658fb9 2020-03-18 stsp char *p, *t, buf[128];
861 93658fb9 2020-03-18 stsp p = o->data;
862 93658fb9 2020-03-18 stsp np = o->size;
863 93658fb9 2020-03-18 stsp o->commit = emalloc(sizeof(Cinfo));
865 93658fb9 2020-03-18 stsp if(scanword(&p, &np, buf, sizeof(buf)) == -1)
867 93658fb9 2020-03-18 stsp if(strcmp(buf, "tree") == 0){
868 93658fb9 2020-03-18 stsp if(scanword(&p, &np, buf, sizeof(buf)) == -1)
869 93658fb9 2020-03-18 stsp errx(1, "invalid commit: tree missing");
870 93658fb9 2020-03-18 stsp if(hparse(&o->commit->tree, buf) == -1)
871 93658fb9 2020-03-18 stsp errx(1, "invalid commit: garbled tree");
872 93658fb9 2020-03-18 stsp }else if(strcmp(buf, "parent") == 0){
873 93658fb9 2020-03-18 stsp if(scanword(&p, &np, buf, sizeof(buf)) == -1)
874 93658fb9 2020-03-18 stsp errx(1, "invalid commit: missing parent");
875 93658fb9 2020-03-18 stsp o->commit->parent = realloc(o->commit->parent, ++o->commit->nparent * sizeof(struct got_object_id));
876 93658fb9 2020-03-18 stsp if(!o->commit->parent)
877 93658fb9 2020-03-18 stsp err(1, "unable to malloc: ");
878 93658fb9 2020-03-18 stsp if(hparse(&o->commit->parent[o->commit->nparent - 1], buf) == -1)
879 93658fb9 2020-03-18 stsp errx(1, "invalid commit: garbled parent");
880 93658fb9 2020-03-18 stsp }else if(strcmp(buf, "author") == 0){
881 93658fb9 2020-03-18 stsp parseauthor(&p, &np, &o->commit->author, &o->commit->mtime);
882 93658fb9 2020-03-18 stsp }else if(strcmp(buf, "committer") == 0){
883 93658fb9 2020-03-18 stsp parseauthor(&p, &np, &o->commit->committer, &o->commit->ctime);
884 93658fb9 2020-03-18 stsp }else if(strcmp(buf, "gpgsig") == 0){
885 93658fb9 2020-03-18 stsp /* just drop it */
886 93658fb9 2020-03-18 stsp if((t = strstr(p, "-----END PGP SIGNATURE-----")) == NULL)
887 93658fb9 2020-03-18 stsp errx(1, "malformed gpg signature");
888 93658fb9 2020-03-18 stsp np -= t - p;
891 93658fb9 2020-03-18 stsp nextline(&p, &np);
893 93658fb9 2020-03-18 stsp while (np && isspace(*p)) {
897 93658fb9 2020-03-18 stsp o->commit->msg = p;
898 93658fb9 2020-03-18 stsp o->commit->nmsg = np;
901 93658fb9 2020-03-18 stsp static void
902 93658fb9 2020-03-18 stsp parsetree(Object *o)
904 93658fb9 2020-03-18 stsp char *p, buf[256];
905 93658fb9 2020-03-18 stsp int np, nn, m;
908 93658fb9 2020-03-18 stsp p = o->data;
909 93658fb9 2020-03-18 stsp np = o->size;
910 93658fb9 2020-03-18 stsp o->tree = emalloc(sizeof(Tinfo));
911 93658fb9 2020-03-18 stsp while(np > 0){
912 93658fb9 2020-03-18 stsp if(scanword(&p, &np, buf, sizeof(buf)) == -1)
914 93658fb9 2020-03-18 stsp o->tree->ent = erealloc(o->tree->ent, ++o->tree->nent * sizeof(Dirent));
915 93658fb9 2020-03-18 stsp t = &o->tree->ent[o->tree->nent - 1];
916 93658fb9 2020-03-18 stsp memset(t, 0, sizeof(Dirent));
917 93658fb9 2020-03-18 stsp m = strtol(buf, NULL, 8);
918 93658fb9 2020-03-18 stsp /* FIXME: symlinks and other BS */
919 93658fb9 2020-03-18 stsp if(m == 0160000){
920 93658fb9 2020-03-18 stsp t->mode |= S_IFDIR;
921 93658fb9 2020-03-18 stsp t->modref = 1;
923 93658fb9 2020-03-18 stsp t->mode = m & 0777;
924 93658fb9 2020-03-18 stsp if(m & 0040000)
925 93658fb9 2020-03-18 stsp t->mode |= S_IFDIR;
926 93658fb9 2020-03-18 stsp t->name = p;
927 93658fb9 2020-03-18 stsp nn = strlen(p) + 1;
930 93658fb9 2020-03-18 stsp if(np < sizeof(t->h.sha1))
931 93658fb9 2020-03-18 stsp errx(1, "malformed tree, remaining %d (%s)", np, p);
932 93658fb9 2020-03-18 stsp memcpy(t->h.sha1, p, sizeof(t->h.sha1));
933 93658fb9 2020-03-18 stsp p += sizeof(t->h.sha1);
934 93658fb9 2020-03-18 stsp np -= sizeof(t->h.sha1);
939 93658fb9 2020-03-18 stsp parseobject(Object *o)
941 93658fb9 2020-03-18 stsp if(o->flag & Cparsed)
943 93658fb9 2020-03-18 stsp switch(o->type){
944 93658fb9 2020-03-18 stsp case GTree: parsetree(o); break;
945 93658fb9 2020-03-18 stsp case GCommit: parsecommit(o); break;
946 93658fb9 2020-03-18 stsp //case GTag: parsetag(o); break;
947 93658fb9 2020-03-18 stsp default: break;
949 93658fb9 2020-03-18 stsp o->flag |= Cparsed;
952 93658fb9 2020-03-18 stsp static Object*
953 93658fb9 2020-03-18 stsp readidxobject(FILE *idx, struct got_object_id h, int flag)
955 93658fb9 2020-03-18 stsp char path[Pathmax];
956 93658fb9 2020-03-18 stsp char hbuf[41];
958 93658fb9 2020-03-18 stsp Object *obj;
961 93658fb9 2020-03-18 stsp struct dirent *ent;
965 93658fb9 2020-03-18 stsp if ((obj = got_object_idset_lookup_data(objcache, &h))) {
966 93658fb9 2020-03-18 stsp if(obj->flag & Cloaded)
967 93658fb9 2020-03-18 stsp return obj;
968 93658fb9 2020-03-18 stsp if(obj->flag & Cidx){
969 93658fb9 2020-03-18 stsp assert(idx != NULL);
970 93658fb9 2020-03-18 stsp o = ftello(idx);
971 93658fb9 2020-03-18 stsp if(fseek(idx, obj->off, 0) == -1)
972 93658fb9 2020-03-18 stsp errx(1, "could not seek to object offset");
973 93658fb9 2020-03-18 stsp if(readpacked(idx, obj, flag) == -1)
974 93658fb9 2020-03-18 stsp errx(1, "could not reload object");
975 93658fb9 2020-03-18 stsp if(fseek(idx, o, 0) == -1)
976 93658fb9 2020-03-18 stsp errx(1, "could not restore offset");
977 93658fb9 2020-03-18 stsp cache(obj);
978 93658fb9 2020-03-18 stsp return obj;
983 93658fb9 2020-03-18 stsp /* We're not putting it in the cache yet... */
984 93658fb9 2020-03-18 stsp obj = emalloc(sizeof(Object));
985 93658fb9 2020-03-18 stsp obj->id = next_object_id + 1;
986 93658fb9 2020-03-18 stsp obj->hash = h;
988 93658fb9 2020-03-18 stsp hashfmt(hbuf, sizeof(hbuf), &h);
989 93658fb9 2020-03-18 stsp snprintf(path, sizeof(path), ".git/objects/%c%c/%s", hbuf[0], hbuf[1], hbuf + 2);
990 93658fb9 2020-03-18 stsp if((f = fopen(path, "r")) != NULL){
991 93658fb9 2020-03-18 stsp if(readloose(f, obj, flag) == -1)
992 93658fb9 2020-03-18 stsp goto error;
994 93658fb9 2020-03-18 stsp parseobject(obj);
995 93658fb9 2020-03-18 stsp hashfmt(hbuf, sizeof(hbuf), &obj->hash);
996 93658fb9 2020-03-18 stsp fprintf(stderr, "object %s cached", hbuf);
997 93658fb9 2020-03-18 stsp cache(obj);
998 93658fb9 2020-03-18 stsp return obj;
1002 93658fb9 2020-03-18 stsp if ((d = opendir(".git/objects/pack")) == NULL)
1003 93658fb9 2020-03-18 stsp err(1, "open pack dir");
1004 93658fb9 2020-03-18 stsp while ((ent = readdir(d)) != NULL) {
1005 93658fb9 2020-03-18 stsp l = strlen(ent->d_name);
1006 93658fb9 2020-03-18 stsp if(l > 4 && strcmp(ent->d_name + l - 4, ".idx") != 0)
1008 93658fb9 2020-03-18 stsp snprintf(path, sizeof(path), ".git/objects/pack/%s", ent->d_name);
1009 93658fb9 2020-03-18 stsp if((f = fopen(path, "r")) == NULL)
1011 93658fb9 2020-03-18 stsp o = searchindex(f, h);
1012 93658fb9 2020-03-18 stsp fclose(f);
1013 93658fb9 2020-03-18 stsp if(o == -1)
1017 93658fb9 2020-03-18 stsp closedir(d);
1019 93658fb9 2020-03-18 stsp if (o == -1)
1020 93658fb9 2020-03-18 stsp goto error;
1022 93658fb9 2020-03-18 stsp if((n = snprintf(path, sizeof(path), "%s", path)) >= sizeof(path) - 4)
1023 93658fb9 2020-03-18 stsp goto error;
1024 93658fb9 2020-03-18 stsp memcpy(path + n - 4, ".pack", 6);
1025 93658fb9 2020-03-18 stsp if((f = fopen(path, "r")) == NULL)
1026 93658fb9 2020-03-18 stsp goto error;
1027 93658fb9 2020-03-18 stsp if(fseek(f, o, 0) == -1)
1028 93658fb9 2020-03-18 stsp goto error;
1029 93658fb9 2020-03-18 stsp if(readpacked(f, obj, flag) == -1)
1030 93658fb9 2020-03-18 stsp goto error;
1031 93658fb9 2020-03-18 stsp fclose(f);
1032 93658fb9 2020-03-18 stsp parseobject(obj);
1033 93658fb9 2020-03-18 stsp cache(obj);
1034 93658fb9 2020-03-18 stsp return obj;
1036 93658fb9 2020-03-18 stsp free(obj);
1037 93658fb9 2020-03-18 stsp return NULL;
1041 93658fb9 2020-03-18 stsp readobject(struct got_object_id h)
1043 93658fb9 2020-03-18 stsp Object *o;
1045 93658fb9 2020-03-18 stsp o = readidxobject(NULL, h, 0);
1052 93658fb9 2020-03-18 stsp objcmp(const void *pa, const void *pb)
1054 93658fb9 2020-03-18 stsp Object *a, *b;
1056 93658fb9 2020-03-18 stsp a = *(Object**)pa;
1057 93658fb9 2020-03-18 stsp b = *(Object**)pb;
1058 93658fb9 2020-03-18 stsp return memcmp(a->hash.sha1, b->hash.sha1, sizeof(a->hash.sha1));
1061 93658fb9 2020-03-18 stsp static int
1062 93658fb9 2020-03-18 stsp hwrite(FILE *b, void *buf, int len, SHA1_CTX *ctx)
1064 93658fb9 2020-03-18 stsp SHA1Update(ctx, buf, len);
1065 93658fb9 2020-03-18 stsp return fwrite(buf, 1, len, b);
1068 93658fb9 2020-03-18 stsp static uint32_t
1069 93658fb9 2020-03-18 stsp objectcrc(FILE *f, Object *o)
1071 93658fb9 2020-03-18 stsp char buf[8096];
1074 93658fb9 2020-03-18 stsp o->crc = 0;
1075 93658fb9 2020-03-18 stsp fseek(f, o->off, 0);
1076 93658fb9 2020-03-18 stsp for(n = o->len; n > 0; n -= r){
1077 93658fb9 2020-03-18 stsp r = fread(buf, 1, n > sizeof(buf) ? sizeof(buf) : n, f);
1078 93658fb9 2020-03-18 stsp if(r == -1)
1079 93658fb9 2020-03-18 stsp return -1;
1080 93658fb9 2020-03-18 stsp if(r == 0)
1082 93658fb9 2020-03-18 stsp o->crc = crc32(o->crc, buf, r);
1088 279090e1 2020-03-18 stsp indexpack(int packfd, int idxfd, struct got_object_id *packhash)
1090 93658fb9 2020-03-18 stsp char hdr[4*3], buf[8];
1091 93658fb9 2020-03-18 stsp int nobj, nvalid, nbig, n, i, step;
1092 93658fb9 2020-03-18 stsp Object *o, **objects;
1093 93658fb9 2020-03-18 stsp char *valid;
1094 93658fb9 2020-03-18 stsp SHA1_CTX ctx, objctx;
1096 93658fb9 2020-03-18 stsp struct got_object_id h;
1099 93658fb9 2020-03-18 stsp if ((f = fdopen(packfd, "r")) == NULL)
1100 93658fb9 2020-03-18 stsp return -1;
1101 93658fb9 2020-03-18 stsp if (fseek(f, 0, SEEK_SET) == -1)
1102 93658fb9 2020-03-18 stsp return -1;
1103 93658fb9 2020-03-18 stsp if (fread(hdr, 1, sizeof(hdr), f) != sizeof(hdr)) {
1104 93658fb9 2020-03-18 stsp fprintf(stderr, "short read on header");
1105 93658fb9 2020-03-18 stsp return -1;
1107 93658fb9 2020-03-18 stsp if (memcmp(hdr, "PACK\0\0\0\2", 8) != 0) {
1108 93658fb9 2020-03-18 stsp fprintf(stderr, "invalid header");
1109 93658fb9 2020-03-18 stsp return -1;
1112 93658fb9 2020-03-18 stsp nvalid = 0;
1113 93658fb9 2020-03-18 stsp nobj = GETBE32(hdr + 8);
1114 93658fb9 2020-03-18 stsp objects = calloc(nobj, sizeof(Object*));
1115 93658fb9 2020-03-18 stsp valid = calloc(nobj, sizeof(char));
1116 93658fb9 2020-03-18 stsp step = nobj/100;
1119 93658fb9 2020-03-18 stsp while (nvalid != nobj) {
1120 93658fb9 2020-03-18 stsp fprintf(stderr, "indexing (%d/%d):", nvalid, nobj);
1122 93658fb9 2020-03-18 stsp for (i = 0; i < nobj; i++) {
1123 93658fb9 2020-03-18 stsp if (valid[i]) {
1127 93658fb9 2020-03-18 stsp if (i % step == 0)
1128 93658fb9 2020-03-18 stsp fprintf(stderr, ".");
1129 93658fb9 2020-03-18 stsp if (!objects[i]) {
1130 93658fb9 2020-03-18 stsp o = emalloc(sizeof(Object));
1131 93658fb9 2020-03-18 stsp o->off = ftello(f);
1132 93658fb9 2020-03-18 stsp objects[i] = o;
1134 93658fb9 2020-03-18 stsp o = objects[i];
1135 93658fb9 2020-03-18 stsp fseek(f, o->off, 0);
1136 93658fb9 2020-03-18 stsp if (readpacked(f, o, Cidx) == 0){
1137 93658fb9 2020-03-18 stsp SHA1Init(&objctx);
1138 93658fb9 2020-03-18 stsp SHA1Update(&objctx, (uint8_t*)o->all, o->size + strlen(o->all) + 1);
1139 93658fb9 2020-03-18 stsp SHA1Final(o->hash.sha1, &objctx);
1141 93658fb9 2020-03-18 stsp valid[i] = 1;
1144 93658fb9 2020-03-18 stsp if(objectcrc(f, o) == -1)
1145 93658fb9 2020-03-18 stsp return -1;
1147 93658fb9 2020-03-18 stsp fprintf(stderr, "\n");
1148 93658fb9 2020-03-18 stsp if (n == nvalid) {
1149 93658fb9 2020-03-18 stsp errx(1, "fix point reached too early: %d/%d", nvalid, nobj);
1150 93658fb9 2020-03-18 stsp goto error;
1152 93658fb9 2020-03-18 stsp nvalid = n;
1154 93658fb9 2020-03-18 stsp fclose(f);
1156 93658fb9 2020-03-18 stsp SHA1Init(&ctx);
1157 93658fb9 2020-03-18 stsp qsort(objects, nobj, sizeof(Object*), objcmp);
1158 93658fb9 2020-03-18 stsp if((f = fdopen(idxfd, "w")) == NULL)
1159 93658fb9 2020-03-18 stsp return -1;
1160 93658fb9 2020-03-18 stsp if(hwrite(f, "\xfftOc\x00\x00\x00\x02", 8, &ctx) != 8)
1161 93658fb9 2020-03-18 stsp goto error;
1162 93658fb9 2020-03-18 stsp /* fanout table */
1164 93658fb9 2020-03-18 stsp for(i = 0; i < 256; i++){
1165 93658fb9 2020-03-18 stsp while(c < nobj && (objects[c]->hash.sha1[0] & 0xff) <= i)
1167 93658fb9 2020-03-18 stsp PUTBE32(buf, c);
1168 93658fb9 2020-03-18 stsp hwrite(f, buf, 4, &ctx);
1170 93658fb9 2020-03-18 stsp for(i = 0; i < nobj; i++){
1171 93658fb9 2020-03-18 stsp o = objects[i];
1172 93658fb9 2020-03-18 stsp hwrite(f, o->hash.sha1, sizeof(o->hash.sha1), &ctx);
1175 93658fb9 2020-03-18 stsp /* pointless, nothing uses this */
1176 93658fb9 2020-03-18 stsp for(i = 0; i < nobj; i++){
1177 93658fb9 2020-03-18 stsp PUTBE32(buf, objects[i]->crc);
1178 93658fb9 2020-03-18 stsp hwrite(f, buf, 4, &ctx);
1182 93658fb9 2020-03-18 stsp for(i = 0; i < nobj; i++){
1183 93658fb9 2020-03-18 stsp if(objects[i]->off <= (1ull<<31))
1184 93658fb9 2020-03-18 stsp PUTBE32(buf, objects[i]->off);
1186 93658fb9 2020-03-18 stsp PUTBE32(buf, (1ull << 31) | nbig++);
1187 93658fb9 2020-03-18 stsp hwrite(f, buf, 4, &ctx);
1189 93658fb9 2020-03-18 stsp for(i = 0; i < nobj; i++){
1190 93658fb9 2020-03-18 stsp if(objects[i]->off > (1ull<<31)){
1191 93658fb9 2020-03-18 stsp PUTBE64(buf, objects[i]->off);
1192 93658fb9 2020-03-18 stsp hwrite(f, buf, 8, &ctx);
1195 279090e1 2020-03-18 stsp hwrite(f, packhash->sha1, sizeof(packhash->sha1), &ctx);
1196 93658fb9 2020-03-18 stsp SHA1Final(h.sha1, &ctx);
1197 93658fb9 2020-03-18 stsp fwrite(h.sha1, 1, sizeof(h.sha1), f);
1199 93658fb9 2020-03-18 stsp free(objects);
1200 93658fb9 2020-03-18 stsp free(valid);
1201 93658fb9 2020-03-18 stsp fclose(f);
1205 93658fb9 2020-03-18 stsp free(objects);
1206 93658fb9 2020-03-18 stsp free(valid);
1207 93658fb9 2020-03-18 stsp fclose(f);
1208 93658fb9 2020-03-18 stsp return -1;
1212 93658fb9 2020-03-18 stsp main(int argc, char **argv)
1214 93658fb9 2020-03-18 stsp const struct got_error *err = NULL;
1215 93658fb9 2020-03-18 stsp struct got_object_id packhash;
1216 93658fb9 2020-03-18 stsp struct imsgbuf ibuf;
1217 93658fb9 2020-03-18 stsp struct imsg imsg;
1218 93658fb9 2020-03-18 stsp int packfd, idxfd;
1220 93658fb9 2020-03-18 stsp objcache = got_object_idset_alloc();
1221 93658fb9 2020-03-18 stsp imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1222 93658fb9 2020-03-18 stsp if((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1223 93658fb9 2020-03-18 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
1224 93658fb9 2020-03-18 stsp err = NULL;
1225 93658fb9 2020-03-18 stsp goto done;
1227 93658fb9 2020-03-18 stsp if (imsg.hdr.type == GOT_IMSG_STOP)
1228 93658fb9 2020-03-18 stsp goto done;
1229 93658fb9 2020-03-18 stsp if (imsg.hdr.type != GOT_IMSG_IDXPACK_REQUEST) {
1230 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1231 93658fb9 2020-03-18 stsp goto done;
1233 93658fb9 2020-03-18 stsp if (imsg.hdr.len - IMSG_HEADER_SIZE != SHA1_DIGEST_LENGTH) {
1234 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
1235 93658fb9 2020-03-18 stsp goto done;
1237 93658fb9 2020-03-18 stsp packfd = imsg.fd;
1238 93658fb9 2020-03-18 stsp memcpy(packhash.sha1, imsg.data, SHA1_DIGEST_LENGTH);
1240 93658fb9 2020-03-18 stsp if((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1241 93658fb9 2020-03-18 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
1242 93658fb9 2020-03-18 stsp err = NULL;
1243 93658fb9 2020-03-18 stsp goto done;
1245 93658fb9 2020-03-18 stsp if (imsg.hdr.type == GOT_IMSG_STOP)
1246 93658fb9 2020-03-18 stsp goto done;
1247 93658fb9 2020-03-18 stsp if (imsg.hdr.type != GOT_IMSG_TMPFD) {
1248 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1249 93658fb9 2020-03-18 stsp goto done;
1251 93658fb9 2020-03-18 stsp if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1252 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
1253 93658fb9 2020-03-18 stsp goto done;
1255 93658fb9 2020-03-18 stsp idxfd = imsg.fd;
1257 279090e1 2020-03-18 stsp indexpack(packfd, idxfd, &packhash);
1259 93658fb9 2020-03-18 stsp if(err != NULL)
1260 93658fb9 2020-03-18 stsp got_privsep_send_error(&ibuf, err);
1262 93658fb9 2020-03-18 stsp err = got_privsep_send_index_pack_done(&ibuf);
1263 93658fb9 2020-03-18 stsp if(err != NULL) {
1264 93658fb9 2020-03-18 stsp fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1265 93658fb9 2020-03-18 stsp got_privsep_send_error(&ibuf, err);
