1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <dirent.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 #include <sha1.h>
26 #include <endian.h>
27 #include <limits.h>
28 #include <uuid.h>
29 
30 #include "got_error.h"
31 #include "got_object.h"
32 #include "got_path.h"
33 
34 #include "got_lib_fileindex.h"
35 #include "got_lib_worktree.h"
36 
37 /* got_fileindex_entry flags */
38 #define GOT_FILEIDX_F_PATH_LEN		0x00000fff
39 #define GOT_FILEIDX_F_STAGE		0x00003000
40 #define GOT_FILEIDX_F_EXTENDED		0x00004000
41 #define GOT_FILEIDX_F_ASSUME_VALID	0x00008000
42 #define GOT_FILEIDX_F_NOT_FLUSHED	0x00010000
43 #define GOT_FILEIDX_F_NO_BLOB		0x00020000
44 #define GOT_FILEIDX_F_NO_COMMIT		0x00040000
45 #define GOT_FILEIDX_F_NO_FILE_ON_DISK	0x00080000
46 
47 struct got_fileindex {
48 	struct got_fileindex_tree entries;
49 	int nentries;
50 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
51 };
52 
53 const struct got_error *
54 got_fileindex_entry_update(struct got_fileindex_entry *entry,
55     const char *ondisk_path, uint8_t *blob_sha1, uint8_t *commit_sha1,
56     int update_timestamps)
57 {
58 	struct stat sb;
59 
60 	if (lstat(ondisk_path, &sb) != 0) {
61 		if ((entry->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0)
62 			return got_error_prefix_errno2("lstat", ondisk_path);
63 	} else
64 		entry->flags &= ~GOT_FILEIDX_F_NO_FILE_ON_DISK;
65 
66 	if ((entry->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0) {
67 		if (update_timestamps) {
68 			entry->ctime_sec = sb.st_ctime;
69 			entry->ctime_nsec = sb.st_ctimensec;
70 			entry->mtime_sec = sb.st_mtime;
71 			entry->mtime_nsec = sb.st_mtimensec;
72 		}
73 		entry->uid = sb.st_uid;
74 		entry->gid = sb.st_gid;
75 		entry->size = (sb.st_size & 0xffffffff);
76 		if (sb.st_mode & S_IFLNK)
77 			entry->mode = GOT_FILEIDX_MODE_SYMLINK;
78 		else
79 			entry->mode = GOT_FILEIDX_MODE_REGULAR_FILE;
80 		entry->mode |= ((sb.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) <<
81 		    GOT_FILEIDX_MODE_PERMS_SHIFT);
82 	}
83 
84 	if (blob_sha1) {
85 		memcpy(entry->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
86 		entry->flags &= ~GOT_FILEIDX_F_NO_BLOB;
87 	} else
88 		entry->flags |= GOT_FILEIDX_F_NO_BLOB;
89 
90 	if (commit_sha1) {
91 		memcpy(entry->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
92 		entry->flags &= ~GOT_FILEIDX_F_NO_COMMIT;
93 	} else
94 		entry->flags |= GOT_FILEIDX_F_NO_COMMIT;
95 
96 	return NULL;
97 }
98 
99 void
100 got_fileindex_entry_mark_deleted_from_disk(struct got_fileindex_entry *entry)
101 {
102 	entry->flags |= GOT_FILEIDX_F_NO_FILE_ON_DISK;
103 }
104 
105 const struct got_error *
106 got_fileindex_entry_alloc(struct got_fileindex_entry **entry,
107     const char *ondisk_path, const char *relpath, uint8_t *blob_sha1,
108     uint8_t *commit_sha1)
109 {
110 	size_t len;
111 
112 	*entry = calloc(1, sizeof(**entry));
113 	if (*entry == NULL)
114 		return got_error_prefix_errno("calloc");
115 
116 	(*entry)->path = strdup(relpath);
117 	if ((*entry)->path == NULL) {
118 		const struct got_error *err = got_error_prefix_errno("strdup");
119 		free(*entry);
120 		*entry = NULL;
121 		return err;
122 	}
123 
124 	len = strlen(relpath);
125 	if (len > GOT_FILEIDX_F_PATH_LEN)
126 		len = GOT_FILEIDX_F_PATH_LEN;
127 	(*entry)->flags |= len;
128 
129 	return got_fileindex_entry_update(*entry, ondisk_path, blob_sha1,
130 	    commit_sha1, 1);
131 }
132 
133 void
134 got_fileindex_entry_free(struct got_fileindex_entry *entry)
135 {
136 	free(entry->path);
137 	free(entry);
138 }
139 
140 int
141 got_fileindex_entry_has_blob(struct got_fileindex_entry *ie)
142 {
143 	return (ie->flags & GOT_FILEIDX_F_NO_BLOB) == 0;
144 }
145 
146 int
147 got_fileindex_entry_has_commit(struct got_fileindex_entry *ie)
148 {
149 	return (ie->flags & GOT_FILEIDX_F_NO_COMMIT) == 0;
150 }
151 
152 int
153 got_fileindex_entry_has_file_on_disk(struct got_fileindex_entry *ie)
154 {
155 	return (ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0;
156 }
157 
158 static const struct got_error *
159 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *entry)
160 {
161 	if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
162 		return got_error(GOT_ERR_NO_SPACE);
163 
164 	RB_INSERT(got_fileindex_tree, &fileindex->entries, entry);
165 	fileindex->nentries++;
166 	return NULL;
167 }
168 
169 const struct got_error *
170 got_fileindex_entry_add(struct got_fileindex *fileindex,
171     struct got_fileindex_entry *entry)
172 {
173 	/* Flag this entry until it gets written out to disk. */
174 	entry->flags |= GOT_FILEIDX_F_NOT_FLUSHED;
175 
176 	return add_entry(fileindex, entry);
177 }
178 
179 void
180 got_fileindex_entry_remove(struct got_fileindex *fileindex,
181     struct got_fileindex_entry *entry)
182 {
183 	RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
184 	fileindex->nentries--;
185 }
186 
187 struct got_fileindex_entry *
188 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path)
189 {
190 	struct got_fileindex_entry key;
191 	memset(&key, 0, sizeof(key));
192 	key.path = (char *)path;
193 	return RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
194 }
195 
196 const struct got_error *
197 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
198     got_fileindex_cb cb, void *cb_arg)
199 {
200 	const struct got_error *err;
201 	struct got_fileindex_entry *entry, *tmp;
202 
203 	RB_FOREACH_SAFE(entry, got_fileindex_tree, &fileindex->entries, tmp) {
204 		err = (*cb)(cb_arg, entry);
205 		if (err)
206 			return err;
207 	}
208 	return NULL;
209 }
210 
211 struct got_fileindex *
212 got_fileindex_alloc(void)
213 {
214 	struct got_fileindex *fileindex;
215 
216 	fileindex = calloc(1, sizeof(*fileindex));
217 	if (fileindex == NULL)
218 		return NULL;
219 
220 	RB_INIT(&fileindex->entries);
221 	return fileindex;
222 }
223 
224 void
225 got_fileindex_free(struct got_fileindex *fileindex)
226 {
227 	struct got_fileindex_entry *entry;
228 
229 	while ((entry = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
230 		RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
231 		got_fileindex_entry_free(entry);
232 	}
233 	free(fileindex);
234 }
235 
236 static const struct got_error *
237 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
238 {
239 	size_t n;
240 
241 	val = htobe64(val);
242 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
243 	n = fwrite(&val, 1, sizeof(val), outfile);
244 	if (n != sizeof(val))
245 		return got_ferror(outfile, GOT_ERR_IO);
246 	return NULL;
247 }
248 
249 static const struct got_error *
250 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
251 {
252 	size_t n;
253 
254 	val = htobe32(val);
255 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
256 	n = fwrite(&val, 1, sizeof(val), outfile);
257 	if (n != sizeof(val))
258 		return got_ferror(outfile, GOT_ERR_IO);
259 	return NULL;
260 }
261 
262 static const struct got_error *
263 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
264 {
265 	size_t n;
266 
267 	val = htobe16(val);
268 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
269 	n = fwrite(&val, 1, sizeof(val), outfile);
270 	if (n != sizeof(val))
271 		return got_ferror(outfile, GOT_ERR_IO);
272 	return NULL;
273 }
274 
275 static const struct got_error *
276 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
277 {
278 	size_t n, len, pad = 0;
279 	static const uint8_t zero[8] = { 0 };
280 
281 	len = strlen(path);
282 	while ((len + pad) % 8 != 0)
283 		pad++;
284 	if (pad == 0)
285 		pad = 8; /* NUL-terminate */
286 
287 	SHA1Update(ctx, path, len);
288 	n = fwrite(path, 1, len, outfile);
289 	if (n != len)
290 		return got_ferror(outfile, GOT_ERR_IO);
291 	SHA1Update(ctx, zero, pad);
292 	n = fwrite(zero, 1, pad, outfile);
293 	if (n != pad)
294 		return got_ferror(outfile, GOT_ERR_IO);
295 	return NULL;
296 }
297 
298 static const struct got_error *
299 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *entry,
300     FILE *outfile)
301 {
302 	const struct got_error *err;
303 	size_t n;
304 
305 	err = write_fileindex_val64(ctx, entry->ctime_sec, outfile);
306 	if (err)
307 		return err;
308 	err = write_fileindex_val64(ctx, entry->ctime_nsec, outfile);
309 	if (err)
310 		return err;
311 	err = write_fileindex_val64(ctx, entry->mtime_sec, outfile);
312 	if (err)
313 		return err;
314 	err = write_fileindex_val64(ctx, entry->mtime_nsec, outfile);
315 	if (err)
316 		return err;
317 
318 	err = write_fileindex_val32(ctx, entry->uid, outfile);
319 	if (err)
320 		return err;
321 	err = write_fileindex_val32(ctx, entry->gid, outfile);
322 	if (err)
323 		return err;
324 	err = write_fileindex_val32(ctx, entry->size, outfile);
325 	if (err)
326 		return err;
327 
328 	err = write_fileindex_val16(ctx, entry->mode, outfile);
329 	if (err)
330 		return err;
331 
332 	SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
333 	n = fwrite(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
334 	if (n != SHA1_DIGEST_LENGTH)
335 		return got_ferror(outfile, GOT_ERR_IO);
336 
337 	SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
338 	n = fwrite(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
339 	if (n != SHA1_DIGEST_LENGTH)
340 		return got_ferror(outfile, GOT_ERR_IO);
341 
342 	err = write_fileindex_val32(ctx, entry->flags, outfile);
343 	if (err)
344 		return err;
345 
346 	err = write_fileindex_path(ctx, entry->path, outfile);
347 	return err;
348 }
349 
350 const struct got_error *
351 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
352 {
353 	const struct got_error *err = NULL;
354 	struct got_fileindex_hdr hdr;
355 	SHA1_CTX ctx;
356 	uint8_t sha1[SHA1_DIGEST_LENGTH];
357 	size_t n;
358 	struct got_fileindex_entry *entry;
359 
360 	SHA1Init(&ctx);
361 
362 	hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
363 	hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
364 	hdr.nentries = htobe32(fileindex->nentries);
365 
366 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
367 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
368 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
369 	n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
370 	if (n != sizeof(hdr.signature))
371 		return got_ferror(outfile, GOT_ERR_IO);
372 	n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
373 	if (n != sizeof(hdr.version))
374 		return got_ferror(outfile, GOT_ERR_IO);
375 	n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
376 	if (n != sizeof(hdr.nentries))
377 		return got_ferror(outfile, GOT_ERR_IO);
378 
379 	RB_FOREACH(entry, got_fileindex_tree, &fileindex->entries) {
380 		entry->flags &= ~GOT_FILEIDX_F_NOT_FLUSHED;
381 		err = write_fileindex_entry(&ctx, entry, outfile);
382 		if (err)
383 			return err;
384 	}
385 
386 	SHA1Final(sha1, &ctx);
387 	n = fwrite(sha1, 1, sizeof(sha1), outfile);
388 	if (n != sizeof(sha1))
389 		return got_ferror(outfile, GOT_ERR_IO);
390 
391 	if (fflush(outfile) != 0)
392 		return got_error_prefix_errno("fflush");
393 
394 	return NULL;
395 }
396 
397 static const struct got_error *
398 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
399 {
400 	size_t n;
401 
402 	n = fread(val, 1, sizeof(*val), infile);
403 	if (n != sizeof(*val))
404 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
405 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
406 	*val = be64toh(*val);
407 	return NULL;
408 }
409 
410 static const struct got_error *
411 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
412 {
413 	size_t n;
414 
415 	n = fread(val, 1, sizeof(*val), infile);
416 	if (n != sizeof(*val))
417 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
418 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
419 	*val = be32toh(*val);
420 	return NULL;
421 }
422 
423 static const struct got_error *
424 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
425 {
426 	size_t n;
427 
428 	n = fread(val, 1, sizeof(*val), infile);
429 	if (n != sizeof(*val))
430 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
431 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
432 	*val = be16toh(*val);
433 	return NULL;
434 }
435 
436 static const struct got_error *
437 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
438 {
439 	const struct got_error *err = NULL;
440 	uint8_t buf[8];
441 	size_t n, len = 0, totlen = sizeof(buf);
442 
443 	*path = malloc(totlen);
444 	if (*path == NULL)
445 		return got_error_prefix_errno("malloc");
446 
447 	do {
448 		n = fread(buf, 1, sizeof(buf), infile);
449 		if (n != sizeof(buf))
450 			return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
451 		if (len + sizeof(buf) > totlen) {
452 			char *p = reallocarray(*path, totlen + sizeof(buf), 1);
453 			if (p == NULL) {
454 				err = got_error_prefix_errno("reallocarray");
455 				break;
456 			}
457 			totlen += sizeof(buf);
458 			*path = p;
459 		}
460 		SHA1Update(ctx, buf, sizeof(buf));
461 		memcpy(*path + len, buf, sizeof(buf));
462 		len += sizeof(buf);
463 	} while (memchr(buf, '\0', sizeof(buf)) == NULL);
464 
465 	if (err) {
466 		free(*path);
467 		*path = NULL;
468 	}
469 	return err;
470 }
471 
472 static const struct got_error *
473 read_fileindex_entry(struct got_fileindex_entry **entryp, SHA1_CTX *ctx,
474     FILE *infile)
475 {
476 	const struct got_error *err;
477 	struct got_fileindex_entry *entry;
478 	size_t n;
479 
480 	*entryp = NULL;
481 
482 	entry = calloc(1, sizeof(*entry));
483 	if (entry == NULL)
484 		return got_error_prefix_errno("calloc");
485 
486 	err = read_fileindex_val64(&entry->ctime_sec, ctx, infile);
487 	if (err)
488 		goto done;
489 	err = read_fileindex_val64(&entry->ctime_nsec, ctx, infile);
490 	if (err)
491 		goto done;
492 	err = read_fileindex_val64(&entry->mtime_sec, ctx, infile);
493 	if (err)
494 		goto done;
495 	err = read_fileindex_val64(&entry->mtime_nsec, ctx, infile);
496 	if (err)
497 		goto done;
498 
499 	err = read_fileindex_val32(&entry->uid, ctx, infile);
500 	if (err)
501 		goto done;
502 	err = read_fileindex_val32(&entry->gid, ctx, infile);
503 	if (err)
504 		goto done;
505 	err = read_fileindex_val32(&entry->size, ctx, infile);
506 	if (err)
507 		goto done;
508 
509 	err = read_fileindex_val16(&entry->mode, ctx, infile);
510 	if (err)
511 		goto done;
512 
513 	n = fread(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
514 	if (n != SHA1_DIGEST_LENGTH) {
515 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
516 		goto done;
517 	}
518 	SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
519 
520 	n = fread(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
521 	if (n != SHA1_DIGEST_LENGTH) {
522 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
523 		goto done;
524 	}
525 	SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
526 
527 	err = read_fileindex_val32(&entry->flags, ctx, infile);
528 	if (err)
529 		goto done;
530 
531 	err = read_fileindex_path(&entry->path, ctx, infile);
532 done:
533 	if (err)
534 		free(entry);
535 	else
536 		*entryp = entry;
537 	return err;
538 }
539 
540 const struct got_error *
541 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
542 {
543 	const struct got_error *err = NULL;
544 	struct got_fileindex_hdr hdr;
545 	SHA1_CTX ctx;
546 	struct got_fileindex_entry *entry;
547 	uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
548 	uint8_t sha1[SHA1_DIGEST_LENGTH];
549 	size_t n;
550 	int i;
551 
552 	SHA1Init(&ctx);
553 
554 	n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
555 	if (n != sizeof(hdr.signature)) {
556 		if (n == 0) /* EOF */
557 			return NULL;
558 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
559 	}
560 	n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
561 	if (n != sizeof(hdr.version)) {
562 		if (n == 0) /* EOF */
563 			return NULL;
564 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
565 	}
566 	n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
567 	if (n != sizeof(hdr.nentries)) {
568 		if (n == 0) /* EOF */
569 			return NULL;
570 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
571 	}
572 
573 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
574 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
575 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
576 
577 	hdr.signature = be32toh(hdr.signature);
578 	hdr.version = be32toh(hdr.version);
579 	hdr.nentries = be32toh(hdr.nentries);
580 
581 	if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
582 		return got_error(GOT_ERR_FILEIDX_SIG);
583 	if (hdr.version != GOT_FILE_INDEX_VERSION)
584 		return got_error(GOT_ERR_FILEIDX_VER);
585 
586 	for (i = 0; i < hdr.nentries; i++) {
587 		err = read_fileindex_entry(&entry, &ctx, infile);
588 		if (err)
589 			return err;
590 		err = add_entry(fileindex, entry);
591 		if (err)
592 			return err;
593 	}
594 
595 	n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
596 	if (n != sizeof(sha1_expected))
597 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
598 	SHA1Final(sha1, &ctx);
599 	if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
600 		return got_error(GOT_ERR_FILEIDX_CSUM);
601 
602 	return NULL;
603 }
604 
605 static struct got_fileindex_entry *
606 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
607 {
608 	struct got_fileindex_entry *next;
609 
610 	next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
611 
612 	/* Skip entries which were newly added by diff callbacks. */
613 	while (next && (next->flags & GOT_FILEIDX_F_NOT_FLUSHED))
614 		next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
615 
616 	return next;
617 }
618 
619 static const struct got_error *
620 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **,
621     struct got_tree_object *, const char *, const char *,
622     struct got_repository *, struct got_fileindex_diff_tree_cb *, void *);
623 
624 static const struct got_error *
625 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
626     struct got_fileindex_entry **ie, struct got_tree_entry *te,
627     const char *path, const char *entry_name, struct got_repository *repo,
628     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
629 {
630 	const struct got_error *err = NULL;
631 
632 	if (S_ISDIR(te->mode)) {
633 		char *subpath;
634 		struct got_tree_object *subtree;
635 
636 		if (asprintf(&subpath, "%s%s%s", path,
637 		    path[0] == '\0' ? "" : "/", te->name) == -1)
638 			return got_error_prefix_errno("asprintf");
639 
640 		err = got_object_open_as_tree(&subtree, repo, te->id);
641 		if (err) {
642 			free(subpath);
643 			return err;
644 		}
645 
646 		err = diff_fileindex_tree(fileindex, ie, subtree,
647 		    subpath, entry_name, repo, cb, cb_arg);
648 		free(subpath);
649 		got_object_tree_close(subtree);
650 		if (err)
651 			return err;
652 	}
653 
654 	*next = SIMPLEQ_NEXT(te, entry);
655 	return NULL;
656 }
657 
658 static const struct got_error *
659 diff_fileindex_tree(struct got_fileindex *fileindex,
660     struct got_fileindex_entry **ie, struct got_tree_object *tree,
661     const char *path, const char *entry_name, struct got_repository *repo,
662     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
663 {
664 	const struct got_error *err = NULL;
665 	struct got_tree_entry *te = NULL;
666 	size_t path_len = strlen(path);
667 	const struct got_tree_entries *entries;
668 	struct got_fileindex_entry *next;
669 
670 	entries = got_object_tree_get_entries(tree);
671 	te = SIMPLEQ_FIRST(&entries->head);
672 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te) {
673 		if (te && *ie) {
674 			char *te_path;
675 			int cmp;
676 			if (asprintf(&te_path, "%s/%s", path, te->name) == -1) {
677 				err = got_error_prefix_errno("asprintf");
678 				break;
679 			}
680 			cmp = got_path_cmp((*ie)->path, te_path);
681 			free(te_path);
682 			if (cmp == 0) {
683 				if (got_path_is_child((*ie)->path, path,
684 				    path_len) && (entry_name == NULL ||
685 				    strcmp(te->name, entry_name) == 0)) {
686 					err = cb->diff_old_new(cb_arg, *ie, te,
687 					    path);
688 					if (err || entry_name)
689 						break;
690 				}
691 				*ie = walk_fileindex(fileindex, *ie);
692 				err = walk_tree(&te, fileindex, ie, te,
693 				    path, entry_name, repo, cb, cb_arg);
694 			} else if (cmp < 0) {
695 				next = walk_fileindex(fileindex, *ie);
696 				if (got_path_is_child((*ie)->path, path,
697 				    path_len) && (entry_name == NULL ||
698 				    strcmp(te->name, entry_name) == 0)) {
699 					err = cb->diff_old(cb_arg, *ie, path);
700 					if (err || entry_name)
701 						break;
702 				}
703 				*ie = next;
704 			} else {
705 				if ((entry_name == NULL ||
706 				    strcmp(te->name, entry_name) == 0)) {
707 					err = cb->diff_new(cb_arg, te, path);
708 					if (err || entry_name)
709 						break;
710 				}
711 				err = walk_tree(&te, fileindex, ie, te,
712 				    path, entry_name, repo, cb, cb_arg);
713 			}
714 			if (err)
715 				break;
716 		} else if (*ie) {
717 			next = walk_fileindex(fileindex, *ie);
718 			if (got_path_is_child((*ie)->path, path, path_len) &&
719 			    (entry_name == NULL ||
720 			    strcmp(te->name, entry_name) == 0)) {
721 				err = cb->diff_old(cb_arg, *ie, path);
722 				if (err || entry_name)
723 					break;
724 			}
725 			*ie = next;
726 		} else if (te) {
727 			if (entry_name == NULL ||
728 			    strcmp(te->name, entry_name) == 0) {
729 				err = cb->diff_new(cb_arg, te, path);
730 				if (err || entry_name)
731 					break;
732 			}
733 			err = walk_tree(&te, fileindex, ie, te, path,
734 			    entry_name, repo, cb, cb_arg);
735 			if (err)
736 				break;
737 		}
738 	}
739 
740 	return err;
741 }
742 
743 const struct got_error *
744 got_fileindex_diff_tree(struct got_fileindex *fileindex,
745     struct got_tree_object *tree, const char *path, const char *entry_name,
746     struct got_repository *repo,
747     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
748 {
749 	struct got_fileindex_entry *min;
750 	min = RB_MIN(got_fileindex_tree, &fileindex->entries);
751 	return diff_fileindex_tree(fileindex, &min, tree, path, entry_name,
752 	    repo, cb, cb_arg);
753 }
754 
755 static const struct got_error *
756 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **, DIR *,
757     const char *, const char *, struct got_repository *,
758     struct got_fileindex_diff_dir_cb *, void *);
759 
760 static const struct got_error *
761 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
762     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle,
763     const char *path, DIR *dir, const char *rootpath,
764     struct got_repository *repo, struct got_fileindex_diff_dir_cb *cb,
765     void *cb_arg)
766 {
767 	const struct got_error *err = NULL;
768 	struct dirent *de = dle->data;
769 
770 	*next = NULL;
771 
772 	if (de->d_type == DT_DIR) {
773 		char *subpath;
774 		char *subdirpath;
775 		DIR *subdir;
776 
777 		if (asprintf(&subpath, "%s%s%s", path,
778 		    path[0] == '\0' ? "" : "/", de->d_name) == -1)
779 			return got_error_prefix_errno("asprintf");
780 
781 		if (asprintf(&subdirpath, "%s/%s", rootpath, subpath) == -1) {
782 			free(subpath);
783 			return got_error_prefix_errno("asprintf");
784 		}
785 
786 		subdir = opendir(subdirpath);
787 		if (subdir == NULL) {
788 			free(subpath);
789 			free(subdirpath);
790 			return got_error_prefix_errno2("opendir", subdirpath);
791 		}
792 
793 		err = diff_fileindex_dir(fileindex, ie, subdir, rootpath,
794 		    subpath, repo, cb, cb_arg);
795 		free(subpath);
796 		free(subdirpath);
797 		closedir(subdir);
798 		if (err)
799 			return err;
800 	}
801 
802 	*next = TAILQ_NEXT(dle, entry);
803 	return NULL;
804 }
805 
806 static const struct got_error *
807 diff_fileindex_dir(struct got_fileindex *fileindex,
808     struct got_fileindex_entry **ie, DIR *dir, const char *rootpath,
809     const char *path, struct got_repository *repo,
810     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
811 {
812 	const struct got_error *err = NULL;
813 	struct dirent *de = NULL;
814 	size_t path_len = strlen(path);
815 	struct got_fileindex_entry *next;
816 	struct got_pathlist_head dirlist;
817 	struct got_pathlist_entry *dle;
818 
819 	TAILQ_INIT(&dirlist);
820 
821 	while (1) {
822 		struct got_pathlist_entry *new = NULL;
823 		struct dirent *dep = NULL;
824 
825 		de = malloc(sizeof(struct dirent) + NAME_MAX + 1);
826 		if (de == NULL) {
827 			err = got_error_prefix_errno("malloc");
828 			goto done;
829 		}
830 
831 		if (readdir_r(dir, de, &dep) != 0) {
832 			err = got_error_prefix_errno("readdir_r");
833 			free(de);
834 			goto done;
835 		}
836 		if (dep == NULL) {
837 			free(de);
838 			break;
839 		}
840 
841 		if (strcmp(de->d_name, ".") == 0 ||
842 		    strcmp(de->d_name, "..") == 0 ||
843 		    (path[0] == '\0' &&
844 		    strcmp(de->d_name, GOT_WORKTREE_GOT_DIR) == 0)) {
845 			free(de);
846 			continue;
847 		}
848 
849 		err = got_pathlist_insert(&new, &dirlist, de->d_name, de);
850 		if (err) {
851 			free(de);
852 			goto done;
853 		}
854 		if (new == NULL) {
855 			err = got_error(GOT_ERR_DIR_DUP_ENTRY);
856 			free(de);
857 			goto done;
858 		}
859 	}
860 
861 	dle = TAILQ_FIRST(&dirlist);
862 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || dle) {
863 		if (dle && *ie) {
864 			char *de_path;
865 			int cmp;
866 			de = dle->data;
867 			if (asprintf(&de_path, "%s/%s", path,
868 			    de->d_name) == -1) {
869 				err = got_error_prefix_errno("asprintf");
870 				break;
871 			}
872 			cmp = got_path_cmp((*ie)->path, de_path);
873 			free(de_path);
874 			if (cmp == 0) {
875 				err = cb->diff_old_new(cb_arg, *ie, de, path);
876 				if (err)
877 					break;
878 				*ie = walk_fileindex(fileindex, *ie);
879 				err = walk_dir(&dle, fileindex, ie, dle, path,
880 				    dir, rootpath, repo, cb, cb_arg);
881 			} else if (cmp < 0 ) {
882 				next = walk_fileindex(fileindex, *ie);
883 				err = cb->diff_old(cb_arg, *ie, path);
884 				if (err)
885 					break;
886 				*ie = next;
887 			} else {
888 				err = cb->diff_new(cb_arg, de, path);
889 				if (err)
890 					break;
891 				err = walk_dir(&dle, fileindex, ie, dle, path,
892 				    dir, rootpath, repo, cb, cb_arg);
893 			}
894 			if (err)
895 				break;
896 		} else if (*ie) {
897 			next = walk_fileindex(fileindex, *ie);
898 			err = cb->diff_old(cb_arg, *ie, path);
899 			if (err)
900 				break;
901 			*ie = next;
902 		} else if (dle) {
903 			de = dle->data;
904 			err = cb->diff_new(cb_arg, de, path);
905 			if (err)
906 				break;
907 			err = walk_dir(&dle, fileindex, ie, dle, path, dir,
908 			    rootpath, repo, cb, cb_arg);
909 			if (err)
910 				break;
911 		}
912 	}
913 done:
914 	TAILQ_FOREACH(dle, &dirlist, entry)
915 		free(dle->data);
916 	got_pathlist_free(&dirlist);
917 	return err;
918 }
919 
920 const struct got_error *
921 got_fileindex_diff_dir(struct got_fileindex *fileindex, DIR *rootdir,
922     const char *rootpath, const char *path, struct got_repository *repo,
923     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
924 {
925 	struct got_fileindex_entry *min;
926 	min = RB_MIN(got_fileindex_tree, &fileindex->entries);
927 	return diff_fileindex_dir(fileindex, &min, rootdir, rootpath, path,
928 	    repo, cb, cb_arg);
929 }
930 
931 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);