Gotweb

Blob

Date:: Wed Mar 27 11:33:41 2019 UTC
Message:: test and improve behaviour for wt edit vs repo rm conflict
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <dirent.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 #include <sha1.h>
26 #include <endian.h>
27 #include <limits.h>
28 #include <uuid.h>
29 
30 #include "got_error.h"
31 #include "got_object.h"
32 
33 #include "got_lib_path.h"
34 #include "got_lib_fileindex.h"
35 #include "got_lib_worktree.h"
36 
37 /* got_fileindex_entry flags */
38 #define GOT_FILEIDX_F_PATH_LEN		0x00000fff
39 #define GOT_FILEIDX_F_STAGE		0x00003000
40 #define GOT_FILEIDX_F_EXTENDED		0x00004000
41 #define GOT_FILEIDX_F_ASSUME_VALID	0x00008000
42 #define GOT_FILEIDX_F_NOT_FLUSHED	0x00010000
43 #define GOT_FILEIDX_F_NO_BLOB		0x00020000
44 #define GOT_FILEIDX_F_NO_COMMIT		0x00040000
45 #define GOT_FILEIDX_F_NO_FILE_ON_DISK	0x00080000
46 
47 struct got_fileindex {
48 	struct got_fileindex_tree entries;
49 	int nentries;
50 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
51 };
52 
53 const struct got_error *
54 got_fileindex_entry_update(struct got_fileindex_entry *entry,
55     const char *ondisk_path, uint8_t *blob_sha1, uint8_t *commit_sha1,
56     int update_timestamps)
57 {
58 	struct stat sb;
59 
60 	if (lstat(ondisk_path, &sb) != 0) {
61 		if ((entry->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0)
62 			return got_error_from_errno();
63 	} else
64 		entry->flags &= ~GOT_FILEIDX_F_NO_FILE_ON_DISK;
65 
66 	if ((entry->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0) {
67 		if (update_timestamps) {
68 			entry->ctime_sec = sb.st_ctime;
69 			entry->ctime_nsec = sb.st_ctimensec;
70 			entry->mtime_sec = sb.st_mtime;
71 			entry->mtime_nsec = sb.st_mtimensec;
72 		}
73 		entry->uid = sb.st_uid;
74 		entry->gid = sb.st_gid;
75 		entry->size = (sb.st_size & 0xffffffff);
76 		if (sb.st_mode & S_IFLNK)
77 			entry->mode = GOT_FILEIDX_MODE_SYMLINK;
78 		else
79 			entry->mode = GOT_FILEIDX_MODE_REGULAR_FILE;
80 		entry->mode |= ((sb.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) <<
81 		    GOT_FILEIDX_MODE_PERMS_SHIFT);
82 	}
83 
84 	if (blob_sha1) {
85 		memcpy(entry->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
86 		entry->flags &= ~GOT_FILEIDX_F_NO_BLOB;
87 	} else
88 		entry->flags |= GOT_FILEIDX_F_NO_BLOB;
89 
90 	if (commit_sha1) {
91 		memcpy(entry->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
92 		entry->flags &= ~GOT_FILEIDX_F_NO_COMMIT;
93 	} else
94 		entry->flags |= GOT_FILEIDX_F_NO_COMMIT;
95 
96 	return NULL;
97 }
98 
99 void
100 got_fileindex_entry_mark_deleted_from_disk(struct got_fileindex_entry *entry)
101 {
102 	entry->flags |= GOT_FILEIDX_F_NO_FILE_ON_DISK;
103 }
104 
105 const struct got_error *
106 got_fileindex_entry_alloc(struct got_fileindex_entry **entry,
107     const char *ondisk_path, const char *relpath, uint8_t *blob_sha1,
108     uint8_t *commit_sha1)
109 {
110 	size_t len;
111 
112 	*entry = calloc(1, sizeof(**entry));
113 	if (*entry == NULL)
114 		return got_error_from_errno();
115 
116 	(*entry)->path = strdup(relpath);
117 	if ((*entry)->path == NULL) {
118 		const struct got_error *err = got_error_from_errno();
119 		free(*entry);
120 		*entry = NULL;
121 		return err;
122 	}
123 
124 	len = strlen(relpath);
125 	if (len > GOT_FILEIDX_F_PATH_LEN)
126 		len = GOT_FILEIDX_F_PATH_LEN;
127 	(*entry)->flags |= len;
128 
129 	return got_fileindex_entry_update(*entry, ondisk_path, blob_sha1,
130 	    commit_sha1, 1);
131 }
132 
133 void
134 got_fileindex_entry_free(struct got_fileindex_entry *entry)
135 {
136 	free(entry->path);
137 	free(entry);
138 }
139 
140 int
141 got_fileindex_entry_has_blob(struct got_fileindex_entry *ie)
142 {
143 	return (ie->flags & GOT_FILEIDX_F_NO_BLOB) == 0;
144 }
145 
146 int
147 got_fileindex_entry_has_commit(struct got_fileindex_entry *ie)
148 {
149 	return (ie->flags & GOT_FILEIDX_F_NO_COMMIT) == 0;
150 }
151 
152 int
153 got_fileindex_entry_has_file_on_disk(struct got_fileindex_entry *ie)
154 {
155 	return (ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0;
156 }
157 
158 static const struct got_error *
159 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *entry)
160 {
161 	if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
162 		return got_error(GOT_ERR_NO_SPACE);
163 
164 	RB_INSERT(got_fileindex_tree, &fileindex->entries, entry);
165 	fileindex->nentries++;
166 	return NULL;
167 }
168 
169 const struct got_error *
170 got_fileindex_entry_add(struct got_fileindex *fileindex,
171     struct got_fileindex_entry *entry)
172 {
173 	/* Flag this entry until it gets written out to disk. */
174 	entry->flags |= GOT_FILEIDX_F_NOT_FLUSHED;
175 
176 	return add_entry(fileindex, entry);
177 }
178 
179 void
180 got_fileindex_entry_remove(struct got_fileindex *fileindex,
181     struct got_fileindex_entry *entry)
182 {
183 	RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
184 	fileindex->nentries--;
185 }
186 
187 struct got_fileindex_entry *
188 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path)
189 {
190 	struct got_fileindex_entry key;
191 	memset(&key, 0, sizeof(key));
192 	key.path = (char *)path;
193 	return RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
194 }
195 
196 const struct got_error *
197 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
198     got_fileindex_cb cb, void *cb_arg)
199 {
200 	const struct got_error *err;
201 	struct got_fileindex_entry *entry, *tmp;
202 
203 	RB_FOREACH_SAFE(entry, got_fileindex_tree, &fileindex->entries, tmp) {
204 		err = (*cb)(cb_arg, entry);
205 		if (err)
206 			return err;
207 	}
208 	return NULL;
209 }
210 
211 struct got_fileindex *
212 got_fileindex_alloc(void)
213 {
214 	struct got_fileindex *fileindex;
215 
216 	fileindex = calloc(1, sizeof(*fileindex));
217 	if (fileindex == NULL)
218 		return NULL;
219 
220 	RB_INIT(&fileindex->entries);
221 	return fileindex;
222 }
223 
224 void
225 got_fileindex_free(struct got_fileindex *fileindex)
226 {
227 	struct got_fileindex_entry *entry;
228 
229 	while ((entry = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
230 		RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
231 		got_fileindex_entry_free(entry);
232 	}
233 	free(fileindex);
234 }
235 
236 static const struct got_error *
237 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
238 {
239 	size_t n;
240 
241 	val = htobe64(val);
242 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
243 	n = fwrite(&val, 1, sizeof(val), outfile);
244 	if (n != sizeof(val))
245 		return got_ferror(outfile, GOT_ERR_IO);
246 	return NULL;
247 }
248 
249 static const struct got_error *
250 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
251 {
252 	size_t n;
253 
254 	val = htobe32(val);
255 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
256 	n = fwrite(&val, 1, sizeof(val), outfile);
257 	if (n != sizeof(val))
258 		return got_ferror(outfile, GOT_ERR_IO);
259 	return NULL;
260 }
261 
262 static const struct got_error *
263 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
264 {
265 	size_t n;
266 
267 	val = htobe16(val);
268 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
269 	n = fwrite(&val, 1, sizeof(val), outfile);
270 	if (n != sizeof(val))
271 		return got_ferror(outfile, GOT_ERR_IO);
272 	return NULL;
273 }
274 
275 static const struct got_error *
276 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
277 {
278 	size_t n, len, pad = 0;
279 	static const uint8_t zero[8] = { 0 };
280 
281 	len = strlen(path);
282 	while ((len + pad) % 8 != 0)
283 		pad++;
284 	if (pad == 0)
285 		pad = 8; /* NUL-terminate */
286 
287 	SHA1Update(ctx, path, len);
288 	n = fwrite(path, 1, len, outfile);
289 	if (n != len)
290 		return got_ferror(outfile, GOT_ERR_IO);
291 	SHA1Update(ctx, zero, pad);
292 	n = fwrite(zero, 1, pad, outfile);
293 	if (n != pad)
294 		return got_ferror(outfile, GOT_ERR_IO);
295 	return NULL;
296 }
297 
298 static const struct got_error *
299 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *entry,
300     FILE *outfile)
301 {
302 	const struct got_error *err;
303 	size_t n;
304 
305 	err = write_fileindex_val64(ctx, entry->ctime_sec, outfile);
306 	if (err)
307 		return err;
308 	err = write_fileindex_val64(ctx, entry->ctime_nsec, outfile);
309 	if (err)
310 		return err;
311 	err = write_fileindex_val64(ctx, entry->mtime_sec, outfile);
312 	if (err)
313 		return err;
314 	err = write_fileindex_val64(ctx, entry->mtime_nsec, outfile);
315 	if (err)
316 		return err;
317 
318 	err = write_fileindex_val32(ctx, entry->uid, outfile);
319 	if (err)
320 		return err;
321 	err = write_fileindex_val32(ctx, entry->gid, outfile);
322 	if (err)
323 		return err;
324 	err = write_fileindex_val32(ctx, entry->size, outfile);
325 	if (err)
326 		return err;
327 
328 	err = write_fileindex_val16(ctx, entry->mode, outfile);
329 	if (err)
330 		return err;
331 
332 	SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
333 	n = fwrite(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
334 	if (n != SHA1_DIGEST_LENGTH)
335 		return got_ferror(outfile, GOT_ERR_IO);
336 
337 	SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
338 	n = fwrite(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
339 	if (n != SHA1_DIGEST_LENGTH)
340 		return got_ferror(outfile, GOT_ERR_IO);
341 
342 	err = write_fileindex_val32(ctx, entry->flags, outfile);
343 	if (err)
344 		return err;
345 
346 	err = write_fileindex_path(ctx, entry->path, outfile);
347 	return err;
348 }
349 
350 const struct got_error *
351 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
352 {
353 	const struct got_error *err = NULL;
354 	struct got_fileindex_hdr hdr;
355 	SHA1_CTX ctx;
356 	uint8_t sha1[SHA1_DIGEST_LENGTH];
357 	size_t n;
358 	struct got_fileindex_entry *entry;
359 
360 	SHA1Init(&ctx);
361 
362 	hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
363 	hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
364 	hdr.nentries = htobe32(fileindex->nentries);
365 
366 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
367 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
368 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
369 	n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
370 	if (n != sizeof(hdr.signature))
371 		return got_ferror(outfile, GOT_ERR_IO);
372 	n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
373 	if (n != sizeof(hdr.version))
374 		return got_ferror(outfile, GOT_ERR_IO);
375 	n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
376 	if (n != sizeof(hdr.nentries))
377 		return got_ferror(outfile, GOT_ERR_IO);
378 
379 	RB_FOREACH(entry, got_fileindex_tree, &fileindex->entries) {
380 		entry->flags &= ~GOT_FILEIDX_F_NOT_FLUSHED;
381 		err = write_fileindex_entry(&ctx, entry, outfile);
382 		if (err)
383 			return err;
384 	}
385 
386 	SHA1Final(sha1, &ctx);
387 	n = fwrite(sha1, 1, sizeof(sha1), outfile);
388 	if (n != sizeof(sha1))
389 		return got_ferror(outfile, GOT_ERR_IO);
390 
391 	if (fflush(outfile) != 0)
392 		return got_error_from_errno();
393 
394 	return NULL;
395 }
396 
397 static const struct got_error *
398 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
399 {
400 	size_t n;
401 
402 	n = fread(val, 1, sizeof(*val), infile);
403 	if (n != sizeof(*val))
404 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
405 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
406 	*val = be64toh(*val);
407 	return NULL;
408 }
409 
410 static const struct got_error *
411 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
412 {
413 	size_t n;
414 
415 	n = fread(val, 1, sizeof(*val), infile);
416 	if (n != sizeof(*val))
417 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
418 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
419 	*val = be32toh(*val);
420 	return NULL;
421 }
422 
423 static const struct got_error *
424 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
425 {
426 	size_t n;
427 
428 	n = fread(val, 1, sizeof(*val), infile);
429 	if (n != sizeof(*val))
430 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
431 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
432 	*val = be16toh(*val);
433 	return NULL;
434 }
435 
436 static const struct got_error *
437 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
438 {
439 	const struct got_error *err = NULL;
440 	uint8_t buf[8];
441 	size_t n, len = 0, totlen = sizeof(buf);
442 
443 	*path = malloc(totlen);
444 	if (*path == NULL)
445 		return got_error_from_errno();
446 
447 	do {
448 		n = fread(buf, 1, sizeof(buf), infile);
449 		if (n != sizeof(buf))
450 			return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
451 		if (len + sizeof(buf) > totlen) {
452 			char *p = reallocarray(*path, totlen + sizeof(buf), 1);
453 			if (p == NULL) {
454 				err = got_error_from_errno();
455 				break;
456 			}
457 			totlen += sizeof(buf);
458 			*path = p;
459 		}
460 		SHA1Update(ctx, buf, sizeof(buf));
461 		memcpy(*path + len, buf, sizeof(buf));
462 		len += sizeof(buf);
463 	} while (memchr(buf, '\0', sizeof(buf)) == NULL);
464 
465 	if (err) {
466 		free(*path);
467 		*path = NULL;
468 	}
469 	return err;
470 }
471 
472 static const struct got_error *
473 read_fileindex_entry(struct got_fileindex_entry **entryp, SHA1_CTX *ctx,
474     FILE *infile)
475 {
476 	const struct got_error *err;
477 	struct got_fileindex_entry *entry;
478 	size_t n;
479 
480 	*entryp = NULL;
481 
482 	entry = calloc(1, sizeof(*entry));
483 	if (entry == NULL)
484 		return got_error_from_errno();
485 
486 	err = read_fileindex_val64(&entry->ctime_sec, ctx, infile);
487 	if (err)
488 		goto done;
489 	err = read_fileindex_val64(&entry->ctime_nsec, ctx, infile);
490 	if (err)
491 		goto done;
492 	err = read_fileindex_val64(&entry->mtime_sec, ctx, infile);
493 	if (err)
494 		goto done;
495 	err = read_fileindex_val64(&entry->mtime_nsec, ctx, infile);
496 	if (err)
497 		goto done;
498 
499 	err = read_fileindex_val32(&entry->uid, ctx, infile);
500 	if (err)
501 		goto done;
502 	err = read_fileindex_val32(&entry->gid, ctx, infile);
503 	if (err)
504 		goto done;
505 	err = read_fileindex_val32(&entry->size, ctx, infile);
506 	if (err)
507 		goto done;
508 
509 	err = read_fileindex_val16(&entry->mode, ctx, infile);
510 	if (err)
511 		goto done;
512 
513 	n = fread(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
514 	if (n != SHA1_DIGEST_LENGTH) {
515 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
516 		goto done;
517 	}
518 	SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
519 
520 	n = fread(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
521 	if (n != SHA1_DIGEST_LENGTH) {
522 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
523 		goto done;
524 	}
525 	SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
526 
527 	err = read_fileindex_val32(&entry->flags, ctx, infile);
528 	if (err)
529 		goto done;
530 
531 	err = read_fileindex_path(&entry->path, ctx, infile);
532 done:
533 	if (err)
534 		free(entry);
535 	else
536 		*entryp = entry;
537 	return err;
538 }
539 
540 const struct got_error *
541 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
542 {
543 	const struct got_error *err = NULL;
544 	struct got_fileindex_hdr hdr;
545 	SHA1_CTX ctx;
546 	struct got_fileindex_entry *entry;
547 	uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
548 	uint8_t sha1[SHA1_DIGEST_LENGTH];
549 	size_t n;
550 	int i;
551 
552 	SHA1Init(&ctx);
553 
554 	n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
555 	if (n != sizeof(hdr.signature)) {
556 		if (n == 0) /* EOF */
557 			return NULL;
558 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
559 	}
560 	n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
561 	if (n != sizeof(hdr.version)) {
562 		if (n == 0) /* EOF */
563 			return NULL;
564 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
565 	}
566 	n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
567 	if (n != sizeof(hdr.nentries)) {
568 		if (n == 0) /* EOF */
569 			return NULL;
570 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
571 	}
572 
573 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
574 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
575 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
576 
577 	hdr.signature = be32toh(hdr.signature);
578 	hdr.version = be32toh(hdr.version);
579 	hdr.nentries = be32toh(hdr.nentries);
580 
581 	if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
582 		return got_error(GOT_ERR_FILEIDX_SIG);
583 	if (hdr.version != GOT_FILE_INDEX_VERSION)
584 		return got_error(GOT_ERR_FILEIDX_VER);
585 
586 	for (i = 0; i < hdr.nentries; i++) {
587 		err = read_fileindex_entry(&entry, &ctx, infile);
588 		if (err)
589 			return err;
590 		err = add_entry(fileindex, entry);
591 		if (err)
592 			return err;
593 	}
594 
595 	n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
596 	if (n != sizeof(sha1_expected))
597 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
598 	SHA1Final(sha1, &ctx);
599 	if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
600 		return got_error(GOT_ERR_FILEIDX_CSUM);
601 
602 	return NULL;
603 }
604 
605 static struct got_fileindex_entry *
606 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
607 {
608 	struct got_fileindex_entry *next;
609 
610 	next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
611 
612 	/* Skip entries which were newly added by diff callbacks. */
613 	while (next && (next->flags & GOT_FILEIDX_F_NOT_FLUSHED))
614 		next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
615 
616 	return next;
617 }
618 
619 static const struct got_error *
620 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **,
621     struct got_tree_object *, const char *, struct got_repository *,
622     struct got_fileindex_diff_tree_cb *, void *);
623 
624 static const struct got_error *
625 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
626     struct got_fileindex_entry **ie, struct got_tree_entry *te,
627     const char *path, struct got_repository *repo,
628     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
629 {
630 	const struct got_error *err = NULL;
631 
632 	if (S_ISDIR(te->mode)) {
633 		char *subpath;
634 		struct got_tree_object *subtree;
635 
636 		if (asprintf(&subpath, "%s%s%s", path,
637 		    path[0] == '\0' ? "" : "/", te->name) == -1)
638 			return got_error_from_errno();
639 
640 		err = got_object_open_as_tree(&subtree, repo, te->id);
641 		if (err) {
642 			free(subpath);
643 			return err;
644 		}
645 
646 		err = diff_fileindex_tree(fileindex, ie, subtree,
647 		    subpath, repo, cb, cb_arg);
648 		free(subpath);
649 		got_object_tree_close(subtree);
650 		if (err)
651 			return err;
652 	}
653 
654 	*next = SIMPLEQ_NEXT(te, entry);
655 	return NULL;
656 }
657 
658 static const struct got_error *
659 diff_fileindex_tree(struct got_fileindex *fileindex,
660     struct got_fileindex_entry **ie, struct got_tree_object *tree,
661     const char *path, struct got_repository *repo,
662     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
663 {
664 	const struct got_error *err = NULL;
665 	struct got_tree_entry *te = NULL;
666 	size_t path_len = strlen(path);
667 	const struct got_tree_entries *entries;
668 	struct got_fileindex_entry *next;
669 
670 	entries = got_object_tree_get_entries(tree);
671 	te = SIMPLEQ_FIRST(&entries->head);
672 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te) {
673 		if (te && *ie) {
674 			char *te_path;
675 			int cmp;
676 			if (asprintf(&te_path, "%s/%s", path, te->name) == -1) {
677 				err = got_error_from_errno();
678 				break;
679 			}
680 			cmp = got_path_cmp((*ie)->path, te_path);
681 			free(te_path);
682 			if (cmp == 0) {
683 				err = cb->diff_old_new(cb_arg, *ie, te,
684 				    path);
685 				if (err)
686 					break;
687 				*ie = walk_fileindex(fileindex, *ie);
688 				err = walk_tree(&te, fileindex, ie, te,
689 				    path, repo, cb, cb_arg);
690 			} else if (cmp < 0 ) {
691 				next = walk_fileindex(fileindex, *ie);
692 				err = cb->diff_old(cb_arg, *ie, path);
693 				if (err)
694 					break;
695 				*ie = next;
696 			} else {
697 				err = cb->diff_new(cb_arg, te, path);
698 				if (err)
699 					break;
700 				err = walk_tree(&te, fileindex, ie, te,
701 				    path, repo, cb, cb_arg);
702 			}
703 			if (err)
704 				break;
705 		} else if (*ie) {
706 			next = walk_fileindex(fileindex, *ie);
707 			err = cb->diff_old(cb_arg, *ie, path);
708 			if (err)
709 				break;
710 			*ie = next;
711 		} else if (te) {
712 			err = cb->diff_new(cb_arg, te, path);
713 			if (err)
714 				break;
715 			err = walk_tree(&te, fileindex, ie, te, path, repo, cb,
716 			    cb_arg);
717 			if (err)
718 				break;
719 		}
720 	}
721 
722 	return err;
723 }
724 
725 const struct got_error *
726 got_fileindex_diff_tree(struct got_fileindex *fileindex,
727     struct got_tree_object *tree, struct got_repository *repo,
728     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
729 {
730 	struct got_fileindex_entry *min;
731 	min = RB_MIN(got_fileindex_tree, &fileindex->entries);
732 	return diff_fileindex_tree(fileindex, &min, tree, "", repo, cb, cb_arg);
733 }
734 
735 static const struct got_error *
736 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **, DIR *,
737     const char *, const char *, struct got_repository *,
738     struct got_fileindex_diff_dir_cb *, void *);
739 
740 static const struct got_error *
741 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
742     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle,
743     const char *path, DIR *dir, const char *rootpath,
744     struct got_repository *repo, struct got_fileindex_diff_dir_cb *cb,
745     void *cb_arg)
746 {
747 	const struct got_error *err = NULL;
748 	struct dirent *de = dle->data;
749 
750 	if (de->d_type == DT_DIR) {
751 		char *subpath;
752 		char *subdirpath;
753 		DIR *subdir;
754 
755 		if (asprintf(&subpath, "%s%s%s", path,
756 		    path[0] == '\0' ? "" : "/", de->d_name) == -1)
757 			return got_error_from_errno();
758 
759 		if (asprintf(&subdirpath, "%s/%s", rootpath, subpath) == -1) {
760 			free(subpath);
761 			return got_error_from_errno();
762 		}
763 
764 		subdir = opendir(subdirpath);
765 		if (subdir == NULL) {
766 			free(subpath);
767 			free(subdirpath);
768 			return got_error_from_errno();
769 		}
770 
771 		err = diff_fileindex_dir(fileindex, ie, subdir, rootpath,
772 		    subpath, repo, cb, cb_arg);
773 		free(subpath);
774 		free(subdirpath);
775 		closedir(subdir);
776 		if (err)
777 			return err;
778 	}
779 
780 	*next = TAILQ_NEXT(dle, entry);
781 	return NULL;
782 }
783 
784 static const struct got_error *
785 diff_fileindex_dir(struct got_fileindex *fileindex,
786     struct got_fileindex_entry **ie, DIR *dir, const char *rootpath,
787     const char *path, struct got_repository *repo,
788     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
789 {
790 	const struct got_error *err = NULL;
791 	struct dirent *de = NULL;
792 	size_t path_len = strlen(path);
793 	struct got_fileindex_entry *next;
794 	struct got_pathlist_head dirlist;
795 	struct got_pathlist_entry *dle;
796 
797 	TAILQ_INIT(&dirlist);
798 
799 	while (1) {
800 		struct got_pathlist_entry *new = NULL;
801 		struct dirent *dep = NULL;
802 
803 		de = malloc(sizeof(struct dirent) + NAME_MAX + 1);
804 		if (de == NULL) {
805 			err = got_error_from_errno();
806 			goto done;
807 		}
808 
809 		if (readdir_r(dir, de, &dep) != 0) {
810 			err = got_error_from_errno();
811 			free(de);
812 			goto done;
813 		}
814 		if (dep == NULL) {
815 			free(de);
816 			break;
817 		}
818 
819 		if (strcmp(de->d_name, ".") == 0 ||
820 		    strcmp(de->d_name, "..") == 0 ||
821 		    (path[0] == '\0' &&
822 		    strcmp(de->d_name, GOT_WORKTREE_GOT_DIR) == 0)) {
823 			free(de);
824 			continue;
825 		}
826 
827 		err = got_pathlist_insert(&new, &dirlist, de->d_name, de);
828 		if (err)
829 			goto done;
830 		if (new == NULL) {
831 			err = got_error(GOT_ERR_DIR_DUP_ENTRY);
832 			goto done;
833 		}
834 	}
835 
836 	dle = TAILQ_FIRST(&dirlist);
837 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || dle) {
838 		if (dle && *ie) {
839 			char *de_path;
840 			int cmp;
841 			de = dle->data;
842 			if (asprintf(&de_path, "%s/%s", path,
843 			    de->d_name) == -1) {
844 				err = got_error_from_errno();
845 				break;
846 			}
847 			cmp = got_path_cmp((*ie)->path, de_path);
848 			free(de_path);
849 			if (cmp == 0) {
850 				err = cb->diff_old_new(cb_arg, *ie, de, path);
851 				if (err)
852 					break;
853 				*ie = walk_fileindex(fileindex, *ie);
854 				err = walk_dir(&dle, fileindex, ie, dle, path,
855 				    dir, rootpath, repo, cb, cb_arg);
856 			} else if (cmp < 0 ) {
857 				next = walk_fileindex(fileindex, *ie);
858 				err = cb->diff_old(cb_arg, *ie, path);
859 				if (err)
860 					break;
861 				*ie = next;
862 			} else {
863 				err = cb->diff_new(cb_arg, de, path);
864 				if (err)
865 					break;
866 				err = walk_dir(&dle, fileindex, ie, dle, path,
867 				    dir, rootpath, repo, cb, cb_arg);
868 			}
869 			if (err)
870 				break;
871 		} else if (*ie) {
872 			next = walk_fileindex(fileindex, *ie);
873 			err = cb->diff_old(cb_arg, *ie, path);
874 			if (err)
875 				break;
876 			*ie = next;
877 		} else if (dle) {
878 			de = dle->data;
879 			err = cb->diff_new(cb_arg, de, path);
880 			if (err)
881 				break;
882 			err = walk_dir(&dle, fileindex, ie, dle, path, dir,
883 			    rootpath, repo, cb, cb_arg);
884 			if (err)
885 				break;
886 		}
887 	}
888 done:
889 	TAILQ_FOREACH(dle, &dirlist, entry)
890 		free(dle->data);
891 	got_pathlist_free(&dirlist);
892 	return err;
893 }
894 
895 const struct got_error *
896 got_fileindex_diff_dir(struct got_fileindex *fileindex, DIR *rootdir,
897     const char *rootpath, const char *path, struct got_repository *repo,
898     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
899 {
900 	struct got_fileindex_entry *min;
901 	min = RB_MIN(got_fileindex_tree, &fileindex->entries);
902 	return diff_fileindex_dir(fileindex, &min, rootdir, rootpath, path,
903 	    repo, cb, cb_arg);
904 }
905 
906 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);