1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include "got_compat.h"
18 
19 #include <sys/stat.h>
20 #include <sys/queue.h>
21 
22 #include <dirent.h>
23 #include <limits.h>
24 #include <stddef.h>
25 #include <string.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <time.h>
29 #include <fcntl.h>
30 #include <errno.h>
31 #include <unistd.h>
32 #include <zlib.h>
33 #include <fnmatch.h>
34 #include <libgen.h>
35 
36 #include "got_error.h"
37 #include "got_repository.h"
38 #include "got_reference.h"
39 #include "got_object.h"
40 #include "got_path.h"
41 #include "got_cancel.h"
42 #include "got_worktree.h"
43 #include "got_worktree_cvg.h"
44 #include "got_opentemp.h"
45 #include "got_diff.h"
46 #include "got_send.h"
47 #include "got_fetch.h"
48 
49 #include "got_lib_worktree.h"
50 #include "got_lib_hash.h"
51 #include "got_lib_fileindex.h"
52 #include "got_lib_inflate.h"
53 #include "got_lib_delta.h"
54 #include "got_lib_object.h"
55 #include "got_lib_object_parse.h"
56 #include "got_lib_object_create.h"
57 #include "got_lib_object_idset.h"
58 #include "got_lib_diff.h"
59 
60 #ifndef MIN
61 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
62 #endif
63 
64 #define GOT_MERGE_LABEL_MERGED	"merged change"
65 #define GOT_MERGE_LABEL_BASE	"3-way merge base"
66 
67 static const struct got_error *
68 lock_worktree(struct got_worktree *worktree, int operation)
69 {
70 	if (flock(worktree->lockfd, operation | LOCK_NB) == -1)
71 		return (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
72 		    : got_error_from_errno2("flock",
73 		    got_worktree_get_root_path(worktree)));
74 	return NULL;
75 }
76 
77 static const struct got_error *
78 is_bad_symlink_target(int *is_bad_symlink, const char *target_path,
79     size_t target_len, const char *ondisk_path, const char *wtroot_path)
80 {
81 	const struct got_error *err = NULL;
82 	char canonpath[PATH_MAX];
83 	char *path_got = NULL;
84 
85 	*is_bad_symlink = 0;
86 
87 	if (target_len >= sizeof(canonpath)) {
88 		*is_bad_symlink = 1;
89 		return NULL;
90 	}
91 
92 	/*
93 	 * We do not use realpath(3) to resolve the symlink's target
94 	 * path because we don't want to resolve symlinks recursively.
95 	 * Instead we make the path absolute and then canonicalize it.
96 	 * Relative symlink target lookup should begin at the directory
97 	 * in which the blob object is being installed.
98 	 */
99 	if (!got_path_is_absolute(target_path)) {
100 		char *abspath, *parent;
101 		err = got_path_dirname(&parent, ondisk_path);
102 		if (err)
103 			return err;
104 		if (asprintf(&abspath, "%s/%s",  parent, target_path) == -1) {
105 			free(parent);
106 			return got_error_from_errno("asprintf");
107 		}
108 		free(parent);
109 		if (strlen(abspath) >= sizeof(canonpath)) {
110 			err = got_error_path(abspath, GOT_ERR_BAD_PATH);
111 			free(abspath);
112 			return err;
113 		}
114 		err = got_canonpath(abspath, canonpath, sizeof(canonpath));
115 		free(abspath);
116 		if (err)
117 			return err;
118 	} else {
119 		err = got_canonpath(target_path, canonpath, sizeof(canonpath));
120 		if (err)
121 			return err;
122 	}
123 
124 	/* Only allow symlinks pointing at paths within the work tree. */
125 	if (!got_path_is_child(canonpath, wtroot_path, strlen(wtroot_path))) {
126 		*is_bad_symlink = 1;
127 		return NULL;
128 	}
129 
130 	/* Do not allow symlinks pointing into the .got directory. */
131 	if (asprintf(&path_got, "%s/%s", wtroot_path,
132 	    GOT_WORKTREE_GOT_DIR) == -1)
133 		return got_error_from_errno("asprintf");
134 	if (got_path_is_child(canonpath, path_got, strlen(path_got)))
135 		*is_bad_symlink = 1;
136 
137 	free(path_got);
138 	return NULL;
139 }
140 
141 /*
142  * Upgrade STATUS_MODIFY to STATUS_CONFLICT if a
143  * conflict marker is found in newly added lines only.
144  */
145 static const struct got_error *
146 get_modified_file_content_status(unsigned char *status,
147     struct got_blob_object *blob, const char *path, struct stat *sb,
148     FILE *ondisk_file)
149 {
150 	const struct got_error *err, *free_err;
151 	const char *markers[3] = {
152 		GOT_DIFF_CONFLICT_MARKER_BEGIN,
153 		GOT_DIFF_CONFLICT_MARKER_SEP,
154 		GOT_DIFF_CONFLICT_MARKER_END
155 	};
156 	FILE *f1 = NULL;
157 	struct got_diffreg_result *diffreg_result = NULL;
158 	struct diff_result *r;
159 	int nchunks_parsed, n, i = 0, ln = 0;
160 	char *line = NULL;
161 	size_t linesize = 0;
162 	ssize_t linelen;
163 
164 	if (*status != GOT_STATUS_MODIFY)
165 		return NULL;
166 
167 	f1 = got_opentemp();
168 	if (f1 == NULL)
169 		return got_error_from_errno("got_opentemp");
170 
171 	if (blob) {
172 		got_object_blob_rewind(blob);
173 		err = got_object_blob_dump_to_file(NULL, NULL, NULL, f1, blob);
174 		if (err)
175 			goto done;
176 	}
177 
178 	err = got_diff_files(&diffreg_result, f1, 1, NULL, ondisk_file,
179 	    1, NULL, 0, 0, 1, NULL, GOT_DIFF_ALGORITHM_MYERS);
180 	if (err)
181 		goto done;
182 
183 	r = diffreg_result->result;
184 
185 	for (n = 0; n < r->chunks.len; n += nchunks_parsed) {
186 		struct diff_chunk *c;
187 		struct diff_chunk_context cc = {};
188 		off_t pos;
189 
190 		/*
191 		 * We can optimise a little by advancing straight
192 		 * to the next chunk if this one has no added lines.
193 		 */
194 		c = diff_chunk_get(r, n);
195 
196 		if (diff_chunk_type(c) != CHUNK_PLUS) {
197 			nchunks_parsed = 1;
198 			continue;  /* removed or unchanged lines */
199 		}
200 
201 		pos = diff_chunk_get_right_start_pos(c);
202 		if (fseek(ondisk_file, pos, SEEK_SET) == -1) {
203 			err = got_ferror(ondisk_file, GOT_ERR_IO);
204 			goto done;
205 		}
206 
207 		diff_chunk_context_load_change(&cc, &nchunks_parsed, r, n, 0);
208 		ln = cc.right.start;
209 
210 		while (ln < cc.right.end) {
211 			linelen = getline(&line, &linesize, ondisk_file);
212 			if (linelen == -1) {
213 				if (feof(ondisk_file))
214 					break;
215 				err = got_ferror(ondisk_file, GOT_ERR_IO);
216 				break;
217 			}
218 
219 			if (line && strncmp(line, markers[i],
220 			    strlen(markers[i])) == 0) {
221 				if (strcmp(markers[i],
222 				    GOT_DIFF_CONFLICT_MARKER_END) == 0) {
223 					*status = GOT_STATUS_CONFLICT;
224 					goto done;
225 				} else
226 					i++;
227 			}
228 			++ln;
229 		}
230 	}
231 
232 done:
233 	free(line);
234 	if (f1 != NULL && fclose(f1) == EOF && err == NULL)
235 		err = got_error_from_errno("fclose");
236 	free_err = got_diffreg_result_free(diffreg_result);
237 	if (err == NULL)
238 		err = free_err;
239 
240 	return err;
241 }
242 
243 static int
244 xbit_differs(struct got_fileindex_entry *ie, uint16_t st_mode)
245 {
246 	mode_t ie_mode = got_fileindex_perms_to_st(ie);
247 	return ((ie_mode & S_IXUSR) != (st_mode & S_IXUSR));
248 }
249 
250 static int
251 stat_info_differs(struct got_fileindex_entry *ie, struct stat *sb)
252 {
253 	return !(ie->ctime_sec == sb->st_ctim.tv_sec &&
254 	    ie->ctime_nsec == sb->st_ctim.tv_nsec &&
255 	    ie->mtime_sec == sb->st_mtim.tv_sec &&
256 	    ie->mtime_nsec == sb->st_mtim.tv_nsec &&
257 	    ie->size == (sb->st_size & 0xffffffff) &&
258 	    !xbit_differs(ie, sb->st_mode));
259 }
260 
261 static unsigned char
262 get_staged_status(struct got_fileindex_entry *ie)
263 {
264 	switch (got_fileindex_entry_stage_get(ie)) {
265 	case GOT_FILEIDX_STAGE_ADD:
266 		return GOT_STATUS_ADD;
267 	case GOT_FILEIDX_STAGE_DELETE:
268 		return GOT_STATUS_DELETE;
269 	case GOT_FILEIDX_STAGE_MODIFY:
270 		return GOT_STATUS_MODIFY;
271 	default:
272 		return GOT_STATUS_NO_CHANGE;
273 	}
274 }
275 
276 static const struct got_error *
277 get_symlink_modification_status(unsigned char *status,
278     struct got_fileindex_entry *ie, const char *abspath,
279     int dirfd, const char *de_name, struct got_blob_object *blob)
280 {
281 	const struct got_error *err = NULL;
282 	char target_path[PATH_MAX];
283 	char etarget[PATH_MAX];
284 	ssize_t elen;
285 	size_t len, target_len = 0;
286 	const uint8_t *buf = got_object_blob_get_read_buf(blob);
287 	size_t hdrlen = got_object_blob_get_hdrlen(blob);
288 
289 	*status = GOT_STATUS_NO_CHANGE;
290 
291 	/* Blob object content specifies the target path of the link. */
292 	do {
293 		err = got_object_blob_read_block(&len, blob);
294 		if (err)
295 			return err;
296 		if (len + target_len >= sizeof(target_path)) {
297 			/*
298 			 * Should not happen. The blob contents were OK
299 			 * when this symlink was installed.
300 			 */
301 			return got_error(GOT_ERR_NO_SPACE);
302 		}
303 		if (len > 0) {
304 			/* Skip blob object header first time around. */
305 			memcpy(target_path + target_len, buf + hdrlen,
306 			    len - hdrlen);
307 			target_len += len - hdrlen;
308 			hdrlen = 0;
309 		}
310 	} while (len != 0);
311 	target_path[target_len] = '\0';
312 
313 	if (dirfd != -1) {
314 		elen = readlinkat(dirfd, de_name, etarget, sizeof(etarget));
315 		if (elen == -1)
316 			return got_error_from_errno2("readlinkat", abspath);
317 	} else {
318 		elen = readlink(abspath, etarget, sizeof(etarget));
319 		if (elen == -1)
320 			return got_error_from_errno2("readlink", abspath);
321 	}
322 
323 	if (elen != target_len || memcmp(etarget, target_path, target_len) != 0)
324 		*status = GOT_STATUS_MODIFY;
325 
326 	return NULL;
327 }
328 
329 static const struct got_error *
330 get_file_status(unsigned char *status, struct stat *sb,
331     struct got_fileindex_entry *ie, const char *abspath,
332     int dirfd, const char *de_name, struct got_repository *repo)
333 {
334 	const struct got_error *err = NULL;
335 	struct got_object_id id;
336 	size_t hdrlen;
337 	int fd = -1, fd1 = -1;
338 	FILE *f = NULL;
339 	uint8_t fbuf[8192];
340 	struct got_blob_object *blob = NULL;
341 	size_t flen, blen;
342 	unsigned char staged_status;
343 
344 	staged_status = get_staged_status(ie);
345 	*status = GOT_STATUS_NO_CHANGE;
346 	memset(sb, 0, sizeof(*sb));
347 
348 	/*
349 	 * Whenever the caller provides a directory descriptor and a
350 	 * directory entry name for the file, use them! This prevents
351 	 * race conditions if filesystem paths change beneath our feet.
352 	 */
353 	if (dirfd != -1) {
354 		if (fstatat(dirfd, de_name, sb, AT_SYMLINK_NOFOLLOW) == -1) {
355 			if (errno == ENOENT) {
356 				if (got_fileindex_entry_has_file_on_disk(ie))
357 					*status = GOT_STATUS_MISSING;
358 				else
359 					*status = GOT_STATUS_DELETE;
360 				goto done;
361 			}
362 			err = got_error_from_errno2("fstatat", abspath);
363 			goto done;
364 		}
365 	} else {
366 		fd = open(abspath, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
367 		if (fd == -1 && errno != ENOENT &&
368 		    !got_err_open_nofollow_on_symlink())
369 			return got_error_from_errno2("open", abspath);
370 		else if (fd == -1 && got_err_open_nofollow_on_symlink()) {
371 			if (lstat(abspath, sb) == -1)
372 				return got_error_from_errno2("lstat", abspath);
373 		} else if (fd == -1 || fstat(fd, sb) == -1) {
374 			if (errno == ENOENT) {
375 				if (got_fileindex_entry_has_file_on_disk(ie))
376 					*status = GOT_STATUS_MISSING;
377 				else
378 					*status = GOT_STATUS_DELETE;
379 				goto done;
380 			}
381 			err = got_error_from_errno2("fstat", abspath);
382 			goto done;
383 		}
384 	}
385 
386 	if (!S_ISREG(sb->st_mode) && !S_ISLNK(sb->st_mode)) {
387 		*status = GOT_STATUS_OBSTRUCTED;
388 		goto done;
389 	}
390 
391 	if (!got_fileindex_entry_has_file_on_disk(ie)) {
392 		*status = GOT_STATUS_DELETE;
393 		goto done;
394 	} else if (!got_fileindex_entry_has_blob(ie) &&
395 	    staged_status != GOT_STATUS_ADD) {
396 		*status = GOT_STATUS_ADD;
397 		goto done;
398 	}
399 
400 	if (!stat_info_differs(ie, sb))
401 		goto done;
402 
403 	if (S_ISLNK(sb->st_mode) &&
404 	    got_fileindex_entry_filetype_get(ie) != GOT_FILEIDX_MODE_SYMLINK) {
405 		*status = GOT_STATUS_MODIFY;
406 		goto done;
407 	}
408 
409 	if (staged_status == GOT_STATUS_MODIFY ||
410 	    staged_status == GOT_STATUS_ADD)
411 		got_fileindex_entry_get_staged_blob_id(&id, ie);
412 	else
413 		got_fileindex_entry_get_blob_id(&id, ie);
414 
415 	fd1 = got_opentempfd();
416 	if (fd1 == -1) {
417 		err = got_error_from_errno("got_opentempfd");
418 		goto done;
419 	}
420 	err = got_object_open_as_blob(&blob, repo, &id, sizeof(fbuf), fd1);
421 	if (err)
422 		goto done;
423 
424 	if (S_ISLNK(sb->st_mode)) {
425 		err = get_symlink_modification_status(status, ie,
426 		    abspath, dirfd, de_name, blob);
427 		goto done;
428 	}
429 
430 	if (dirfd != -1) {
431 		fd = openat(dirfd, de_name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
432 		if (fd == -1) {
433 			err = got_error_from_errno2("openat", abspath);
434 			goto done;
435 		}
436 	}
437 
438 	f = fdopen(fd, "r");
439 	if (f == NULL) {
440 		err = got_error_from_errno2("fdopen", abspath);
441 		goto done;
442 	}
443 	fd = -1;
444 	hdrlen = got_object_blob_get_hdrlen(blob);
445 	for (;;) {
446 		const uint8_t *bbuf = got_object_blob_get_read_buf(blob);
447 		err = got_object_blob_read_block(&blen, blob);
448 		if (err)
449 			goto done;
450 		/* Skip length of blob object header first time around. */
451 		flen = fread(fbuf, 1, sizeof(fbuf) - hdrlen, f);
452 		if (flen == 0 && ferror(f)) {
453 			err = got_error_from_errno("fread");
454 			goto done;
455 		}
456 		if (blen - hdrlen == 0) {
457 			if (flen != 0)
458 				*status = GOT_STATUS_MODIFY;
459 			break;
460 		} else if (flen == 0) {
461 			if (blen - hdrlen != 0)
462 				*status = GOT_STATUS_MODIFY;
463 			break;
464 		} else if (blen - hdrlen == flen) {
465 			/* Skip blob object header first time around. */
466 			if (memcmp(bbuf + hdrlen, fbuf, flen) != 0) {
467 				*status = GOT_STATUS_MODIFY;
468 				break;
469 			}
470 		} else {
471 			*status = GOT_STATUS_MODIFY;
472 			break;
473 		}
474 		hdrlen = 0;
475 	}
476 
477 	if (*status == GOT_STATUS_MODIFY) {
478 		rewind(f);
479 		err = get_modified_file_content_status(status, blob, ie->path,
480 		    sb, f);
481 	} else if (xbit_differs(ie, sb->st_mode))
482 		*status = GOT_STATUS_MODE_CHANGE;
483 done:
484 	if (fd1 != -1 && close(fd1) == -1 && err == NULL)
485 		err = got_error_from_errno("close");
486 	if (blob)
487 		got_object_blob_close(blob);
488 	if (f != NULL && fclose(f) == EOF && err == NULL)
489 		err = got_error_from_errno2("fclose", abspath);
490 	if (fd != -1 && close(fd) == -1 && err == NULL)
491 		err = got_error_from_errno2("close", abspath);
492 	return err;
493 }
494 
495 static const struct got_error *
496 get_ref_name(char **refname, struct got_worktree *worktree, const char *prefix)
497 {
498 	const struct got_error *err = NULL;
499 	char *uuidstr = NULL;
500 
501 	*refname = NULL;
502 
503 	err = got_worktree_get_uuid(&uuidstr, worktree);
504 	if (err)
505 		return err;
506 
507 	if (asprintf(refname, "%s-%s", prefix, uuidstr) == -1) {
508 		err = got_error_from_errno("asprintf");
509 		*refname = NULL;
510 	}
511 	free(uuidstr);
512 	return err;
513 }
514 
515 static const struct got_error *
516 get_base_ref_name(char **refname, struct got_worktree *worktree)
517 {
518 	return get_ref_name(refname, worktree, GOT_WORKTREE_BASE_REF_PREFIX);
519 }
520 
521 /*
522  * Prevent Git's garbage collector from deleting our base commit by
523  * setting a reference to our base commit's ID.
524  */
525 static const struct got_error *
526 ref_base_commit(struct got_worktree *worktree, struct got_repository *repo)
527 {
528 	const struct got_error *err = NULL;
529 	struct got_reference *ref = NULL;
530 	char *refname;
531 
532 	err = get_base_ref_name(&refname, worktree);
533 	if (err)
534 		return err;
535 
536 	err = got_ref_alloc(&ref, refname, worktree->base_commit_id);
537 	if (err)
538 		goto done;
539 
540 	err = got_ref_write(ref, repo);
541 done:
542 	free(refname);
543 	if (ref)
544 		got_ref_close(ref);
545 	return err;
546 }
547 
548 static const struct got_error *
549 get_fileindex_path(char **fileindex_path, struct got_worktree *worktree)
550 {
551 	const struct got_error *err = NULL;
552 
553 	if (asprintf(fileindex_path, "%s/%s/%s", worktree->root_path,
554 	    GOT_WORKTREE_GOT_DIR, GOT_WORKTREE_FILE_INDEX) == -1) {
555 		err = got_error_from_errno("asprintf");
556 		*fileindex_path = NULL;
557 	}
558 	return err;
559 }
560 
561 static const struct got_error *
562 open_fileindex(struct got_fileindex **fileindex, char **fileindex_path,
563     struct got_worktree *worktree)
564 {
565 	const struct got_error *err = NULL;
566 	FILE *index = NULL;
567 
568 	*fileindex_path = NULL;
569 	*fileindex = got_fileindex_alloc();
570 	if (*fileindex == NULL)
571 		return got_error_from_errno("got_fileindex_alloc");
572 
573 	err = get_fileindex_path(fileindex_path, worktree);
574 	if (err)
575 		goto done;
576 
577 	index = fopen(*fileindex_path, "rbe");
578 	if (index == NULL) {
579 		if (errno != ENOENT)
580 			err = got_error_from_errno2("fopen", *fileindex_path);
581 	} else {
582 		err = got_fileindex_read(*fileindex, index);
583 		if (fclose(index) == EOF && err == NULL)
584 			err = got_error_from_errno("fclose");
585 	}
586 done:
587 	if (err) {
588 		free(*fileindex_path);
589 		*fileindex_path = NULL;
590 		got_fileindex_free(*fileindex);
591 		*fileindex = NULL;
592 	}
593 	return err;
594 }
595 
596 static const struct got_error *
597 sync_fileindex(struct got_fileindex *fileindex, const char *fileindex_path)
598 {
599 	const struct got_error *err = NULL;
600 	char *new_fileindex_path = NULL;
601 	FILE *new_index = NULL;
602 	struct timespec timeout;
603 
604 	err = got_opentemp_named(&new_fileindex_path, &new_index,
605 	    fileindex_path, "");
606 	if (err)
607 		goto done;
608 
609 	err = got_fileindex_write(fileindex, new_index);
610 	if (err)
611 		goto done;
612 
613 	if (rename(new_fileindex_path, fileindex_path) != 0) {
614 		err = got_error_from_errno3("rename", new_fileindex_path,
615 		    fileindex_path);
616 		unlink(new_fileindex_path);
617 	}
618 
619 	/*
620 	 * Sleep for a short amount of time to ensure that files modified after
621 	 * this program exits have a different time stamp from the one which
622 	 * was recorded in the file index.
623 	 */
624 	timeout.tv_sec = 0;
625 	timeout.tv_nsec = 1;
626 	nanosleep(&timeout, NULL);
627 done:
628 	if (new_index)
629 		fclose(new_index);
630 	free(new_fileindex_path);
631 	return err;
632 }
633 
634 struct diff_dir_cb_arg {
635 	struct got_fileindex *fileindex;
636 	struct got_worktree *worktree;
637 	const char *status_path;
638 	size_t status_path_len;
639 	struct got_repository *repo;
640 	got_worktree_status_cb status_cb;
641 	void *status_arg;
642 	got_cancel_cb cancel_cb;
643 	void *cancel_arg;
644 	/* A pathlist containing per-directory pathlists of ignore patterns. */
645 	struct got_pathlist_head *ignores;
646 	int report_unchanged;
647 	int no_ignores;
648 };
649 
650 static const struct got_error *
651 report_file_status(struct got_fileindex_entry *ie, const char *abspath,
652     int dirfd, const char *de_name,
653     got_worktree_status_cb status_cb, void *status_arg,
654     struct got_repository *repo, int report_unchanged)
655 {
656 	const struct got_error *err = NULL;
657 	unsigned char status = GOT_STATUS_NO_CHANGE;
658 	unsigned char staged_status;
659 	struct stat sb;
660 	struct got_object_id blob_id, commit_id, staged_blob_id;
661 	struct got_object_id *blob_idp = NULL, *commit_idp = NULL;
662 	struct got_object_id *staged_blob_idp = NULL;
663 
664 	staged_status = get_staged_status(ie);
665 	err = get_file_status(&status, &sb, ie, abspath, dirfd, de_name, repo);
666 	if (err)
667 		return err;
668 
669 	if (status == GOT_STATUS_NO_CHANGE &&
670 	    staged_status == GOT_STATUS_NO_CHANGE && !report_unchanged)
671 		return NULL;
672 
673 	if (got_fileindex_entry_has_blob(ie))
674 		blob_idp = got_fileindex_entry_get_blob_id(&blob_id, ie);
675 	if (got_fileindex_entry_has_commit(ie))
676 		commit_idp = got_fileindex_entry_get_commit_id(&commit_id, ie);
677 	if (staged_status == GOT_STATUS_ADD ||
678 	    staged_status == GOT_STATUS_MODIFY) {
679 		staged_blob_idp = got_fileindex_entry_get_staged_blob_id(
680 		    &staged_blob_id, ie);
681 	}
682 
683 	return (*status_cb)(status_arg, status, staged_status,
684 	    ie->path, blob_idp, staged_blob_idp, commit_idp, dirfd, de_name);
685 }
686 
687 static const struct got_error *
688 status_old_new(void *arg, struct got_fileindex_entry *ie,
689     struct dirent *de, const char *parent_path, int dirfd)
690 {
691 	const struct got_error *err = NULL;
692 	struct diff_dir_cb_arg *a = arg;
693 	char *abspath;
694 
695 	if (a->cancel_cb) {
696 		err = a->cancel_cb(a->cancel_arg);
697 		if (err)
698 			return err;
699 	}
700 
701 	if (got_path_cmp(parent_path, a->status_path,
702 	    strlen(parent_path), a->status_path_len) != 0 &&
703 	    !got_path_is_child(parent_path, a->status_path, a->status_path_len))
704 		return NULL;
705 
706 	if (parent_path[0]) {
707 		if (asprintf(&abspath, "%s/%s/%s", a->worktree->root_path,
708 		    parent_path, de->d_name) == -1)
709 			return got_error_from_errno("asprintf");
710 	} else {
711 		if (asprintf(&abspath, "%s/%s", a->worktree->root_path,
712 		    de->d_name) == -1)
713 			return got_error_from_errno("asprintf");
714 	}
715 
716 	err = report_file_status(ie, abspath, dirfd, de->d_name,
717 	    a->status_cb, a->status_arg, a->repo, a->report_unchanged);
718 	free(abspath);
719 	return err;
720 }
721 
722 static const struct got_error *
723 status_old(void *arg, struct got_fileindex_entry *ie, const char *parent_path)
724 {
725 	const struct got_error *err = NULL;
726 	struct diff_dir_cb_arg *a = arg;
727 	struct got_object_id blob_id, commit_id;
728 	unsigned char status;
729 
730 	if (a->cancel_cb) {
731 		err = a->cancel_cb(a->cancel_arg);
732 		if (err)
733 			return err;
734 	}
735 
736 	if (!got_path_is_child(ie->path, a->status_path, a->status_path_len))
737 		return NULL;
738 
739 	got_fileindex_entry_get_blob_id(&blob_id, ie);
740 	got_fileindex_entry_get_commit_id(&commit_id, ie);
741 	if (got_fileindex_entry_has_file_on_disk(ie))
742 		status = GOT_STATUS_MISSING;
743 	else
744 		status = GOT_STATUS_DELETE;
745 	return (*a->status_cb)(a->status_arg, status, get_staged_status(ie),
746 	    ie->path, &blob_id, NULL, &commit_id, -1, NULL);
747 }
748 
749 static void
750 free_ignores(struct got_pathlist_head *ignores)
751 {
752 	struct got_pathlist_entry *pe;
753 
754 	TAILQ_FOREACH(pe, ignores, entry) {
755 		struct got_pathlist_head *ignorelist = pe->data;
756 
757 		got_pathlist_free(ignorelist, GOT_PATHLIST_FREE_PATH);
758 	}
759 	got_pathlist_free(ignores, GOT_PATHLIST_FREE_PATH);
760 }
761 
762 static const struct got_error *
763 read_ignores(struct got_pathlist_head *ignores, const char *path, FILE *f)
764 {
765 	const struct got_error *err = NULL;
766 	struct got_pathlist_entry *pe = NULL;
767 	struct got_pathlist_head *ignorelist;
768 	char *line = NULL, *pattern, *dirpath = NULL;
769 	size_t linesize = 0;
770 	ssize_t linelen;
771 
772 	ignorelist = calloc(1, sizeof(*ignorelist));
773 	if (ignorelist == NULL)
774 		return got_error_from_errno("calloc");
775 	TAILQ_INIT(ignorelist);
776 
777 	while ((linelen = getline(&line, &linesize, f)) != -1) {
778 		if (linelen > 0 && line[linelen - 1] == '\n')
779 			line[linelen - 1] = '\0';
780 
781 		/* Git's ignores may contain comments. */
782 		if (line[0] == '#')
783 			continue;
784 
785 		/* Git's negated patterns are not (yet?) supported. */
786 		if (line[0] == '!')
787 			continue;
788 
789 		if (asprintf(&pattern, "%s%s%s", path, path[0] ? "/" : "",
790 		    line) == -1) {
791 			err = got_error_from_errno("asprintf");
792 			goto done;
793 		}
794 		err = got_pathlist_insert(NULL, ignorelist, pattern, NULL);
795 		if (err)
796 			goto done;
797 	}
798 	if (ferror(f)) {
799 		err = got_error_from_errno("getline");
800 		goto done;
801 	}
802 
803 	dirpath = strdup(path);
804 	if (dirpath == NULL) {
805 		err = got_error_from_errno("strdup");
806 		goto done;
807 	}
808 	err = got_pathlist_insert(&pe, ignores, dirpath, ignorelist);
809 done:
810 	free(line);
811 	if (err || pe == NULL) {
812 		free(dirpath);
813 		got_pathlist_free(ignorelist, GOT_PATHLIST_FREE_PATH);
814 	}
815 	return err;
816 }
817 
818 static int
819 match_path(const char *pattern, size_t pattern_len, const char *path,
820     int flags)
821 {
822 	char buf[PATH_MAX];
823 
824 	/*
825 	 * Trailing slashes signify directories.
826 	 * Append a * to make such patterns conform to fnmatch rules.
827 	 */
828 	if (pattern_len > 0 && pattern[pattern_len - 1] == '/') {
829 		if (snprintf(buf, sizeof(buf), "%s*", pattern) >= sizeof(buf))
830 			return FNM_NOMATCH; /* XXX */
831 
832 		return fnmatch(buf, path, flags);
833 	}
834 
835 	return fnmatch(pattern, path, flags);
836 }
837 
838 static int
839 match_ignores(struct got_pathlist_head *ignores, const char *path)
840 {
841 	struct got_pathlist_entry *pe;
842 
843 	/* Handle patterns which match in all directories. */
844 	TAILQ_FOREACH(pe, ignores, entry) {
845 		struct got_pathlist_head *ignorelist = pe->data;
846 		struct got_pathlist_entry *pi;
847 
848 		TAILQ_FOREACH(pi, ignorelist, entry) {
849 			const char *p;
850 
851 			if (pi->path_len < 3 ||
852 			    strncmp(pi->path, "**/", 3) != 0)
853 				continue;
854 			p = path;
855 			while (*p) {
856 				if (match_path(pi->path + 3,
857 				    pi->path_len - 3, p,
858 				    FNM_PATHNAME | FNM_LEADING_DIR)) {
859 					/* Retry in next directory. */
860 					while (*p && *p != '/')
861 						p++;
862 					while (*p == '/')
863 						p++;
864 					continue;
865 				}
866 				return 1;
867 			}
868 		}
869 	}
870 
871 	/*
872 	 * The ignores pathlist contains ignore lists from children before
873 	 * parents, so we can find the most specific ignorelist by walking
874 	 * ignores backwards.
875 	 */
876 	pe = TAILQ_LAST(ignores, got_pathlist_head);
877 	while (pe) {
878 		if (got_path_is_child(path, pe->path, pe->path_len)) {
879 			struct got_pathlist_head *ignorelist = pe->data;
880 			struct got_pathlist_entry *pi;
881 			TAILQ_FOREACH(pi, ignorelist, entry) {
882 				int flags = FNM_LEADING_DIR;
883 				if (strstr(pi->path, "/**/") == NULL)
884 					flags |= FNM_PATHNAME;
885 				if (match_path(pi->path, pi->path_len,
886 				    path, flags))
887 					continue;
888 				return 1;
889 			}
890 		}
891 		pe = TAILQ_PREV(pe, got_pathlist_head, entry);
892 	}
893 
894 	return 0;
895 }
896 
897 static const struct got_error *
898 add_ignores(struct got_pathlist_head *ignores, const char *root_path,
899     const char *path, int dirfd, const char *ignores_filename)
900 {
901 	const struct got_error *err = NULL;
902 	char *ignorespath;
903 	int fd = -1;
904 	FILE *ignoresfile = NULL;
905 
906 	if (asprintf(&ignorespath, "%s/%s%s%s", root_path, path,
907 	    path[0] ? "/" : "", ignores_filename) == -1)
908 		return got_error_from_errno("asprintf");
909 
910 	if (dirfd != -1) {
911 		fd = openat(dirfd, ignores_filename,
912 		    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
913 		if (fd == -1) {
914 			if (errno != ENOENT && errno != EACCES)
915 				err = got_error_from_errno2("openat",
916 				    ignorespath);
917 		} else {
918 			ignoresfile = fdopen(fd, "r");
919 			if (ignoresfile == NULL)
920 				err = got_error_from_errno2("fdopen",
921 				    ignorespath);
922 			else {
923 				fd = -1;
924 				err = read_ignores(ignores, path, ignoresfile);
925 			}
926 		}
927 	} else {
928 		ignoresfile = fopen(ignorespath, "re");
929 		if (ignoresfile == NULL) {
930 			if (errno != ENOENT && errno != EACCES)
931 				err = got_error_from_errno2("fopen",
932 				    ignorespath);
933 		} else
934 			err = read_ignores(ignores, path, ignoresfile);
935 	}
936 
937 	if (ignoresfile && fclose(ignoresfile) == EOF && err == NULL)
938 		err = got_error_from_errno2("fclose", path);
939 	if (fd != -1 && close(fd) == -1 && err == NULL)
940 		err = got_error_from_errno2("close", path);
941 	free(ignorespath);
942 	return err;
943 }
944 
945 static const struct got_error *
946 status_new(int *ignore, void *arg, struct dirent *de, const char *parent_path,
947     int dirfd)
948 {
949 	const struct got_error *err = NULL;
950 	struct diff_dir_cb_arg *a = arg;
951 	char *path = NULL;
952 
953 	if (ignore != NULL)
954 		*ignore = 0;
955 
956 	if (a->cancel_cb) {
957 		err = a->cancel_cb(a->cancel_arg);
958 		if (err)
959 			return err;
960 	}
961 
962 	if (parent_path[0]) {
963 		if (asprintf(&path, "%s/%s", parent_path, de->d_name) == -1)
964 			return got_error_from_errno("asprintf");
965 	} else {
966 		path = de->d_name;
967 	}
968 
969 	if (de->d_type == DT_DIR) {
970 		if (!a->no_ignores && ignore != NULL &&
971 		    match_ignores(a->ignores, path))
972 			*ignore = 1;
973 	} else if (!match_ignores(a->ignores, path) &&
974 	    got_path_is_child(path, a->status_path, a->status_path_len))
975 		err = (*a->status_cb)(a->status_arg, GOT_STATUS_UNVERSIONED,
976 		    GOT_STATUS_NO_CHANGE, path, NULL, NULL, NULL, -1, NULL);
977 	if (parent_path[0])
978 		free(path);
979 	return err;
980 }
981 
982 static const struct got_error *
983 status_traverse(void *arg, const char *path, int dirfd)
984 {
985 	const struct got_error *err = NULL;
986 	struct diff_dir_cb_arg *a = arg;
987 
988 	if (a->no_ignores)
989 		return NULL;
990 
991 	err = add_ignores(a->ignores, a->worktree->root_path,
992 	    path, dirfd, ".cvsignore");
993 	if (err)
994 		return err;
995 
996 	err = add_ignores(a->ignores, a->worktree->root_path, path,
997 	    dirfd, ".gitignore");
998 
999 	return err;
1000 }
1001 
1002 static const struct got_error *
1003 report_single_file_status(const char *path, const char *ondisk_path,
1004     struct got_fileindex *fileindex, got_worktree_status_cb status_cb,
1005     void *status_arg, struct got_repository *repo, int report_unchanged,
1006     struct got_pathlist_head *ignores, int no_ignores)
1007 {
1008 	struct got_fileindex_entry *ie;
1009 	struct stat sb;
1010 
1011 	ie = got_fileindex_entry_get(fileindex, path, strlen(path));
1012 	if (ie)
1013 		return report_file_status(ie, ondisk_path, -1, NULL,
1014 		    status_cb, status_arg, repo, report_unchanged);
1015 
1016 	if (lstat(ondisk_path, &sb) == -1) {
1017 		if (errno != ENOENT)
1018 			return got_error_from_errno2("lstat", ondisk_path);
1019 		return (*status_cb)(status_arg, GOT_STATUS_NONEXISTENT,
1020 		    GOT_STATUS_NO_CHANGE, path, NULL, NULL, NULL, -1, NULL);
1021 	}
1022 
1023 	if (!no_ignores && match_ignores(ignores, path))
1024 		return NULL;
1025 
1026 	if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode))
1027 		return (*status_cb)(status_arg, GOT_STATUS_UNVERSIONED,
1028 		    GOT_STATUS_NO_CHANGE, path, NULL, NULL, NULL, -1, NULL);
1029 
1030 	return NULL;
1031 }
1032 
1033 static const struct got_error *
1034 add_ignores_from_parent_paths(struct got_pathlist_head *ignores,
1035     const char *root_path, const char *path)
1036 {
1037 	const struct got_error *err;
1038 	char *parent_path, *next_parent_path = NULL;
1039 
1040 	err = add_ignores(ignores, root_path, "", -1,
1041 	    ".cvsignore");
1042 	if (err)
1043 		return err;
1044 
1045 	err = add_ignores(ignores, root_path, "", -1,
1046 	    ".gitignore");
1047 	if (err)
1048 		return err;
1049 
1050 	err = got_path_dirname(&parent_path, path);
1051 	if (err) {
1052 		if (err->code == GOT_ERR_BAD_PATH)
1053 			return NULL; /* cannot traverse parent */
1054 		return err;
1055 	}
1056 	for (;;) {
1057 		err = add_ignores(ignores, root_path, parent_path, -1,
1058 		    ".cvsignore");
1059 		if (err)
1060 			break;
1061 		err = add_ignores(ignores, root_path, parent_path, -1,
1062 		    ".gitignore");
1063 		if (err)
1064 			break;
1065 		err = got_path_dirname(&next_parent_path, parent_path);
1066 		if (err) {
1067 			if (err->code == GOT_ERR_BAD_PATH)
1068 				err = NULL; /* traversed everything */
1069 			break;
1070 		}
1071 		if (got_path_is_root_dir(parent_path))
1072 			break;
1073 		free(parent_path);
1074 		parent_path = next_parent_path;
1075 		next_parent_path = NULL;
1076 	}
1077 
1078 	free(parent_path);
1079 	free(next_parent_path);
1080 	return err;
1081 }
1082 
1083 struct find_missing_children_args {
1084 	const char *parent_path;
1085 	size_t parent_len;
1086 	struct got_pathlist_head *children;
1087 	got_cancel_cb cancel_cb;
1088 	void *cancel_arg;
1089 };
1090 
1091 static const struct got_error *
1092 find_missing_children(void *arg, struct got_fileindex_entry *ie)
1093 {
1094 	const struct got_error *err = NULL;
1095 	struct find_missing_children_args *a = arg;
1096 
1097 	if (a->cancel_cb) {
1098 		err = a->cancel_cb(a->cancel_arg);
1099 		if (err)
1100 			return err;
1101 	}
1102 
1103 	if (got_path_is_child(ie->path, a->parent_path, a->parent_len))
1104 		err = got_pathlist_append(a->children, ie->path, NULL);
1105 
1106 	return err;
1107 }
1108 
1109 static const struct got_error *
1110 report_children(struct got_pathlist_head *children,
1111     struct got_worktree *worktree, struct got_fileindex *fileindex,
1112     struct got_repository *repo, int is_root_dir, int report_unchanged,
1113     struct got_pathlist_head *ignores, int no_ignores,
1114     got_worktree_status_cb status_cb, void *status_arg,
1115     got_cancel_cb cancel_cb, void *cancel_arg)
1116 {
1117 	const struct got_error *err = NULL;
1118 	struct got_pathlist_entry *pe;
1119 	char *ondisk_path = NULL;
1120 
1121 	TAILQ_FOREACH(pe, children, entry) {
1122 		if (cancel_cb) {
1123 			err = cancel_cb(cancel_arg);
1124 			if (err)
1125 				break;
1126 		}
1127 
1128 		if (asprintf(&ondisk_path, "%s%s%s", worktree->root_path,
1129 		    !is_root_dir ? "/" : "", pe->path) == -1) {
1130 			err = got_error_from_errno("asprintf");
1131 			ondisk_path = NULL;
1132 			break;
1133 		}
1134 
1135 		err = report_single_file_status(pe->path, ondisk_path,
1136 		    fileindex, status_cb, status_arg, repo, report_unchanged,
1137 		    ignores, no_ignores);
1138 		if (err)
1139 			break;
1140 
1141 		free(ondisk_path);
1142 		ondisk_path = NULL;
1143 	}
1144 
1145 	free(ondisk_path);
1146 	return err;
1147 }
1148 
1149 static const struct got_error *
1150 worktree_status(struct got_worktree *worktree, const char *path,
1151     struct got_fileindex *fileindex, struct got_repository *repo,
1152     got_worktree_status_cb status_cb, void *status_arg,
1153     got_cancel_cb cancel_cb, void *cancel_arg, int no_ignores,
1154     int report_unchanged)
1155 {
1156 	const struct got_error *err = NULL;
1157 	int fd = -1;
1158 	struct got_fileindex_diff_dir_cb fdiff_cb;
1159 	struct diff_dir_cb_arg arg;
1160 	char *ondisk_path = NULL;
1161 	struct got_pathlist_head ignores, missing_children;
1162 	struct got_fileindex_entry *ie;
1163 
1164 	TAILQ_INIT(&ignores);
1165 	TAILQ_INIT(&missing_children);
1166 
1167 	if (asprintf(&ondisk_path, "%s%s%s",
1168 	    worktree->root_path, path[0] ? "/" : "", path) == -1)
1169 		return got_error_from_errno("asprintf");
1170 
1171 	ie = got_fileindex_entry_get(fileindex, path, strlen(path));
1172 	if (ie) {
1173 		err = report_single_file_status(path, ondisk_path,
1174 		    fileindex, status_cb, status_arg, repo,
1175 		    report_unchanged, &ignores, no_ignores);
1176 		goto done;
1177 	} else {
1178 		struct find_missing_children_args fmca;
1179 		fmca.parent_path = path;
1180 		fmca.parent_len = strlen(path);
1181 		fmca.children = &missing_children;
1182 		fmca.cancel_cb = cancel_cb;
1183 		fmca.cancel_arg = cancel_arg;
1184 		err = got_fileindex_for_each_entry_safe(fileindex,
1185 		    find_missing_children, &fmca);
1186 		if (err)
1187 			goto done;
1188 	}
1189 
1190 	fd = open(ondisk_path, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC);
1191 	if (fd == -1) {
1192 		if (errno != ENOTDIR && errno != ENOENT && errno != EACCES &&
1193 		    !got_err_open_nofollow_on_symlink())
1194 			err = got_error_from_errno2("open", ondisk_path);
1195 		else {
1196 			if (!no_ignores) {
1197 				err = add_ignores_from_parent_paths(&ignores,
1198 				    worktree->root_path, ondisk_path);
1199 				if (err)
1200 					goto done;
1201 			}
1202 			if (TAILQ_EMPTY(&missing_children)) {
1203 				err = report_single_file_status(path,
1204 				    ondisk_path, fileindex,
1205 				    status_cb, status_arg, repo,
1206 				    report_unchanged, &ignores, no_ignores);
1207 				if (err)
1208 					goto done;
1209 			} else {
1210 				err = report_children(&missing_children,
1211 				    worktree, fileindex, repo,
1212 				    (path[0] == '\0'), report_unchanged,
1213 				    &ignores, no_ignores,
1214 				    status_cb, status_arg,
1215 				    cancel_cb, cancel_arg);
1216 				if (err)
1217 					goto done;
1218 			}
1219 		}
1220 	} else {
1221 		fdiff_cb.diff_old_new = status_old_new;
1222 		fdiff_cb.diff_old = status_old;
1223 		fdiff_cb.diff_new = status_new;
1224 		fdiff_cb.diff_traverse = status_traverse;
1225 		arg.fileindex = fileindex;
1226 		arg.worktree = worktree;
1227 		arg.status_path = path;
1228 		arg.status_path_len = strlen(path);
1229 		arg.repo = repo;
1230 		arg.status_cb = status_cb;
1231 		arg.status_arg = status_arg;
1232 		arg.cancel_cb = cancel_cb;
1233 		arg.cancel_arg = cancel_arg;
1234 		arg.report_unchanged = report_unchanged;
1235 		arg.no_ignores = no_ignores;
1236 		if (!no_ignores) {
1237 			err = add_ignores_from_parent_paths(&ignores,
1238 			    worktree->root_path, path);
1239 			if (err)
1240 				goto done;
1241 		}
1242 		arg.ignores = &ignores;
1243 		err = got_fileindex_diff_dir(fileindex, fd,
1244 		    worktree->root_path, path, repo, &fdiff_cb, &arg);
1245 	}
1246 done:
1247 	free_ignores(&ignores);
1248 	if (fd != -1 && close(fd) == -1 && err == NULL)
1249 		err = got_error_from_errno("close");
1250 	free(ondisk_path);
1251 	return err;
1252 }
1253 
1254 static void
1255 free_commitable(struct got_commitable *ct)
1256 {
1257 	free(ct->path);
1258 	free(ct->in_repo_path);
1259 	free(ct->ondisk_path);
1260 	free(ct->blob_id);
1261 	free(ct->base_blob_id);
1262 	free(ct->staged_blob_id);
1263 	free(ct->base_commit_id);
1264 	free(ct);
1265 }
1266 
1267 struct collect_commitables_arg {
1268 	struct got_pathlist_head *commitable_paths;
1269 	struct got_repository *repo;
1270 	struct got_worktree *worktree;
1271 	struct got_fileindex *fileindex;
1272 	int have_staged_files;
1273 	int allow_bad_symlinks;
1274 	int diff_header_shown;
1275 	int commit_conflicts;
1276 	FILE *diff_outfile;
1277 	FILE *f1;
1278 	FILE *f2;
1279 };
1280 
1281 /*
1282  * Create a file which contains the target path of a symlink so we can feed
1283  * it as content to the diff engine.
1284  */
1285 static const struct got_error *
1286 get_symlink_target_file(int *fd, int dirfd, const char *de_name,
1287     const char *abspath)
1288 {
1289 	const struct got_error *err = NULL;
1290 	char target_path[PATH_MAX];
1291 	ssize_t target_len, outlen;
1292 
1293 	*fd = -1;
1294 
1295 	if (dirfd != -1) {
1296 		target_len = readlinkat(dirfd, de_name, target_path, PATH_MAX);
1297 		if (target_len == -1)
1298 			return got_error_from_errno2("readlinkat", abspath);
1299 	} else {
1300 		target_len = readlink(abspath, target_path, PATH_MAX);
1301 		if (target_len == -1)
1302 			return got_error_from_errno2("readlink", abspath);
1303 	}
1304 
1305 	*fd = got_opentempfd();
1306 	if (*fd == -1)
1307 		return got_error_from_errno("got_opentempfd");
1308 
1309 	outlen = write(*fd, target_path, target_len);
1310 	if (outlen == -1) {
1311 		err = got_error_from_errno("got_opentempfd");
1312 		goto done;
1313 	}
1314 
1315 	if (lseek(*fd, 0, SEEK_SET) == -1) {
1316 		err = got_error_from_errno2("lseek", abspath);
1317 		goto done;
1318 	}
1319 done:
1320 	if (err) {
1321 		close(*fd);
1322 		*fd = -1;
1323 	}
1324 	return err;
1325 }
1326 
1327 static const struct got_error *
1328 append_ct_diff(struct got_commitable *ct, int *diff_header_shown,
1329     FILE *diff_outfile, FILE *f1, FILE *f2, int dirfd, const char *de_name,
1330     int diff_staged, struct got_repository *repo, struct got_worktree *worktree)
1331 {
1332 	const struct got_error *err = NULL;
1333 	struct got_blob_object *blob1 = NULL;
1334 	int fd = -1, fd1 = -1, fd2 = -1;
1335 	FILE *ondisk_file = NULL;
1336 	char *label1 = NULL;
1337 	struct stat sb;
1338 	off_t size1 = 0;
1339 	int f2_exists = 0;
1340 	char *id_str = NULL;
1341 
1342 	memset(&sb, 0, sizeof(sb));
1343 
1344 	if (diff_staged) {
1345 		if (ct->staged_status != GOT_STATUS_MODIFY &&
1346 		    ct->staged_status != GOT_STATUS_ADD &&
1347 		    ct->staged_status != GOT_STATUS_DELETE)
1348 			return NULL;
1349 	} else {
1350 		if (ct->status != GOT_STATUS_MODIFY &&
1351 		    ct->status != GOT_STATUS_ADD &&
1352 		    ct->status != GOT_STATUS_DELETE &&
1353 		    ct->status != GOT_STATUS_CONFLICT)
1354 			return NULL;
1355 	}
1356 
1357 	err = got_opentemp_truncate(f1);
1358 	if (err)
1359 		return got_error_from_errno("got_opentemp_truncate");
1360 	err = got_opentemp_truncate(f2);
1361 	if (err)
1362 		return got_error_from_errno("got_opentemp_truncate");
1363 
1364 	if (!*diff_header_shown) {
1365 		err = got_object_id_str(&id_str, worktree->base_commit_id);
1366 		if (err)
1367 			return err;
1368 		fprintf(diff_outfile, "diff %s%s\n", diff_staged ? "-s " : "",
1369 		    got_worktree_get_root_path(worktree));
1370 		fprintf(diff_outfile, "commit - %s\n", id_str);
1371 		fprintf(diff_outfile, "path + %s%s\n",
1372 		    got_worktree_get_root_path(worktree),
1373 		    diff_staged ? " (staged changes)" : "");
1374 		*diff_header_shown = 1;
1375 	}
1376 
1377 	if (diff_staged) {
1378 		const char *label1 = NULL, *label2 = NULL;
1379 		switch (ct->staged_status) {
1380 		case GOT_STATUS_MODIFY:
1381 			label1 = ct->path;
1382 			label2 = ct->path;
1383 			break;
1384 		case GOT_STATUS_ADD:
1385 			label2 = ct->path;
1386 			break;
1387 		case GOT_STATUS_DELETE:
1388 			label1 = ct->path;
1389 			break;
1390 		default:
1391 			return got_error(GOT_ERR_FILE_STATUS);
1392 		}
1393 		fd1 = got_opentempfd();
1394 		if (fd1 == -1) {
1395 			err = got_error_from_errno("got_opentempfd");
1396 			goto done;
1397 		}
1398 		fd2 = got_opentempfd();
1399 		if (fd2 == -1) {
1400 			err = got_error_from_errno("got_opentempfd");
1401 			goto done;
1402 		}
1403 		err = got_diff_objects_as_blobs(NULL, NULL, f1, f2,
1404 		    fd1, fd2, ct->base_blob_id, ct->staged_blob_id,
1405 		    label1, label2, GOT_DIFF_ALGORITHM_PATIENCE, 3, 0, 0,
1406 		    NULL, repo, diff_outfile);
1407 		goto done;
1408 	}
1409 
1410 	fd1 = got_opentempfd();
1411 	if (fd1 == -1) {
1412 		err = got_error_from_errno("got_opentempfd");
1413 		goto done;
1414 	}
1415 
1416 	if (ct->status != GOT_STATUS_ADD) {
1417 		err = got_object_open_as_blob(&blob1, repo, ct->base_blob_id,
1418 		    8192, fd1);
1419 		if (err)
1420 			goto done;
1421 	}
1422 
1423 	if (ct->status != GOT_STATUS_DELETE) {
1424 		if (dirfd != -1) {
1425 			fd = openat(dirfd, de_name,
1426 			    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1427 			if (fd == -1) {
1428 				if (!got_err_open_nofollow_on_symlink()) {
1429 					err = got_error_from_errno2("openat",
1430 					    ct->ondisk_path);
1431 					goto done;
1432 				}
1433 				err = get_symlink_target_file(&fd, dirfd,
1434 				    de_name, ct->ondisk_path);
1435 				if (err)
1436 					goto done;
1437 			}
1438 		} else {
1439 			fd = open(ct->ondisk_path,
1440 			    O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1441 			if (fd == -1) {
1442 				if (!got_err_open_nofollow_on_symlink()) {
1443 					err = got_error_from_errno2("open",
1444 					    ct->ondisk_path);
1445 					goto done;
1446 				}
1447 				err = get_symlink_target_file(&fd, dirfd,
1448 				    de_name, ct->ondisk_path);
1449 				if (err)
1450 					goto done;
1451 			}
1452 		}
1453 		if (fstatat(fd, ct->ondisk_path, &sb,
1454 		    AT_SYMLINK_NOFOLLOW) == -1) {
1455 			err = got_error_from_errno2("fstatat", ct->ondisk_path);
1456 			goto done;
1457 		}
1458 		ondisk_file = fdopen(fd, "r");
1459 		if (ondisk_file == NULL) {
1460 			err = got_error_from_errno2("fdopen", ct->ondisk_path);
1461 			goto done;
1462 		}
1463 		fd = -1;
1464 		f2_exists = 1;
1465 	}
1466 
1467 	if (blob1) {
1468 		err = got_object_blob_dump_to_file(&size1, NULL, NULL,
1469 		    f1, blob1);
1470 		if (err)
1471 			goto done;
1472 	}
1473 
1474 	err = got_diff_blob_file(blob1, f1, size1, label1,
1475 	    ondisk_file ? ondisk_file : f2, f2_exists, &sb, ct->path,
1476 	    GOT_DIFF_ALGORITHM_PATIENCE, 3, 0, 0, NULL, diff_outfile);
1477 done:
1478 	if (fd1 != -1 && close(fd1) == -1 && err == NULL)
1479 		err = got_error_from_errno("close");
1480 	if (fd2 != -1 && close(fd2) == -1 && err == NULL)
1481 		err = got_error_from_errno("close");
1482 	if (blob1)
1483 		got_object_blob_close(blob1);
1484 	if (fd != -1 && close(fd) == -1 && err == NULL)
1485 		err = got_error_from_errno("close");
1486 	if (ondisk_file && fclose(ondisk_file) == EOF && err == NULL)
1487 		err = got_error_from_errno("fclose");
1488 	return err;
1489 }
1490 
1491 static const struct got_error *
1492 collect_commitables(void *arg, unsigned char status,
1493     unsigned char staged_status, const char *relpath,
1494     struct got_object_id *blob_id, struct got_object_id *staged_blob_id,
1495     struct got_object_id *commit_id, int dirfd, const char *de_name)
1496 {
1497 	struct collect_commitables_arg *a = arg;
1498 	const struct got_error *err = NULL;
1499 	struct got_commitable *ct = NULL;
1500 	struct got_pathlist_entry *new = NULL;
1501 	char *parent_path = NULL, *path = NULL;
1502 	struct stat sb;
1503 
1504 	if (a->have_staged_files) {
1505 		if (staged_status != GOT_STATUS_MODIFY &&
1506 		    staged_status != GOT_STATUS_ADD &&
1507 		    staged_status != GOT_STATUS_DELETE)
1508 			return NULL;
1509 	} else {
1510 		if (status == GOT_STATUS_CONFLICT && !a->commit_conflicts) {
1511 			printf("C  %s\n", relpath);
1512 			return got_error(GOT_ERR_COMMIT_CONFLICT);
1513 		}
1514 
1515 		if (status != GOT_STATUS_MODIFY &&
1516 		    status != GOT_STATUS_MODE_CHANGE &&
1517 		    status != GOT_STATUS_ADD &&
1518 		    status != GOT_STATUS_DELETE &&
1519 		    status != GOT_STATUS_CONFLICT)
1520 			return NULL;
1521 	}
1522 
1523 	if (asprintf(&path, "/%s", relpath) == -1) {
1524 		err = got_error_from_errno("asprintf");
1525 		goto done;
1526 	}
1527 	if (strcmp(path, "/") == 0) {
1528 		parent_path = strdup("");
1529 		if (parent_path == NULL)
1530 			return got_error_from_errno("strdup");
1531 	} else {
1532 		err = got_path_dirname(&parent_path, path);
1533 		if (err)
1534 			return err;
1535 	}
1536 
1537 	ct = calloc(1, sizeof(*ct));
1538 	if (ct == NULL) {
1539 		err = got_error_from_errno("calloc");
1540 		goto done;
1541 	}
1542 
1543 	if (asprintf(&ct->ondisk_path, "%s/%s", a->worktree->root_path,
1544 	    relpath) == -1) {
1545 		err = got_error_from_errno("asprintf");
1546 		goto done;
1547 	}
1548 
1549 	if (staged_status == GOT_STATUS_ADD ||
1550 	    staged_status == GOT_STATUS_MODIFY) {
1551 		struct got_fileindex_entry *ie;
1552 		ie = got_fileindex_entry_get(a->fileindex, path, strlen(path));
1553 		switch (got_fileindex_entry_staged_filetype_get(ie)) {
1554 		case GOT_FILEIDX_MODE_REGULAR_FILE:
1555 		case GOT_FILEIDX_MODE_BAD_SYMLINK:
1556 			ct->mode = S_IFREG;
1557 			break;
1558 		case GOT_FILEIDX_MODE_SYMLINK:
1559 			ct->mode = S_IFLNK;
1560 			break;
1561 		default:
1562 			err = got_error_path(path, GOT_ERR_BAD_FILETYPE);
1563 			goto done;
1564 		}
1565 		ct->mode |= got_fileindex_entry_perms_get(ie);
1566 	} else if (status != GOT_STATUS_DELETE &&
1567 	    staged_status != GOT_STATUS_DELETE) {
1568 		if (dirfd != -1) {
1569 			if (fstatat(dirfd, de_name, &sb,
1570 			    AT_SYMLINK_NOFOLLOW) == -1) {
1571 				err = got_error_from_errno2("fstatat",
1572 				    ct->ondisk_path);
1573 				goto done;
1574 			}
1575 		} else if (lstat(ct->ondisk_path, &sb) == -1) {
1576 			err = got_error_from_errno2("lstat", ct->ondisk_path);
1577 			goto done;
1578 		}
1579 		ct->mode = sb.st_mode;
1580 	}
1581 
1582 	if (asprintf(&ct->in_repo_path, "%s%s%s", a->worktree->path_prefix,
1583 	    got_path_is_root_dir(a->worktree->path_prefix) ? "" : "/",
1584 	    relpath) == -1) {
1585 		err = got_error_from_errno("asprintf");
1586 		goto done;
1587 	}
1588 
1589 	if (S_ISLNK(ct->mode) && staged_status == GOT_STATUS_NO_CHANGE &&
1590 	    status == GOT_STATUS_ADD && !a->allow_bad_symlinks) {
1591 		int is_bad_symlink;
1592 		char target_path[PATH_MAX];
1593 		ssize_t target_len;
1594 		target_len = readlink(ct->ondisk_path, target_path,
1595 		    sizeof(target_path));
1596 		if (target_len == -1) {
1597 			err = got_error_from_errno2("readlink",
1598 			    ct->ondisk_path);
1599 			goto done;
1600 		}
1601 		err = is_bad_symlink_target(&is_bad_symlink, target_path,
1602 		    target_len, ct->ondisk_path, a->worktree->root_path);
1603 		if (err)
1604 			goto done;
1605 		if (is_bad_symlink) {
1606 			err = got_error_path(ct->ondisk_path,
1607 			    GOT_ERR_BAD_SYMLINK);
1608 			goto done;
1609 		}
1610 	}
1611 
1612 	ct->status = status;
1613 	ct->staged_status = staged_status;
1614 	ct->blob_id = NULL; /* will be filled in when blob gets created */
1615 	if (ct->status != GOT_STATUS_ADD &&
1616 	    ct->staged_status != GOT_STATUS_ADD) {
1617 		ct->base_blob_id = got_object_id_dup(blob_id);
1618 		if (ct->base_blob_id == NULL) {
1619 			err = got_error_from_errno("got_object_id_dup");
1620 			goto done;
1621 		}
1622 		ct->base_commit_id = got_object_id_dup(commit_id);
1623 		if (ct->base_commit_id == NULL) {
1624 			err = got_error_from_errno("got_object_id_dup");
1625 			goto done;
1626 		}
1627 	}
1628 	if (ct->staged_status == GOT_STATUS_ADD ||
1629 	    ct->staged_status == GOT_STATUS_MODIFY) {
1630 		ct->staged_blob_id = got_object_id_dup(staged_blob_id);
1631 		if (ct->staged_blob_id == NULL) {
1632 			err = got_error_from_errno("got_object_id_dup");
1633 			goto done;
1634 		}
1635 	}
1636 	ct->path = strdup(path);
1637 	if (ct->path == NULL) {
1638 		err = got_error_from_errno("strdup");
1639 		goto done;
1640 	}
1641 	err = got_pathlist_insert(&new, a->commitable_paths, ct->path, ct);
1642 	if (err)
1643 		goto done;
1644 
1645 	if (a->diff_outfile && ct && new != NULL) {
1646 		err = append_ct_diff(ct, &a->diff_header_shown,
1647 		    a->diff_outfile, a->f1, a->f2, dirfd, de_name,
1648 		    a->have_staged_files, a->repo, a->worktree);
1649 		if (err)
1650 			goto done;
1651 	}
1652 done:
1653 	if (ct && (err || new == NULL))
1654 		free_commitable(ct);
1655 	free(parent_path);
1656 	free(path);
1657 	return err;
1658 }
1659 
1660 static const struct got_error *write_tree(struct got_object_id **, int *,
1661     struct got_tree_object *, const char *, struct got_pathlist_head *,
1662     got_worktree_status_cb status_cb, void *status_arg,
1663     struct got_repository *);
1664 
1665 static const struct got_error *
1666 write_subtree(struct got_object_id **new_subtree_id, int *nentries,
1667     struct got_tree_entry *te, const char *parent_path,
1668     struct got_pathlist_head *commitable_paths,
1669     got_worktree_status_cb status_cb, void *status_arg,
1670     struct got_repository *repo)
1671 {
1672 	const struct got_error *err = NULL;
1673 	struct got_tree_object *subtree;
1674 	char *subpath;
1675 
1676 	if (asprintf(&subpath, "%s%s%s", parent_path,
1677 	    got_path_is_root_dir(parent_path) ? "" : "/", te->name) == -1)
1678 		return got_error_from_errno("asprintf");
1679 
1680 	err = got_object_open_as_tree(&subtree, repo, &te->id);
1681 	if (err)
1682 		return err;
1683 
1684 	err = write_tree(new_subtree_id, nentries, subtree, subpath,
1685 	    commitable_paths, status_cb, status_arg, repo);
1686 	got_object_tree_close(subtree);
1687 	free(subpath);
1688 	return err;
1689 }
1690 
1691 static const struct got_error *
1692 match_ct_parent_path(int *match, struct got_commitable *ct, const char *path)
1693 {
1694 	const struct got_error *err = NULL;
1695 	char *ct_parent_path = NULL;
1696 
1697 	*match = 0;
1698 
1699 	if (strchr(ct->in_repo_path, '/') == NULL) {
1700 		*match = got_path_is_root_dir(path);
1701 		return NULL;
1702 	}
1703 
1704 	err = got_path_dirname(&ct_parent_path, ct->in_repo_path);
1705 	if (err)
1706 		return err;
1707 	*match = (strcmp(path, ct_parent_path) == 0);
1708 	free(ct_parent_path);
1709 	return err;
1710 }
1711 
1712 static mode_t
1713 get_ct_file_mode(struct got_commitable *ct)
1714 {
1715 	if (S_ISLNK(ct->mode))
1716 		return S_IFLNK;
1717 
1718 	return S_IFREG | (ct->mode & ((S_IRWXU | S_IRWXG | S_IRWXO)));
1719 }
1720 
1721 static const struct got_error *
1722 alloc_modified_blob_tree_entry(struct got_tree_entry **new_te,
1723     struct got_tree_entry *te, struct got_commitable *ct)
1724 {
1725 	const struct got_error *err = NULL;
1726 
1727 	*new_te = NULL;
1728 
1729 	err = got_object_tree_entry_dup(new_te, te);
1730 	if (err)
1731 		goto done;
1732 
1733 	(*new_te)->mode = get_ct_file_mode(ct);
1734 
1735 	if (ct->staged_status == GOT_STATUS_MODIFY)
1736 		memcpy(&(*new_te)->id, ct->staged_blob_id,
1737 		    sizeof((*new_te)->id));
1738 	else
1739 		memcpy(&(*new_te)->id, ct->blob_id, sizeof((*new_te)->id));
1740 done:
1741 	if (err && *new_te) {
1742 		free(*new_te);
1743 		*new_te = NULL;
1744 	}
1745 	return err;
1746 }
1747 
1748 static const struct got_error *
1749 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
1750     struct got_commitable *ct)
1751 {
1752 	const struct got_error *err = NULL;
1753 	char *ct_name = NULL;
1754 
1755 	 *new_te = NULL;
1756 
1757 	*new_te = calloc(1, sizeof(**new_te));
1758 	if (*new_te == NULL)
1759 		return got_error_from_errno("calloc");
1760 
1761 	err = got_path_basename(&ct_name, ct->path);
1762 	if (err)
1763 		goto done;
1764 	if (strlcpy((*new_te)->name, ct_name, sizeof((*new_te)->name)) >=
1765 	    sizeof((*new_te)->name)) {
1766 		err = got_error(GOT_ERR_NO_SPACE);
1767 		goto done;
1768 	}
1769 
1770 	(*new_te)->mode = get_ct_file_mode(ct);
1771 
1772 	if (ct->staged_status == GOT_STATUS_ADD)
1773 		memcpy(&(*new_te)->id, ct->staged_blob_id,
1774 		    sizeof((*new_te)->id));
1775 	else
1776 		memcpy(&(*new_te)->id, ct->blob_id, sizeof((*new_te)->id));
1777 done:
1778 	free(ct_name);
1779 	if (err && *new_te) {
1780 		free(*new_te);
1781 		*new_te = NULL;
1782 	}
1783 	return err;
1784 }
1785 
1786 static const struct got_error *
1787 insert_tree_entry(struct got_tree_entry *new_te,
1788     struct got_pathlist_head *paths)
1789 {
1790 	const struct got_error *err = NULL;
1791 	struct got_pathlist_entry *new_pe;
1792 
1793 	err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
1794 	if (err)
1795 		return err;
1796 	if (new_pe == NULL)
1797 		return got_error(GOT_ERR_TREE_DUP_ENTRY);
1798 	return NULL;
1799 }
1800 
1801 static const struct got_error *
1802 report_ct_status(struct got_commitable *ct,
1803     got_worktree_status_cb status_cb, void *status_arg)
1804 {
1805 	const char *ct_path = ct->path;
1806 	unsigned char status;
1807 
1808 	if (status_cb == NULL) /* no commit progress output desired */
1809 		return NULL;
1810 
1811 	while (ct_path[0] == '/')
1812 		ct_path++;
1813 
1814 	if (ct->staged_status != GOT_STATUS_NO_CHANGE)
1815 		status = ct->staged_status;
1816 	else
1817 		status = ct->status;
1818 
1819 	return (*status_cb)(status_arg, status, GOT_STATUS_NO_CHANGE,
1820 	    ct_path, ct->blob_id, NULL, NULL, -1, NULL);
1821 }
1822 
1823 static const struct got_error *
1824 match_modified_subtree(int *modified, struct got_tree_entry *te,
1825     const char *base_tree_path, struct got_pathlist_head *commitable_paths)
1826 {
1827 	const struct got_error *err = NULL;
1828 	struct got_pathlist_entry *pe;
1829 	char *te_path;
1830 
1831 	*modified = 0;
1832 
1833 	if (asprintf(&te_path, "%s%s%s", base_tree_path,
1834 	    got_path_is_root_dir(base_tree_path) ? "" : "/",
1835 	    te->name) == -1)
1836 		return got_error_from_errno("asprintf");
1837 
1838 	TAILQ_FOREACH(pe, commitable_paths, entry) {
1839 		struct got_commitable *ct = pe->data;
1840 		*modified = got_path_is_child(ct->in_repo_path, te_path,
1841 		    strlen(te_path));
1842 		if (*modified)
1843 			break;
1844 	}
1845 
1846 	free(te_path);
1847 	return err;
1848 }
1849 
1850 static const struct got_error *
1851 match_deleted_or_modified_ct(struct got_commitable **ctp,
1852     struct got_tree_entry *te, const char *base_tree_path,
1853     struct got_pathlist_head *commitable_paths)
1854 {
1855 	const struct got_error *err = NULL;
1856 	struct got_pathlist_entry *pe;
1857 
1858 	*ctp = NULL;
1859 
1860 	TAILQ_FOREACH(pe, commitable_paths, entry) {
1861 		struct got_commitable *ct = pe->data;
1862 		char *ct_name = NULL;
1863 		int path_matches;
1864 
1865 		if (ct->staged_status == GOT_STATUS_NO_CHANGE) {
1866 			if (ct->status != GOT_STATUS_MODIFY &&
1867 			    ct->status != GOT_STATUS_MODE_CHANGE &&
1868 			    ct->status != GOT_STATUS_DELETE &&
1869 			    ct->status != GOT_STATUS_CONFLICT)
1870 				continue;
1871 		} else {
1872 			if (ct->staged_status != GOT_STATUS_MODIFY &&
1873 			    ct->staged_status != GOT_STATUS_DELETE)
1874 				continue;
1875 		}
1876 
1877 		if (got_object_id_cmp(ct->base_blob_id, &te->id) != 0)
1878 			continue;
1879 
1880 		err = match_ct_parent_path(&path_matches, ct, base_tree_path);
1881 		if (err)
1882 			return err;
1883 		if (!path_matches)
1884 			continue;
1885 
1886 		err = got_path_basename(&ct_name, pe->path);
1887 		if (err)
1888 			return err;
1889 
1890 		if (strcmp(te->name, ct_name) != 0) {
1891 			free(ct_name);
1892 			continue;
1893 		}
1894 		free(ct_name);
1895 
1896 		*ctp = ct;
1897 		break;
1898 	}
1899 
1900 	return err;
1901 }
1902 
1903 static const struct got_error *
1904 make_subtree_for_added_blob(struct got_tree_entry **new_tep,
1905     const char *child_path, const char *path_base_tree,
1906     struct got_pathlist_head *commitable_paths,
1907     got_worktree_status_cb status_cb, void *status_arg,
1908     struct got_repository *repo)
1909 {
1910 	const struct got_error *err = NULL;
1911 	struct got_tree_entry *new_te;
1912 	char *subtree_path;
1913 	struct got_object_id *id = NULL;
1914 	int nentries;
1915 
1916 	*new_tep = NULL;
1917 
1918 	if (asprintf(&subtree_path, "%s%s%s", path_base_tree,
1919 	    got_path_is_root_dir(path_base_tree) ? "" : "/",
1920 	    child_path) == -1)
1921 		return got_error_from_errno("asprintf");
1922 
1923 	new_te = calloc(1, sizeof(*new_te));
1924 	if (new_te == NULL)
1925 		return got_error_from_errno("calloc");
1926 	new_te->mode = S_IFDIR;
1927 
1928 	if (strlcpy(new_te->name, child_path, sizeof(new_te->name)) >=
1929 	    sizeof(new_te->name)) {
1930 		err = got_error(GOT_ERR_NO_SPACE);
1931 		goto done;
1932 	}
1933 	err = write_tree(&id, &nentries, NULL, subtree_path,
1934 	    commitable_paths, status_cb, status_arg, repo);
1935 	if (err) {
1936 		free(new_te);
1937 		goto done;
1938 	}
1939 	memcpy(&new_te->id, id, sizeof(new_te->id));
1940 done:
1941 	free(id);
1942 	free(subtree_path);
1943 	if (err == NULL)
1944 		*new_tep = new_te;
1945 	return err;
1946 }
1947 
1948 static const struct got_error *
1949 write_tree(struct got_object_id **new_tree_id, int *nentries,
1950     struct got_tree_object *base_tree, const char *path_base_tree,
1951     struct got_pathlist_head *commitable_paths,
1952     got_worktree_status_cb status_cb, void *status_arg,
1953     struct got_repository *repo)
1954 {
1955 	const struct got_error *err = NULL;
1956 	struct got_pathlist_head paths;
1957 	struct got_tree_entry *te, *new_te = NULL;
1958 	struct got_pathlist_entry *pe;
1959 
1960 	TAILQ_INIT(&paths);
1961 	*nentries = 0;
1962 
1963 	/* Insert, and recurse into, newly added entries first. */
1964 	TAILQ_FOREACH(pe, commitable_paths, entry) {
1965 		struct got_commitable *ct = pe->data;
1966 		char *child_path = NULL, *slash;
1967 
1968 		if ((ct->status != GOT_STATUS_ADD &&
1969 		    ct->staged_status != GOT_STATUS_ADD) ||
1970 		    (ct->flags & GOT_COMMITABLE_ADDED))
1971 			continue;
1972 
1973 		if (!got_path_is_child(ct->in_repo_path, path_base_tree,
1974 		    strlen(path_base_tree)))
1975 			continue;
1976 
1977 		err = got_path_skip_common_ancestor(&child_path, path_base_tree,
1978 		    ct->in_repo_path);
1979 		if (err)
1980 			goto done;
1981 
1982 		slash = strchr(child_path, '/');
1983 		if (slash == NULL) {
1984 			err = alloc_added_blob_tree_entry(&new_te, ct);
1985 			if (err)
1986 				goto done;
1987 			err = report_ct_status(ct, status_cb, status_arg);
1988 			if (err)
1989 				goto done;
1990 			ct->flags |= GOT_COMMITABLE_ADDED;
1991 			err = insert_tree_entry(new_te, &paths);
1992 			if (err)
1993 				goto done;
1994 			(*nentries)++;
1995 		} else {
1996 			*slash = '\0'; /* trim trailing path components */
1997 			if (base_tree == NULL ||
1998 			    got_object_tree_find_entry(base_tree, child_path)
1999 			    == NULL) {
2000 				err = make_subtree_for_added_blob(&new_te,
2001 				    child_path, path_base_tree,
2002 				    commitable_paths, status_cb, status_arg,
2003 				    repo);
2004 				if (err)
2005 					goto done;
2006 				err = insert_tree_entry(new_te, &paths);
2007 				if (err)
2008 					goto done;
2009 				(*nentries)++;
2010 			}
2011 		}
2012 	}
2013 
2014 	if (base_tree) {
2015 		int i, nbase_entries;
2016 		/* Handle modified and deleted entries. */
2017 		nbase_entries = got_object_tree_get_nentries(base_tree);
2018 		for (i = 0; i < nbase_entries; i++) {
2019 			struct got_commitable *ct = NULL;
2020 
2021 			te = got_object_tree_get_entry(base_tree, i);
2022 			if (got_object_tree_entry_is_submodule(te)) {
2023 				/* Entry is a submodule; just copy it. */
2024 				err = got_object_tree_entry_dup(&new_te, te);
2025 				if (err)
2026 					goto done;
2027 				err = insert_tree_entry(new_te, &paths);
2028 				if (err)
2029 					goto done;
2030 				(*nentries)++;
2031 				continue;
2032 			}
2033 
2034 			if (S_ISDIR(te->mode)) {
2035 				int modified;
2036 				err = got_object_tree_entry_dup(&new_te, te);
2037 				if (err)
2038 					goto done;
2039 				err = match_modified_subtree(&modified, te,
2040 				    path_base_tree, commitable_paths);
2041 				if (err)
2042 					goto done;
2043 				/* Avoid recursion into unmodified subtrees. */
2044 				if (modified) {
2045 					struct got_object_id *new_id;
2046 					int nsubentries;
2047 					err = write_subtree(&new_id,
2048 					    &nsubentries, te,
2049 					    path_base_tree, commitable_paths,
2050 					    status_cb, status_arg, repo);
2051 					if (err)
2052 						goto done;
2053 					if (nsubentries == 0) {
2054 						/* All entries were deleted. */
2055 						free(new_id);
2056 						continue;
2057 					}
2058 					memcpy(&new_te->id, new_id,
2059 					    sizeof(new_te->id));
2060 					free(new_id);
2061 				}
2062 				err = insert_tree_entry(new_te, &paths);
2063 				if (err)
2064 					goto done;
2065 				(*nentries)++;
2066 				continue;
2067 			}
2068 
2069 			err = match_deleted_or_modified_ct(&ct, te,
2070 			    path_base_tree, commitable_paths);
2071 			if (err)
2072 				goto done;
2073 			if (ct) {
2074 				/* NB: Deleted entries get dropped here. */
2075 				if (ct->status == GOT_STATUS_MODIFY ||
2076 				    ct->status == GOT_STATUS_MODE_CHANGE ||
2077 				    ct->status == GOT_STATUS_CONFLICT ||
2078 				    ct->staged_status == GOT_STATUS_MODIFY) {
2079 					err = alloc_modified_blob_tree_entry(
2080 					    &new_te, te, ct);
2081 					if (err)
2082 						goto done;
2083 					err = insert_tree_entry(new_te, &paths);
2084 					if (err)
2085 						goto done;
2086 					(*nentries)++;
2087 				}
2088 				err = report_ct_status(ct, status_cb,
2089 				    status_arg);
2090 				if (err)
2091 					goto done;
2092 			} else {
2093 				/* Entry is unchanged; just copy it. */
2094 				err = got_object_tree_entry_dup(&new_te, te);
2095 				if (err)
2096 					goto done;
2097 				err = insert_tree_entry(new_te, &paths);
2098 				if (err)
2099 					goto done;
2100 				(*nentries)++;
2101 			}
2102 		}
2103 	}
2104 
2105 	/* Write new list of entries; deleted entries have been dropped. */
2106 	err = got_object_tree_create(new_tree_id, &paths, *nentries, repo);
2107 done:
2108 	got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2109 	return err;
2110 }
2111 
2112 static const struct got_error *
2113 update_fileindex_after_commit(struct got_worktree *worktree,
2114     struct got_pathlist_head *commitable_paths,
2115     struct got_object_id *new_base_commit_id,
2116     struct got_fileindex *fileindex, int have_staged_files)
2117 {
2118 	const struct got_error *err = NULL;
2119 	struct got_pathlist_entry *pe;
2120 	char *relpath = NULL;
2121 
2122 	TAILQ_FOREACH(pe, commitable_paths, entry) {
2123 		struct got_fileindex_entry *ie;
2124 		struct got_commitable *ct = pe->data;
2125 
2126 		ie = got_fileindex_entry_get(fileindex, pe->path, pe->path_len);
2127 
2128 		err = got_path_skip_common_ancestor(&relpath,
2129 		    worktree->root_path, ct->ondisk_path);
2130 		if (err)
2131 			goto done;
2132 
2133 		if (ie) {
2134 			if (ct->status == GOT_STATUS_DELETE ||
2135 			    ct->staged_status == GOT_STATUS_DELETE) {
2136 				got_fileindex_entry_remove(fileindex, ie);
2137 			} else if (ct->staged_status == GOT_STATUS_ADD ||
2138 			    ct->staged_status == GOT_STATUS_MODIFY) {
2139 				got_fileindex_entry_stage_set(ie,
2140 				    GOT_FILEIDX_STAGE_NONE);
2141 				got_fileindex_entry_staged_filetype_set(ie, 0);
2142 
2143 				err = got_fileindex_entry_update(ie,
2144 				    worktree->root_fd, relpath,
2145 				    ct->staged_blob_id->sha1,
2146 				    new_base_commit_id->sha1,
2147 				    !have_staged_files);
2148 			} else
2149 				err = got_fileindex_entry_update(ie,
2150 				    worktree->root_fd, relpath,
2151 				    ct->blob_id->sha1,
2152 				    new_base_commit_id->sha1,
2153 				    !have_staged_files);
2154 		} else {
2155 			err = got_fileindex_entry_alloc(&ie, pe->path);
2156 			if (err)
2157 				goto done;
2158 			err = got_fileindex_entry_update(ie,
2159 			    worktree->root_fd, relpath, ct->blob_id->sha1,
2160 			    new_base_commit_id->sha1, 1);
2161 			if (err) {
2162 				got_fileindex_entry_free(ie);
2163 				goto done;
2164 			}
2165 			err = got_fileindex_entry_add(fileindex, ie);
2166 			if (err) {
2167 				got_fileindex_entry_free(ie);
2168 				goto done;
2169 			}
2170 		}
2171 		free(relpath);
2172 		relpath = NULL;
2173 	}
2174 done:
2175 	free(relpath);
2176 	return err;
2177 }
2178 
2179 static const struct got_error *
2180 check_out_of_date(const char *in_repo_path, unsigned char status,
2181     unsigned char staged_status, struct got_object_id *base_blob_id,
2182     struct got_object_id *base_commit_id,
2183     struct got_object_id *head_commit_id, struct got_repository *repo,
2184     int ood_errcode)
2185 {
2186 	const struct got_error *err = NULL;
2187 	struct got_commit_object *commit = NULL;
2188 	struct got_object_id *id = NULL;
2189 
2190 	if (status != GOT_STATUS_ADD && staged_status != GOT_STATUS_ADD) {
2191 		/* Trivial case: base commit == head commit */
2192 		if (got_object_id_cmp(base_commit_id, head_commit_id) == 0)
2193 			return NULL;
2194 		/*
2195 		 * Ensure file content which local changes were based
2196 		 * on matches file content in the branch head.
2197 		 */
2198 		err = got_object_open_as_commit(&commit, repo, head_commit_id);
2199 		if (err)
2200 			goto done;
2201 		err = got_object_id_by_path(&id, repo, commit, in_repo_path);
2202 		if (err) {
2203 			if (err->code == GOT_ERR_NO_TREE_ENTRY)
2204 				err = got_error(ood_errcode);
2205 			goto done;
2206 		} else if (got_object_id_cmp(id, base_blob_id) != 0)
2207 			err = got_error(ood_errcode);
2208 	} else {
2209 		/* Require that added files don't exist in the branch head. */
2210 		err = got_object_open_as_commit(&commit, repo, head_commit_id);
2211 		if (err)
2212 			goto done;
2213 		err = got_object_id_by_path(&id, repo, commit, in_repo_path);
2214 		if (err && err->code != GOT_ERR_NO_TREE_ENTRY)
2215 			goto done;
2216 		err = id ? got_error(ood_errcode) : NULL;
2217 	}
2218 done:
2219 	free(id);
2220 	if (commit)
2221 		got_object_commit_close(commit);
2222 	return err;
2223 }
2224 
2225 static const struct got_error *
2226 commit_worktree(struct got_object_id **new_commit_id,
2227     struct got_pathlist_head *commitable_paths,
2228     struct got_object_id *head_commit_id,
2229     struct got_object_id *parent_id2,
2230     struct got_worktree *worktree,
2231     const char *author, const char *committer, char *diff_path,
2232     got_worktree_commit_msg_cb commit_msg_cb, void *commit_arg,
2233     got_worktree_status_cb status_cb, void *status_arg,
2234     struct got_repository *repo)
2235 {
2236 	const struct got_error *err = NULL;
2237 	struct got_pathlist_entry *pe;
2238 	struct got_commit_object *head_commit = NULL;
2239 	struct got_tree_object *head_tree = NULL;
2240 	struct got_object_id *new_tree_id = NULL;
2241 	int nentries, nparents = 0;
2242 	struct got_object_id_queue parent_ids;
2243 	struct got_object_qid *pid = NULL;
2244 	char *logmsg = NULL;
2245 	time_t timestamp;
2246 
2247 	*new_commit_id = NULL;
2248 
2249 	STAILQ_INIT(&parent_ids);
2250 
2251 	err = got_object_open_as_commit(&head_commit, repo, head_commit_id);
2252 	if (err)
2253 		goto done;
2254 
2255 	err = got_object_open_as_tree(&head_tree, repo, head_commit->tree_id);
2256 	if (err)
2257 		goto done;
2258 
2259 	if (commit_msg_cb != NULL) {
2260 		err = commit_msg_cb(commitable_paths, diff_path,
2261 		    &logmsg, commit_arg);
2262 		if (err)
2263 			goto done;
2264 	}
2265 
2266 	if (logmsg == NULL || strlen(logmsg) == 0) {
2267 		err = got_error(GOT_ERR_COMMIT_MSG_EMPTY);
2268 		goto done;
2269 	}
2270 
2271 	/* Create blobs from added and modified files and record their IDs. */
2272 	TAILQ_FOREACH(pe, commitable_paths, entry) {
2273 		struct got_commitable *ct = pe->data;
2274 		char *ondisk_path;
2275 
2276 		/* Blobs for staged files already exist. */
2277 		if (ct->staged_status == GOT_STATUS_ADD ||
2278 		    ct->staged_status == GOT_STATUS_MODIFY)
2279 			continue;
2280 
2281 		if (ct->status != GOT_STATUS_ADD &&
2282 		    ct->status != GOT_STATUS_MODIFY &&
2283 		    ct->status != GOT_STATUS_MODE_CHANGE &&
2284 		    ct->status != GOT_STATUS_CONFLICT)
2285 			continue;
2286 
2287 		if (asprintf(&ondisk_path, "%s/%s",
2288 		    worktree->root_path, pe->path) == -1) {
2289 			err = got_error_from_errno("asprintf");
2290 			goto done;
2291 		}
2292 		err = got_object_blob_create(&ct->blob_id, ondisk_path, repo);
2293 		free(ondisk_path);
2294 		if (err)
2295 			goto done;
2296 	}
2297 
2298 	/* Recursively write new tree objects. */
2299 	err = write_tree(&new_tree_id, &nentries, head_tree, "/",
2300 	    commitable_paths, status_cb, status_arg, repo);
2301 	if (err)
2302 		goto done;
2303 
2304 	err = got_object_qid_alloc(&pid, head_commit_id);
2305 	if (err)
2306 		goto done;
2307 	STAILQ_INSERT_TAIL(&parent_ids, pid, entry);
2308 	nparents++;
2309 	if (parent_id2) {
2310 		err = got_object_qid_alloc(&pid, parent_id2);
2311 		if (err)
2312 			goto done;
2313 		STAILQ_INSERT_TAIL(&parent_ids, pid, entry);
2314 		nparents++;
2315 	}
2316 	timestamp = time(NULL);
2317 	err = got_object_commit_create(new_commit_id, new_tree_id, &parent_ids,
2318 	    nparents, author, timestamp, committer, timestamp, logmsg, repo);
2319 	if (logmsg != NULL)
2320 		free(logmsg);
2321 	if (err)
2322 		goto done;
2323 done:
2324 	got_object_id_queue_free(&parent_ids);
2325 	if (head_tree)
2326 		got_object_tree_close(head_tree);
2327 	if (head_commit)
2328 		got_object_commit_close(head_commit);
2329 	return err;
2330 }
2331 
2332 static const struct got_error *
2333 check_path_is_commitable(const char *path,
2334     struct got_pathlist_head *commitable_paths)
2335 {
2336 	struct got_pathlist_entry *cpe = NULL;
2337 	size_t path_len = strlen(path);
2338 
2339 	TAILQ_FOREACH(cpe, commitable_paths, entry) {
2340 		struct got_commitable *ct = cpe->data;
2341 		const char *ct_path = ct->path;
2342 
2343 		while (ct_path[0] == '/')
2344 			ct_path++;
2345 
2346 		if (strcmp(path, ct_path) == 0 ||
2347 		    got_path_is_child(ct_path, path, path_len))
2348 			break;
2349 	}
2350 
2351 	if (cpe == NULL)
2352 		return got_error_path(path, GOT_ERR_BAD_PATH);
2353 
2354 	return NULL;
2355 }
2356 
2357 static const struct got_error *
2358 check_staged_file(void *arg, struct got_fileindex_entry *ie)
2359 {
2360 	int *have_staged_files = arg;
2361 
2362 	if (got_fileindex_entry_stage_get(ie) != GOT_FILEIDX_STAGE_NONE) {
2363 		*have_staged_files = 1;
2364 		return got_error(GOT_ERR_CANCELLED);
2365 	}
2366 
2367 	return NULL;
2368 }
2369 
2370 static const struct got_error *
2371 check_non_staged_files(struct got_fileindex *fileindex,
2372     struct got_pathlist_head *paths)
2373 {
2374 	struct got_pathlist_entry *pe;
2375 	struct got_fileindex_entry *ie;
2376 
2377 	TAILQ_FOREACH(pe, paths, entry) {
2378 		if (pe->path[0] == '\0')
2379 			continue;
2380 		ie = got_fileindex_entry_get(fileindex, pe->path, pe->path_len);
2381 		if (ie == NULL)
2382 			return got_error_path(pe->path, GOT_ERR_BAD_PATH);
2383 		if (got_fileindex_entry_stage_get(ie) == GOT_FILEIDX_STAGE_NONE)
2384 			return got_error_path(pe->path,
2385 			    GOT_ERR_FILE_NOT_STAGED);
2386 	}
2387 
2388 	return NULL;
2389 }
2390 
2391 static void
2392 print_load_info(int print_colored, int print_found, int print_trees,
2393     int ncolored, int nfound, int ntrees)
2394 {
2395 	if (print_colored) {
2396 		printf("%d commit%s colored", ncolored,
2397 		    ncolored == 1 ? "" : "s");
2398 	}
2399 	if (print_found) {
2400 		printf("%s%d object%s found",
2401 		    ncolored > 0 ? "; " : "",
2402 		    nfound, nfound == 1 ? "" : "s");
2403 	}
2404 	if (print_trees) {
2405 		printf("; %d tree%s scanned", ntrees,
2406 		    ntrees == 1 ? "" : "s");
2407 	}
2408 }
2409 
2410 struct got_send_progress_arg {
2411 	char last_scaled_packsize[FMT_SCALED_STRSIZE];
2412 	int verbosity;
2413 	int last_ncolored;
2414 	int last_nfound;
2415 	int last_ntrees;
2416 	int loading_done;
2417 	int last_ncommits;
2418 	int last_nobj_total;
2419 	int last_p_deltify;
2420 	int last_p_written;
2421 	int last_p_sent;
2422 	int printed_something;
2423 	int sent_something;
2424 	struct got_pathlist_head *delete_branches;
2425 };
2426 
2427 static const struct got_error *
2428 send_progress(void *arg, int ncolored, int nfound, int ntrees,
2429     off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
2430     int nobj_written, off_t bytes_sent, const char *refname,
2431     const char *errmsg, int success)
2432 {
2433 	struct got_send_progress_arg *a = arg;
2434 	char scaled_packsize[FMT_SCALED_STRSIZE];
2435 	char scaled_sent[FMT_SCALED_STRSIZE];
2436 	int p_deltify = 0, p_written = 0, p_sent = 0;
2437 	int print_colored = 0, print_found = 0, print_trees = 0;
2438 	int print_searching = 0, print_total = 0;
2439 	int print_deltify = 0, print_written = 0, print_sent = 0;
2440 
2441 	if (a->verbosity < 0)
2442 		return NULL;
2443 
2444 	if (refname) {
2445 		const char *status = success ? "accepted" : "rejected";
2446 
2447 		if (success) {
2448 			struct got_pathlist_entry *pe;
2449 			TAILQ_FOREACH(pe, a->delete_branches, entry) {
2450 				const char *branchname = pe->path;
2451 				if (got_path_cmp(branchname, refname,
2452 				    strlen(branchname), strlen(refname)) == 0) {
2453 					status = "deleted";
2454 					a->sent_something = 1;
2455 					break;
2456 				}
2457 			}
2458 		}
2459 
2460 		if (a->printed_something)
2461 			putchar('\n');
2462 		printf("Server has %s %s", status, refname);
2463 		if (errmsg)
2464 			printf(": %s", errmsg);
2465 		a->printed_something = 1;
2466 		return NULL;
2467 	}
2468 
2469 	if (a->last_ncolored != ncolored) {
2470 		print_colored = 1;
2471 		a->last_ncolored = ncolored;
2472 	}
2473 
2474 	if (a->last_nfound != nfound) {
2475 		print_colored = 1;
2476 		print_found = 1;
2477 		a->last_nfound = nfound;
2478 	}
2479 
2480 	if (a->last_ntrees != ntrees) {
2481 		print_colored = 1;
2482 		print_found = 1;
2483 		print_trees = 1;
2484 		a->last_ntrees = ntrees;
2485 	}
2486 
2487 	if ((print_colored || print_found || print_trees) &&
2488 	    !a->loading_done) {
2489 		printf("\r");
2490 		print_load_info(print_colored, print_found, print_trees,
2491 		    ncolored, nfound, ntrees);
2492 		a->printed_something = 1;
2493 		fflush(stdout);
2494 		return NULL;
2495 	} else if (!a->loading_done) {
2496 		printf("\r");
2497 		print_load_info(1, 1, 1, ncolored, nfound, ntrees);
2498 		printf("\n");
2499 		a->loading_done = 1;
2500 	}
2501 
2502 	if (fmt_scaled(packfile_size, scaled_packsize) == -1)
2503 		return got_error_from_errno("fmt_scaled");
2504 	if (fmt_scaled(bytes_sent, scaled_sent) == -1)
2505 		return got_error_from_errno("fmt_scaled");
2506 
2507 	if (a->last_ncommits != ncommits) {
2508 		print_searching = 1;
2509 		a->last_ncommits = ncommits;
2510 	}
2511 
2512 	if (a->last_nobj_total != nobj_total) {
2513 		print_searching = 1;
2514 		print_total = 1;
2515 		a->last_nobj_total = nobj_total;
2516 	}
2517 
2518 	if (packfile_size > 0 && (a->last_scaled_packsize[0] == '\0' ||
2519 	    strcmp(scaled_packsize, a->last_scaled_packsize)) != 0) {
2520 		if (strlcpy(a->last_scaled_packsize, scaled_packsize,
2521 		    FMT_SCALED_STRSIZE) >= FMT_SCALED_STRSIZE)
2522 			return got_error(GOT_ERR_NO_SPACE);
2523 	}
2524 
2525 	if (nobj_deltify > 0 || nobj_written > 0) {
2526 		if (nobj_deltify > 0) {
2527 			p_deltify = (nobj_deltify * 100) / nobj_total;
2528 			if (p_deltify != a->last_p_deltify) {
2529 				a->last_p_deltify = p_deltify;
2530 				print_searching = 1;
2531 				print_total = 1;
2532 				print_deltify = 1;
2533 			}
2534 		}
2535 		if (nobj_written > 0) {
2536 			p_written = (nobj_written * 100) / nobj_total;
2537 			if (p_written != a->last_p_written) {
2538 				a->last_p_written = p_written;
2539 				print_searching = 1;
2540 				print_total = 1;
2541 				print_deltify = 1;
2542 				print_written = 1;
2543 			}
2544 		}
2545 	}
2546 
2547 	if (bytes_sent > 0) {
2548 		p_sent = (bytes_sent * 100) / packfile_size;
2549 		if (p_sent != a->last_p_sent) {
2550 			a->last_p_sent = p_sent;
2551 			print_searching = 1;
2552 			print_total = 1;
2553 			print_deltify = 1;
2554 			print_written = 1;
2555 			print_sent = 1;
2556 		}
2557 		a->sent_something = 1;
2558 	}
2559 
2560 	if (print_searching || print_total || print_deltify || print_written ||
2561 	    print_sent)
2562 		printf("\r");
2563 	if (print_searching)
2564 		printf("packing %d reference%s", ncommits,
2565 		    ncommits == 1 ? "" : "s");
2566 	if (print_total)
2567 		printf("; %d object%s", nobj_total,
2568 		    nobj_total == 1 ? "" : "s");
2569 	if (print_deltify)
2570 		printf("; deltify: %d%%", p_deltify);
2571 	if (print_sent)
2572 		printf("; uploading pack: %*s %d%%", FMT_SCALED_STRSIZE - 2,
2573 		    scaled_packsize, p_sent);
2574 	else if (print_written)
2575 		printf("; writing pack: %*s %d%%", FMT_SCALED_STRSIZE - 2,
2576 		    scaled_packsize, p_written);
2577 	if (print_searching || print_total || print_deltify ||
2578 	    print_written || print_sent) {
2579 		a->printed_something = 1;
2580 		fflush(stdout);
2581 	}
2582 	return NULL;
2583 }
2584 
2585 struct got_fetch_progress_arg {
2586 	char last_scaled_size[FMT_SCALED_STRSIZE];
2587 	int last_p_indexed;
2588 	int last_p_resolved;
2589 	int verbosity;
2590 
2591 	struct got_repository *repo;
2592 };
2593 
2594 static const struct got_error *
2595 fetch_progress(void *arg, const char *message, off_t packfile_size,
2596     int nobj_total, int nobj_indexed, int nobj_loose, int nobj_resolved)
2597 {
2598 	struct got_fetch_progress_arg *a = arg;
2599 	char scaled_size[FMT_SCALED_STRSIZE];
2600 	int p_indexed, p_resolved;
2601 	int print_size = 0, print_indexed = 0, print_resolved = 0;
2602 
2603 	if (a->verbosity < 0)
2604 		return NULL;
2605 
2606 	if (message && message[0] != '\0') {
2607 		printf("\rserver: %s", message);
2608 		fflush(stdout);
2609 		return NULL;
2610 	}
2611 
2612 	if (packfile_size > 0 || nobj_indexed > 0) {
2613 		if (fmt_scaled(packfile_size, scaled_size) == 0 &&
2614 		    (a->last_scaled_size[0] == '\0' ||
2615 		    strcmp(scaled_size, a->last_scaled_size)) != 0) {
2616 			print_size = 1;
2617 			if (strlcpy(a->last_scaled_size, scaled_size,
2618 			    FMT_SCALED_STRSIZE) >= FMT_SCALED_STRSIZE)
2619 				return got_error(GOT_ERR_NO_SPACE);
2620 		}
2621 		if (nobj_indexed > 0) {
2622 			p_indexed = (nobj_indexed * 100) / nobj_total;
2623 			if (p_indexed != a->last_p_indexed) {
2624 				a->last_p_indexed = p_indexed;
2625 				print_indexed = 1;
2626 				print_size = 1;
2627 			}
2628 		}
2629 		if (nobj_resolved > 0) {
2630 			p_resolved = (nobj_resolved * 100) /
2631 			    (nobj_total - nobj_loose);
2632 			if (p_resolved != a->last_p_resolved) {
2633 				a->last_p_resolved = p_resolved;
2634 				print_resolved = 1;
2635 				print_indexed = 1;
2636 				print_size = 1;
2637 			}
2638 		}
2639 
2640 	}
2641 	if (print_size || print_indexed || print_resolved)
2642 		printf("\r");
2643 	if (print_size)
2644 		printf("%*s fetched", FMT_SCALED_STRSIZE - 2, scaled_size);
2645 	if (print_indexed)
2646 		printf("; indexing %d%%", p_indexed);
2647 	if (print_resolved)
2648 		printf("; resolving deltas %d%%", p_resolved);
2649 	if (print_size || print_indexed || print_resolved) {
2650 		putchar('\n');
2651 		fflush(stdout);
2652 	}
2653 
2654 	return NULL;
2655 }
2656 
2657 static const struct got_error *
2658 create_symref(const char *refname, struct got_reference *target_ref,
2659     int verbosity, struct got_repository *repo)
2660 {
2661 	const struct got_error *err;
2662 	struct got_reference *head_symref;
2663 
2664 	err = got_ref_alloc_symref(&head_symref, refname, target_ref);
2665 	if (err)
2666 		return err;
2667 
2668 	err = got_ref_write(head_symref, repo);
2669 	if (err == NULL && verbosity > 0) {
2670 		printf("Created reference %s: %s\n", GOT_REF_HEAD,
2671 		    got_ref_get_name(target_ref));
2672 	}
2673 	got_ref_close(head_symref);
2674 	return err;
2675 }
2676 
2677 static const struct got_error *
2678 create_ref(const char *refname, struct got_object_id *id,
2679     int verbosity, struct got_repository *repo)
2680 {
2681 	const struct got_error *err = NULL;
2682 	struct got_reference *ref;
2683 	char *id_str;
2684 
2685 	err = got_object_id_str(&id_str, id);
2686 	if (err)
2687 		return err;
2688 
2689 	err = got_ref_alloc(&ref, refname, id);
2690 	if (err)
2691 		goto done;
2692 
2693 	err = got_ref_write(ref, repo);
2694 	got_ref_close(ref);
2695 
2696 	if (err == NULL && verbosity >= 0)
2697 		printf("Created reference %s: %s\n", refname, id_str);
2698 done:
2699 	free(id_str);
2700 	return err;
2701 }
2702 
2703 static const struct got_error *
2704 update_ref(struct got_reference *ref, struct got_object_id *new_id,
2705     int verbosity, struct got_repository *repo)
2706 {
2707 	const struct got_error *err = NULL;
2708 	char *new_id_str = NULL;
2709 	struct got_object_id *old_id = NULL;
2710 
2711 	err = got_object_id_str(&new_id_str, new_id);
2712 	if (err)
2713 		goto done;
2714 
2715 	if (strncmp(got_ref_get_name(ref), "refs/tags/", 10) == 0) {
2716 		err = got_ref_resolve(&old_id, repo, ref);
2717 		if (err)
2718 			goto done;
2719 		if (got_object_id_cmp(old_id, new_id) == 0)
2720 			goto done;
2721 		if (verbosity >= 0) {
2722 			printf("Rejecting update of existing tag %s: %s\n",
2723 			    got_ref_get_name(ref), new_id_str);
2724 		}
2725 		goto done;
2726 	}
2727 
2728 	if (got_ref_is_symbolic(ref)) {
2729 		if (verbosity >= 0) {
2730 			printf("Replacing reference %s: %s\n",
2731 			    got_ref_get_name(ref),
2732 			    got_ref_get_symref_target(ref));
2733 		}
2734 		err = got_ref_change_symref_to_ref(ref, new_id);
2735 		if (err)
2736 			goto done;
2737 		err = got_ref_write(ref, repo);
2738 		if (err)
2739 			goto done;
2740 	} else {
2741 		err = got_ref_resolve(&old_id, repo, ref);
2742 		if (err)
2743 			goto done;
2744 		if (got_object_id_cmp(old_id, new_id) == 0)
2745 			goto done;
2746 
2747 		err = got_ref_change_ref(ref, new_id);
2748 		if (err)
2749 			goto done;
2750 		err = got_ref_write(ref, repo);
2751 		if (err)
2752 			goto done;
2753 	}
2754 
2755 	if (verbosity >= 0)
2756 		printf("Updated %s: %s\n", got_ref_get_name(ref),
2757 		    new_id_str);
2758 done:
2759 	free(old_id);
2760 	free(new_id_str);
2761 	return err;
2762 }
2763 
2764 static const struct got_error *
2765 fetch_updated_remote(const char *proto, const char *host, const char *port,
2766     const char *server_path, int verbosity,
2767     const struct got_remote_repo *remote, struct got_repository *repo,
2768     struct got_reference *head_ref, const char *head_refname)
2769 {
2770 	const struct got_error *err = NULL, *unlock_err = NULL;
2771 	struct got_pathlist_entry *pe;
2772 	struct got_pathlist_head learned_refs;
2773 	struct got_pathlist_head symrefs;
2774 	struct got_pathlist_head wanted_branches;
2775 	struct got_pathlist_head wanted_refs;
2776 	struct got_object_id *pack_hash;
2777 	struct got_fetch_progress_arg fpa;
2778 	int fetchfd = -1;
2779 	pid_t fetchpid = -1;
2780 
2781 	TAILQ_INIT(&learned_refs);
2782 	TAILQ_INIT(&symrefs);
2783 	TAILQ_INIT(&wanted_branches);
2784 	TAILQ_INIT(&wanted_refs);
2785 
2786 	err = got_pathlist_insert(NULL, &wanted_branches, head_refname,
2787 	    NULL);
2788 	if (err)
2789 		goto done;
2790 
2791 	err = got_fetch_connect(&fetchpid, &fetchfd, proto, host,
2792 	    port, server_path, verbosity);
2793 	if (err)
2794 		goto done;
2795 
2796 	fpa.last_scaled_size[0] = '\0';
2797 	fpa.last_p_indexed = -1;
2798 	fpa.last_p_resolved = -1;
2799 	fpa.verbosity = verbosity;
2800 	fpa.repo = repo;
2801 
2802 	err = got_fetch_pack(&pack_hash, &learned_refs, &symrefs,
2803 	    remote->name, 1, 0, &wanted_branches, &wanted_refs, 0, verbosity,
2804 	    fetchfd, repo, head_refname, NULL, 0, fetch_progress, &fpa);
2805 	if (err)
2806 		goto done;
2807 
2808 	/* Update references provided with the pack file. */
2809 	TAILQ_FOREACH(pe, &learned_refs, entry) {
2810 		const char *refname = pe->path;
2811 		struct got_object_id *id = pe->data;
2812 		struct got_reference *ref;
2813 
2814 		err = got_ref_open(&ref, repo, refname, 0);
2815 		if (err) {
2816 			if (err->code != GOT_ERR_NOT_REF)
2817 				goto done;
2818 			err = create_ref(refname, id, verbosity, repo);
2819 			if (err)
2820 				goto done;
2821 		} else {
2822 			err = update_ref(ref, id, verbosity, repo);
2823 			unlock_err = got_ref_unlock(ref);
2824 			if (unlock_err && err == NULL)
2825 				err = unlock_err;
2826 			got_ref_close(ref);
2827 			if (err)
2828 				goto done;
2829 		}
2830 	}
2831 
2832 	/* Set the HEAD reference if the server provided one. */
2833 	TAILQ_FOREACH(pe, &symrefs, entry) {
2834 		struct got_reference *target_ref;
2835 		const char *refname = pe->path;
2836 		const char *target = pe->data;
2837 		char *remote_refname = NULL, *remote_target = NULL;
2838 
2839 		if (strcmp(refname, GOT_REF_HEAD) != 0)
2840 			continue;
2841 
2842 		err = got_ref_open(&target_ref, repo, target, 0);
2843 		if (err) {
2844 			if (err->code == GOT_ERR_NOT_REF) {
2845 				err = NULL;
2846 				continue;
2847 			}
2848 			goto done;
2849 		}
2850 
2851 		err = create_symref(refname, target_ref, verbosity, repo);
2852 		got_ref_close(target_ref);
2853 		if (err)
2854 			goto done;
2855 
2856 		if (remote->mirror_references)
2857 			continue;
2858 
2859 		if (strncmp("refs/heads/", target, 11) != 0)
2860 			continue;
2861 
2862 		if (asprintf(&remote_refname,
2863 		    "refs/remotes/%s/%s", GOT_FETCH_DEFAULT_REMOTE_NAME,
2864 		    refname) == -1) {
2865 			err = got_error_from_errno("asprintf");
2866 			goto done;
2867 		}
2868 		if (asprintf(&remote_target,
2869 		    "refs/remotes/%s/%s", GOT_FETCH_DEFAULT_REMOTE_NAME,
2870 		    target + 11) == -1) {
2871 			err = got_error_from_errno("asprintf");
2872 			free(remote_refname);
2873 			goto done;
2874 		}
2875 		err = got_ref_open(&target_ref, repo, remote_target, 0);
2876 		if (err) {
2877 			free(remote_refname);
2878 			free(remote_target);
2879 			if (err->code == GOT_ERR_NOT_REF) {
2880 				err = NULL;
2881 				continue;
2882 			}
2883 			goto done;
2884 		}
2885 		err = create_symref(remote_refname, target_ref,
2886 		    verbosity - 1, repo);
2887 		free(remote_refname);
2888 		free(remote_target);
2889 		got_ref_close(target_ref);
2890 		if (err)
2891 			goto done;
2892 	}
2893 
2894 done:
2895 	got_pathlist_free(&learned_refs, GOT_PATHLIST_FREE_NONE);
2896 	got_pathlist_free(&symrefs, GOT_PATHLIST_FREE_NONE);
2897 	got_pathlist_free(&wanted_branches, GOT_PATHLIST_FREE_NONE);
2898 	got_pathlist_free(&wanted_refs, GOT_PATHLIST_FREE_NONE);
2899 	return err;
2900 }
2901 
2902 
2903 const struct got_error *
2904 got_worktree_cvg_commit(struct got_object_id **new_commit_id,
2905     struct got_worktree *worktree, struct got_pathlist_head *paths,
2906     const char *author, const char *committer, int allow_bad_symlinks,
2907     int show_diff, int commit_conflicts,
2908     got_worktree_commit_msg_cb commit_msg_cb, void *commit_arg,
2909     got_worktree_status_cb status_cb, void *status_arg,
2910     const char *proto, const char *host, const char *port,
2911     const char *server_path, int verbosity,
2912     const struct got_remote_repo *remote,
2913     got_cancel_cb check_cancelled,
2914     struct got_repository *repo)
2915 {
2916 	const struct got_error *err = NULL, *unlockerr = NULL, *sync_err;
2917 	struct got_fileindex *fileindex = NULL;
2918 	char *fileindex_path = NULL;
2919 	struct got_pathlist_head commitable_paths;
2920 	struct collect_commitables_arg cc_arg;
2921 	struct got_pathlist_entry *pe;
2922 	struct got_reference *head_ref = NULL, *head_ref2 = NULL;
2923 	struct got_reference *commit_ref = NULL;
2924 	struct got_object_id *head_commit_id = NULL;
2925 	struct got_object_id *head_commit_id2 = NULL;
2926 	char *head_refname = NULL;
2927 	char *commit_refname = NULL;
2928 	char *diff_path = NULL;
2929 	int have_staged_files = 0;
2930 	int sendfd = -1;
2931 	pid_t sendpid = -1;
2932 	struct got_send_progress_arg spa;
2933 	struct got_pathlist_head commit_reflist;
2934 	struct got_pathlist_head tag_names;
2935 	struct got_pathlist_head delete_branches;
2936 
2937 	*new_commit_id = NULL;
2938 
2939 	memset(&cc_arg, 0, sizeof(cc_arg));
2940 	TAILQ_INIT(&commitable_paths);
2941 	TAILQ_INIT(&commit_reflist);
2942 	TAILQ_INIT(&tag_names);
2943 	TAILQ_INIT(&delete_branches);
2944 
2945 	err = lock_worktree(worktree, LOCK_EX);
2946 	if (err)
2947 		goto done;
2948 
2949 	err = got_worktree_cvg_get_commit_ref_name(&commit_refname,
2950 	    worktree);
2951 	if (err)
2952 		goto done;
2953 
2954 	head_refname = worktree->head_ref_name;
2955 	err = got_ref_open(&head_ref, repo, head_refname, 0);
2956 	if (err)
2957 		goto done;
2958 	err = got_ref_resolve(&head_commit_id, repo, head_ref);
2959 	if (err)
2960 		goto done;
2961 
2962 	err = got_ref_alloc(&commit_ref, commit_refname, head_commit_id);
2963 	if (err)
2964 		goto done;
2965 	err = got_ref_write(commit_ref, repo);
2966 	if (err)
2967 		goto done;
2968 
2969 	err = open_fileindex(&fileindex, &fileindex_path, worktree);
2970 	if (err)
2971 		goto done;
2972 
2973 	err = got_fileindex_for_each_entry_safe(fileindex, check_staged_file,
2974 	    &have_staged_files);
2975 	if (err && err->code != GOT_ERR_CANCELLED)
2976 		goto done;
2977 	if (have_staged_files) {
2978 		err = check_non_staged_files(fileindex, paths);
2979 		if (err)
2980 			goto done;
2981 	}
2982 
2983 	cc_arg.commitable_paths = &commitable_paths;
2984 	cc_arg.worktree = worktree;
2985 	cc_arg.fileindex = fileindex;
2986 	cc_arg.repo = repo;
2987 	cc_arg.have_staged_files = have_staged_files;
2988 	cc_arg.allow_bad_symlinks = allow_bad_symlinks;
2989 	cc_arg.diff_header_shown = 0;
2990 	cc_arg.commit_conflicts = commit_conflicts;
2991 	if (show_diff) {
2992 		err = got_opentemp_named(&diff_path, &cc_arg.diff_outfile,
2993 		    GOT_TMPDIR_STR "/got", ".diff");
2994 		if (err)
2995 			goto done;
2996 		cc_arg.f1 = got_opentemp();
2997 		if (cc_arg.f1 == NULL) {
2998 			err = got_error_from_errno("got_opentemp");
2999 			goto done;
3000 		}
3001 		cc_arg.f2 = got_opentemp();
3002 		if (cc_arg.f2 == NULL) {
3003 			err = got_error_from_errno("got_opentemp");
3004 			goto done;
3005 		}
3006 	}
3007 
3008 	TAILQ_FOREACH(pe, paths, entry) {
3009 		err = worktree_status(worktree, pe->path, fileindex, repo,
3010 		    collect_commitables, &cc_arg, NULL, NULL, 0, 0);
3011 		if (err)
3012 			goto done;
3013 	}
3014 
3015 	if (show_diff) {
3016 		if (fflush(cc_arg.diff_outfile) == EOF) {
3017 			err = got_error_from_errno("fflush");
3018 			goto done;
3019 		}
3020 	}
3021 
3022 	if (TAILQ_EMPTY(&commitable_paths)) {
3023 		err = got_error(GOT_ERR_COMMIT_NO_CHANGES);
3024 		goto done;
3025 	}
3026 
3027 	TAILQ_FOREACH(pe, paths, entry) {
3028 		err = check_path_is_commitable(pe->path, &commitable_paths);
3029 		if (err)
3030 			goto done;
3031 	}
3032 
3033 	TAILQ_FOREACH(pe, &commitable_paths, entry) {
3034 		struct got_commitable *ct = pe->data;
3035 		const char *ct_path = ct->in_repo_path;
3036 
3037 		while (ct_path[0] == '/')
3038 			ct_path++;
3039 		err = check_out_of_date(ct_path, ct->status,
3040 		    ct->staged_status, ct->base_blob_id, ct->base_commit_id,
3041 		    head_commit_id, repo, GOT_ERR_COMMIT_OUT_OF_DATE);
3042 		if (err)
3043 			goto done;
3044 	}
3045 
3046 	err = commit_worktree(new_commit_id, &commitable_paths,
3047 	    head_commit_id, NULL, worktree, author, committer,
3048 	    (diff_path && cc_arg.diff_header_shown) ? diff_path : NULL,
3049 	    commit_msg_cb, commit_arg, status_cb, status_arg, repo);
3050 	if (err)
3051 		goto done;
3052 
3053 	/*
3054 	 * Check if a concurrent commit to our branch has occurred.
3055 	 * Lock the reference here to prevent concurrent modification.
3056 	 */
3057 	err = got_ref_open(&head_ref2, repo, head_refname, 1);
3058 	if (err)
3059 		goto done;
3060 	err = got_ref_resolve(&head_commit_id2, repo, head_ref2);
3061 	if (err)
3062 		goto done;
3063 	if (got_object_id_cmp(head_commit_id, head_commit_id2) != 0) {
3064 		err = got_error(GOT_ERR_COMMIT_HEAD_CHANGED);
3065 		goto done;
3066 	}
3067 
3068 	err = got_pathlist_append(&commit_reflist, commit_refname,
3069 	    head_refname);
3070 	if (err)
3071 		goto done;
3072 
3073 	/* Update commit ref in repository. */
3074 	err = got_ref_change_ref(commit_ref, *new_commit_id);
3075 	if (err)
3076 		goto done;
3077 	err = got_ref_write(commit_ref, repo);
3078 	if (err)
3079 		goto done;
3080 
3081 	if (verbosity >= 0) {
3082 		printf("Connecting to \"%s\" %s://%s%s%s%s%s\n",
3083 		    remote->name, proto, host,
3084 		    port ? ":" : "", port ? port : "",
3085 		    *server_path == '/' ? "" : "/", server_path);
3086 	}
3087 
3088 	/* Attempt send to remote branch. */
3089 	err = got_send_connect(&sendpid, &sendfd, proto, host, port,
3090 	    server_path, verbosity);
3091 	if (err)
3092 		goto done;
3093 
3094 	memset(&spa, 0, sizeof(spa));
3095 	spa.last_scaled_packsize[0] = '\0';
3096 	spa.last_p_deltify = -1;
3097 	spa.last_p_written = -1;
3098 	spa.verbosity = verbosity;
3099 	spa.delete_branches = &delete_branches;
3100 	err = got_send_pack(remote->name, &commit_reflist, &tag_names,
3101 	    &delete_branches, verbosity, 0, sendfd, repo, send_progress, &spa,
3102 	    check_cancelled, NULL);
3103 	if (spa.printed_something)
3104 		putchar('\n');
3105 	if (err != NULL && err->code == GOT_ERR_SEND_ANCESTRY) {
3106 		/*
3107 		 * Fetch new changes since remote has diverged.
3108 		 * No trivial-rebase yet; require update to be run manually.
3109 		 */
3110 		err = fetch_updated_remote(proto, host, port, server_path,
3111 		    verbosity, remote, repo, head_ref, head_refname);
3112 		if (err == NULL)
3113 			goto done;
3114 		err = got_error(GOT_ERR_COMMIT_OUT_OF_DATE);
3115 		goto done;
3116 		/* XXX: Rebase commit over fetched remote branch. */
3117 	}
3118 	if (err) {
3119 		goto done;
3120 	}
3121 
3122 	/* Update branch head in repository. */
3123 	err = got_ref_change_ref(head_ref2, *new_commit_id);
3124 	if (err)
3125 		goto done;
3126 	err = got_ref_write(head_ref2, repo);
3127 	if (err)
3128 		goto done;
3129 
3130 	err = got_worktree_set_base_commit_id(worktree, repo, *new_commit_id);
3131 	if (err)
3132 		goto done;
3133 
3134 	err = ref_base_commit(worktree, repo);
3135 	if (err)
3136 		goto done;
3137 
3138 	/* XXX: fileindex must be updated for other fetched changes? */
3139 	err = update_fileindex_after_commit(worktree, &commitable_paths,
3140 	    *new_commit_id, fileindex, have_staged_files);
3141 	sync_err = sync_fileindex(fileindex, fileindex_path);
3142 	if (sync_err && err == NULL)
3143 		err = sync_err;
3144 done:
3145 	if (head_ref2) {
3146 		unlockerr = got_ref_unlock(head_ref2);
3147 		if (unlockerr && err == NULL)
3148 			err = unlockerr;
3149 		got_ref_close(head_ref2);
3150 	}
3151 	if (commit_ref)
3152 		got_ref_close(commit_ref);
3153 	if (fileindex)
3154 		got_fileindex_free(fileindex);
3155 	unlockerr = lock_worktree(worktree, LOCK_SH);
3156 	if (unlockerr && err == NULL)
3157 		err = unlockerr;
3158 	TAILQ_FOREACH(pe, &commitable_paths, entry) {
3159 		struct got_commitable *ct = pe->data;
3160 
3161 		free_commitable(ct);
3162 	}
3163 	got_pathlist_free(&commitable_paths, GOT_PATHLIST_FREE_NONE);
3164 	if (diff_path && unlink(diff_path) == -1 && err == NULL)
3165 		err = got_error_from_errno2("unlink", diff_path);
3166 	if (cc_arg.diff_outfile && fclose(cc_arg.diff_outfile) == EOF &&
3167 	    err == NULL)
3168 		err = got_error_from_errno("fclose");
3169 	free(head_commit_id);
3170 	free(head_commit_id2);
3171 	free(commit_refname);
3172 	free(fileindex_path);
3173 	free(diff_path);
3174 	return err;
3175 }
3176 
3177 const struct got_error *
3178 got_worktree_cvg_get_commit_ref_name(char **refname,
3179     struct got_worktree *worktree)
3180 {
3181 	return get_ref_name(refname, worktree, GOT_WORKTREE_COMMIT_REF_PREFIX);
3182 }