1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  * Copyright (c) 2020 Neels Hofmeyr <neels@hofmeyr.de>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  */
17 
18 #include <sys/mman.h>
19 #include <sys/stat.h>
20 
21 #include <errno.h>
22 #include <string.h>
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <time.h>
26 #include <limits.h>
27 #include <zlib.h>
28 
29 #include "got_compat.h"
30 
31 #include "got_error.h"
32 #include "got_object.h"
33 #include "got_cancel.h"
34 #include "got_blame.h"
35 #include "got_commit_graph.h"
36 #include "got_opentemp.h"
37 
38 #include "got_lib_inflate.h"
39 #include "got_lib_delta.h"
40 #include "got_lib_object.h"
41 #include "got_lib_diff.h"
42 
43 #ifndef MAX
44 #define	MAX(_a,_b) ((_a) > (_b) ? (_a) : (_b))
45 #endif
46 
47 struct got_blame_line {
48 	int annotated;
49 	struct got_object_id id;
50 };
51 
52 struct got_blame {
53 	struct diff_config *cfg;
54 	int nlines;	/* number of lines in file being blamed */
55 	int nannotated;	/* number of lines already annotated */
56 	struct got_blame_line *lines; /* one per line */
57 	int ncommits;
58 
59 	/*
60 	 * These change with every traversed commit. After diffing
61 	 * commits N:N-1, in preparation for diffing commits N-1:N-2,
62 	 * data for commit N is retained and flipped into data for N-1.
63 	 * 
64 	 */
65 	FILE *f1; /* older version from commit N-1. */
66 	FILE *f2; /* newer version from commit N. */
67 	unsigned char *map1;
68 	unsigned char *map2;
69 	off_t size1;
70 	off_t size2;
71 	int nlines1;
72 	int nlines2;
73 	off_t *line_offsets1;
74 	off_t *line_offsets2;
75 
76 	/*
77 	 * Map line numbers of an older version of the file to valid line
78 	 * numbers in the version of the file being blamed. This map is
79 	 * updated with each commit we traverse throughout the file's history.
80 	 * Lines mapped to -1 do not correspond to any line in the version
81 	 * being blamed.
82 	 */
83 	int *linemap1;
84 	int *linemap2;
85 
86 	struct diff_data *data1;
87 	struct diff_data *data2;
88 };
89 
90 static const struct got_error *
91 annotate_line(struct got_blame *blame, int lineno,
92     struct got_commit_object *commit, struct got_object_id *id,
93     got_blame_cb cb, void *arg)
94 {
95 	const struct got_error *err = NULL;
96 	struct got_blame_line *line;
97 
98 	if (lineno < 0 || lineno >= blame->nlines)
99 		return NULL;
100 
101 	line = &blame->lines[lineno];
102 	if (line->annotated)
103 		return NULL;
104 
105 	memcpy(&line->id, id, sizeof(line->id));
106 	line->annotated = 1;
107 	blame->nannotated++;
108 	if (cb)
109 		err = cb(arg, blame->nlines, lineno + 1, commit, id);
110 	return err;
111 }
112 
113 static const struct got_error *
114 blame_changes(struct got_blame *blame, struct diff_result *diff_result,
115     struct got_commit_object *commit, struct got_object_id *commit_id,
116     got_blame_cb cb, void *arg)
117 {
118 	const struct got_error *err = NULL;
119 	int i;
120 	int idx1 = 0, idx2 = 0;
121 
122 	for (i = 0; i < diff_result->chunks.len &&
123 	    blame->nannotated < blame->nlines; i++) {
124 		struct diff_chunk *c = diff_chunk_get(diff_result, i);
125 		unsigned int left_count, right_count;
126 		int j;
127 
128 		/*
129 		 * We do not need to worry about idx1/idx2 growing out
130 		 * of bounds because the diff implementation ensures
131 		 * that chunk ranges never exceed the number of lines
132 		 * in the left/right input files.
133 		 */
134 		left_count = diff_chunk_get_left_count(c);
135 		right_count = diff_chunk_get_right_count(c);
136 
137 		if (left_count == right_count) {
138 			for (j = 0; j < left_count; j++) {
139 				blame->linemap1[idx1++] =
140 				    blame->linemap2[idx2++];
141 			}
142 			continue;
143 		}
144 
145 		if (right_count == 0) {
146 			for (j = 0; j < left_count; j++) {
147 				blame->linemap1[idx1++] = -1;
148 			}
149 			continue;
150 		}
151 
152 		for (j = 0; j < right_count; j++) {
153 			int ln = blame->linemap2[idx2++];
154 			err = annotate_line(blame, ln, commit, commit_id,
155 			    cb, arg);
156 			if (err)
157 				return err;
158 			if (blame->nlines == blame->nannotated)
159 				break;
160 		}
161 	}
162 
163 	return NULL;
164 }
165 
166 static const struct got_error *
167 blame_prepare_file(FILE *f, unsigned char **p, off_t *size,
168     int *nlines, off_t **line_offsets, struct diff_data *diff_data,
169     const struct diff_config *cfg, struct got_blob_object *blob)
170 {
171 	const struct got_error *err = NULL;
172 	int diff_flags = 0, rc;
173 
174 	err = got_object_blob_dump_to_file(size, nlines, line_offsets,
175 	    f, blob);
176 	if (err)
177 		return err;
178 
179 #ifndef GOT_DIFF_NO_MMAP
180 	*p = mmap(NULL, *size, PROT_READ, MAP_PRIVATE, fileno(f), 0);
181 	if (*p == MAP_FAILED)
182 #endif
183 		*p = NULL; /* fall back on file I/O */
184 
185 	/* Allow blaming lines in binary files even though it's useless. */
186 	diff_flags |= DIFF_FLAG_FORCE_TEXT_DATA;
187 
188 	rc = diff_atomize_file(diff_data, cfg, f, *p, *size, diff_flags);
189 	if (rc)
190 		return got_error_set_errno(rc, "diff_atomize_file");
191 
192 	return NULL;
193 }
194 
195 static const struct got_error *
196 blame_commit(struct got_blame *blame, struct got_object_id *id,
197     const char *path, struct got_repository *repo,
198     got_blame_cb cb, void *arg)
199 {
200 	const struct got_error *err = NULL;
201 	struct got_commit_object *commit = NULL, *pcommit = NULL;
202 	struct got_object_qid *pid = NULL;
203 	struct got_object_id *pblob_id = NULL;
204 	struct got_blob_object *pblob = NULL;
205 	struct diff_result *diff_result = NULL;
206 	int fd = -1;
207 
208 	err = got_object_open_as_commit(&commit, repo, id);
209 	if (err)
210 		return err;
211 
212 	pid = STAILQ_FIRST(got_object_commit_get_parent_ids(commit));
213 	if (pid == NULL) {
214 		got_object_commit_close(commit);
215 		return NULL;
216 	}
217 
218 	fd = got_opentempfd();
219 	if (fd == -1) {
220 		err = got_error_from_errno("got_opentempfd");
221 		goto done;
222 	}
223 
224 	err = got_object_open_as_commit(&pcommit, repo, &pid->id);
225 	if (err)
226 		goto done;
227 
228 	err = got_object_id_by_path(&pblob_id, repo, pcommit, path);
229 	if (err) {
230 		if (err->code == GOT_ERR_NO_TREE_ENTRY)
231 			err = NULL;
232 		goto done;
233 	}
234 
235 	err = got_object_open_as_blob(&pblob, repo, pblob_id, 8192, fd);
236 	if (err)
237 		goto done;
238 
239 	blame->f1 = got_opentemp();
240 	if (blame->f1 == NULL) {
241 		err = got_error_from_errno("got_opentemp");
242 		goto done;
243 	}
244 
245 	err = blame_prepare_file(blame->f1, &blame->map1, &blame->size1,
246 	    &blame->nlines1, &blame->line_offsets1, blame->data1,
247 	    blame->cfg, pblob);
248 	if (err)
249 		goto done;
250 
251 	diff_result = diff_main(blame->cfg, blame->data1, blame->data2);
252 	if (diff_result == NULL) {
253 		err = got_error_set_errno(ENOMEM, "malloc");
254 		goto done;
255 	}
256 	if (diff_result->rc != DIFF_RC_OK) {
257 		err = got_error_set_errno(diff_result->rc, "diff");
258 		goto done;
259 	}
260 	if (diff_result->chunks.len > 0) {
261 		if (blame->nlines1 > 0) {
262 			blame->linemap1 = calloc(blame->nlines1,
263 			    sizeof(*blame->linemap1));
264 			if (blame->linemap1 == NULL) {
265 				err = got_error_from_errno("malloc");
266 				goto done;
267 			}
268 		}
269 		err = blame_changes(blame, diff_result, commit, id, cb, arg);
270 		if (err)
271 			goto done;
272 	} else if (cb)
273 		err = cb(arg, blame->nlines, -1, commit, id);
274 done:
275 	if (diff_result)
276 		diff_result_free(diff_result);
277 	if (commit)
278 		got_object_commit_close(commit);
279 	if (pcommit)
280 		got_object_commit_close(pcommit);
281 	free(pblob_id);
282 	if (fd != -1 && close(fd) == -1 && err == NULL)
283 		err = got_error_from_errno("close");
284 	if (pblob)
285 		got_object_blob_close(pblob);
286 	return err;
287 }
288 
289 static const struct got_error *
290 blame_close(struct got_blame *blame)
291 {
292 	const struct got_error *err = NULL;
293 
294 	diff_data_free(blame->data1);
295 	free(blame->data1);
296 	diff_data_free(blame->data2);
297 	free(blame->data2);
298 	if (blame->map1) {
299 		if (munmap(blame->map1, blame->size1) == -1 && err == NULL)
300 			err = got_error_from_errno("munmap");
301 	}
302 	if (blame->map2) {
303 		if (munmap(blame->map2, blame->size2) == -1 && err == NULL)
304 			err = got_error_from_errno("munmap");
305 	}
306 	if (blame->f1 && fclose(blame->f1) == EOF && err == NULL)
307 		err = got_error_from_errno("fclose");
308 	if (blame->f2 && fclose(blame->f2) == EOF && err == NULL)
309 		err = got_error_from_errno("fclose");
310 	free(blame->lines);
311 	free(blame->line_offsets1);
312 	free(blame->line_offsets2);
313 	free(blame->linemap1);
314 	free(blame->linemap2);
315 	free(blame->cfg);
316 	free(blame);
317 	return err;
318 }
319 
320 static int
321 atomize_file(struct diff_data *d, FILE *f, off_t filesize, int nlines,
322     off_t *line_offsets)
323 {
324 	int i, rc = DIFF_RC_OK;
325 	int embedded_nul = 0;
326 
327 	ARRAYLIST_INIT(d->atoms, nlines);
328 
329 	for (i = 0; i < nlines; i++) {
330 		struct diff_atom *atom;
331 		off_t len, pos = line_offsets[i];
332 		unsigned int hash = 0;
333 		int j;
334 
335 		ARRAYLIST_ADD(atom, d->atoms);
336 		if (atom == NULL) {
337 			rc = errno;
338 			break;
339 		}
340 
341 		if (i < nlines - 1)
342 			len = line_offsets[i + 1] - pos;
343 		else
344 			len = filesize - pos;
345 
346 		if (fseeko(f, pos, SEEK_SET) == -1) {
347 			rc = errno;
348 			break;
349 		}
350 		for (j = 0; j < len; j++) {
351 			int c = fgetc(f);
352 			if (c == EOF) {
353 				if (feof(f))
354 					rc = EIO; /* unexpected EOF */
355 				else
356 					rc = errno;
357 				goto done;
358 			}
359 
360 			hash = diff_atom_hash_update(hash, (unsigned char)c);
361 
362 			if (c == '\0')
363 				embedded_nul = 1;
364 
365 		}
366 		*atom = (struct diff_atom){
367 			.root = d,
368 			.pos = pos,
369 			.at = NULL,	/* atom data is not memory-mapped */
370 			.len = len,
371 			.hash = hash,
372 		};
373 	}
374 
375 	/* File are considered binary if they contain embedded '\0' bytes. */
376 	if (embedded_nul)
377 		d->atomizer_flags |= DIFF_ATOMIZER_FOUND_BINARY_DATA;
378 done:
379 	if (rc)
380 		ARRAYLIST_FREE(d->atoms);
381 
382 	return rc;
383 }
384 
385 static int
386 atomize_file_mmap(struct diff_data *d, unsigned char *p,
387     off_t filesize, int nlines, off_t *line_offsets)
388 {
389 	int i, rc = DIFF_RC_OK;
390 	int embedded_nul = 0;
391 
392 	ARRAYLIST_INIT(d->atoms, nlines);
393 
394 	for (i = 0; i < nlines; i++) {
395 		struct diff_atom *atom;
396 		off_t len, pos = line_offsets[i];
397 		unsigned int hash = 0;
398 		int j;
399 
400 		ARRAYLIST_ADD(atom, d->atoms);
401 		if (atom == NULL) {
402 			rc = errno;
403 			break;
404 		}
405 
406 		if (i < nlines - 1)
407 			len = line_offsets[i + 1] - pos;
408 		else
409 			len = filesize - pos;
410 
411 		for (j = 0; j < len; j++)
412 			hash = diff_atom_hash_update(hash, p[pos + j]);
413 
414 		if (!embedded_nul && memchr(&p[pos], '\0', len) != NULL)
415 			embedded_nul = 1;
416 
417 		*atom = (struct diff_atom){
418 			.root = d,
419 			.pos = pos,
420 			.at = &p[pos],
421 			.len = len,
422 			.hash = hash,
423 		};
424 	}
425 
426 	/* File are considered binary if they contain embedded '\0' bytes. */
427 	if (embedded_nul)
428 		d->atomizer_flags |= DIFF_ATOMIZER_FOUND_BINARY_DATA;
429 
430 	if (rc)
431 		ARRAYLIST_FREE(d->atoms);
432 
433 	return rc;
434 }
435 
436 /* Implements diff_atomize_func_t */
437 static int
438 blame_atomize_file(void *arg, struct diff_data *d)
439 {
440 	struct got_blame *blame = arg;
441 
442 	if (d->f == blame->f1) {
443 		if (blame->map1)
444 			return atomize_file_mmap(d, blame->map1,
445 			    blame->size1, blame->nlines1,
446 			    blame->line_offsets1);
447 		else
448 			return atomize_file(d, blame->f1, blame->size1,
449 			    blame->nlines1, blame->line_offsets1);
450 	} else if (d->f == blame->f2) {
451 		if (d->atoms.len > 0) {
452 			/* Re-use data from previous commit. */
453 			return DIFF_RC_OK;
454 		}
455 		if (blame->map2)
456 			return atomize_file_mmap(d, blame->map2,
457 			    blame->size2, blame->nlines2,
458 			    blame->line_offsets2);
459 		else
460 			return atomize_file(d, blame->f2, blame->size2,
461 			    blame->nlines2, blame->line_offsets2);
462 	}
463 
464 	return DIFF_RC_OK;
465 }
466 
467 static const struct got_error *
468 close_file2_and_reuse_file1(struct got_blame *blame)
469 {
470 	struct diff_data *d;
471 
472 	free(blame->line_offsets2);
473 	blame->line_offsets2 = blame->line_offsets1;
474 	blame->line_offsets1 = NULL;
475 
476 	free(blame->linemap2);
477 	blame->linemap2 = blame->linemap1;
478 	blame->linemap1 = NULL;
479 
480 	if (blame->map2) {
481 		if (munmap(blame->map2, blame->size2) == -1)
482 			return got_error_from_errno("munmap");
483 		blame->map2 = blame->map1;
484 		blame->map1 = NULL;
485 
486 	}
487 	blame->size2 = blame->size1;
488 	blame->size1 = 0;
489 
490 	if (fclose(blame->f2) == EOF)
491 		return got_error_from_errno("fclose");
492 	blame->f2 = blame->f1;
493 	blame->f1 = NULL;
494 
495 	blame->nlines2 = blame->nlines1;
496 	blame->nlines1 = 0;
497 
498 	diff_data_free(blame->data2); /* does not free pointer itself */
499 	memset(blame->data2, 0, sizeof(*blame->data2));
500 	d = blame->data2;
501 	blame->data2 = blame->data1;
502 	blame->data1 = d;
503 
504 	return NULL;
505 }
506 
507 static const struct got_error *
508 blame_open(struct got_blame **blamep, const char *path,
509     struct got_object_id *start_commit_id, struct got_repository *repo,
510     got_blame_cb cb, void *arg, got_cancel_cb cancel_cb, void *cancel_arg,
511     int fd)
512 {
513 	const struct got_error *err = NULL;
514 	struct got_commit_object *start_commit = NULL, *last_commit = NULL;
515 	struct got_object_id *obj_id = NULL;
516 	struct got_blob_object *blob = NULL;
517 	struct got_blame *blame = NULL;
518 	struct got_object_id *id = NULL;
519 	int lineno;
520 	struct got_commit_graph *graph = NULL;
521 
522 	*blamep = NULL;
523 
524 	err = got_object_open_as_commit(&start_commit, repo, start_commit_id);
525 	if (err)
526 		goto done;
527 
528 	err = got_object_id_by_path(&obj_id, repo, start_commit, path);
529 	if (err)
530 		goto done;
531 
532 	err = got_object_open_as_blob(&blob, repo, obj_id, 8192, fd);
533 	if (err)
534 		goto done;
535 
536 	blame = calloc(1, sizeof(*blame));
537 	if (blame == NULL) {
538 		err = got_error_from_errno("calloc");
539 		goto done;
540 	}
541 
542 	blame->data1 = calloc(1, sizeof(*blame->data1));
543 	if (blame->data1 == NULL) {
544 		err = got_error_from_errno("calloc");
545 		goto done;
546 	}
547 	blame->data2 = calloc(1, sizeof(*blame->data2));
548 	if (blame->data2 == NULL) {
549 		err = got_error_from_errno("calloc");
550 		goto done;
551 	}
552 
553 	blame->f2 = got_opentemp();
554 	if (blame->f2 == NULL) {
555 		err = got_error_from_errno("got_opentemp");
556 		goto done;
557 	}
558 	err = got_diff_get_config(&blame->cfg, GOT_DIFF_ALGORITHM_PATIENCE,
559 	    blame_atomize_file, blame);
560 	if (err)
561 		goto done;
562 
563 	err = blame_prepare_file(blame->f2, &blame->map2, &blame->size2,
564 	    &blame->nlines2, &blame->line_offsets2, blame->data2,
565 	    blame->cfg, blob);
566 	blame->nlines = blame->nlines2;
567 	if (err || blame->nlines == 0)
568 		goto done;
569 
570 	got_object_blob_close(blob);
571 	blob = NULL;
572 
573 	/* Don't include \n at EOF in the blame line count. */
574 	if (blame->line_offsets2[blame->nlines - 1] == blame->size2)
575 		blame->nlines--;
576 
577 	blame->lines = calloc(blame->nlines, sizeof(*blame->lines));
578 	if (blame->lines == NULL) {
579 		err = got_error_from_errno("calloc");
580 		goto done;
581 	}
582 
583 	blame->linemap2 = calloc(blame->nlines2, sizeof(*blame->linemap2));
584 	if (blame->linemap2 == NULL) {
585 		err = got_error_from_errno("calloc");
586 		goto done;
587 	}
588 	for (lineno = 0; lineno < blame->nlines2; lineno++)
589 		blame->linemap2[lineno] = lineno;
590 
591 	err = got_commit_graph_open(&graph, path, 1);
592 	if (err)
593 		goto done;
594 
595 	err = got_commit_graph_iter_start(graph, start_commit_id, repo,
596 	    cancel_cb, cancel_arg);
597 	if (err)
598 		goto done;
599 	for (;;) {
600 		struct got_object_id *next_id;
601 		err = got_commit_graph_iter_next(&next_id, graph, repo,
602 		    cancel_cb, cancel_arg);
603 		if (err) {
604 			if (err->code == GOT_ERR_ITER_COMPLETED) {
605 				err = NULL;
606 				break;
607 			}
608 			goto done;
609 		}
610 		if (next_id) {
611 			id = next_id;
612 			err = blame_commit(blame, id, path, repo, cb, arg);
613 			if (err) {
614 				if (err->code == GOT_ERR_ITER_COMPLETED)
615 					err = NULL;
616 				goto done;
617 			}
618 			if (blame->nannotated == blame->nlines)
619 				break;
620 
621 			err = close_file2_and_reuse_file1(blame);
622 			if (err)
623 				goto done;
624 		}
625 	}
626 
627 	if (id && blame->nannotated < blame->nlines) {
628 		/* Annotate remaining non-annotated lines with last commit. */
629 		err = got_object_open_as_commit(&last_commit, repo, id);
630 		if (err)
631 			goto done;
632 		for (lineno = 0; lineno < blame->nlines; lineno++) {
633 			err = annotate_line(blame, lineno, last_commit, id,
634 			    cb, arg);
635 			if (err)
636 				goto done;
637 		}
638 	}
639 
640 done:
641 	if (graph)
642 		got_commit_graph_close(graph);
643 	free(obj_id);
644 	if (blob)
645 		got_object_blob_close(blob);
646 	if (start_commit)
647 		got_object_commit_close(start_commit);
648 	if (last_commit)
649 		got_object_commit_close(last_commit);
650 	if (err) {
651 		if (blame)
652 			blame_close(blame);
653 	} else
654 		*blamep = blame;
655 
656 	return err;
657 }
658 
659 const struct got_error *
660 got_blame(const char *path, struct got_object_id *commit_id,
661     struct got_repository *repo, got_blame_cb cb, void *arg,
662     got_cancel_cb cancel_cb, void* cancel_arg, int fd)
663 {
664 	const struct got_error *err = NULL, *close_err = NULL;
665 	struct got_blame *blame;
666 	char *abspath;
667 
668 	if (asprintf(&abspath, "%s%s", path[0] == '/' ? "" : "/", path) == -1)
669 		return got_error_from_errno2("asprintf", path);
670 
671 	err = blame_open(&blame, abspath, commit_id, repo, cb, arg,
672 	    cancel_cb, cancel_arg, fd);
673 	free(abspath);
674 	if (blame)
675 		close_err = blame_close(blame);
676 	return err ? err : close_err;
677 }