1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  * Copyright (c) 2020 Neels Hofmeyr <neels@hofmeyr.de>
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  */
17 
18 #include <sys/queue.h>
19 #include <sys/mman.h>
20 #include <sys/stat.h>
21 
22 #include <errno.h>
23 #include <sha1.h>
24 #include <sha2.h>
25 #include <string.h>
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <time.h>
29 #include <limits.h>
30 #include <zlib.h>
31 
32 #include "got_compat.h"
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_cancel.h"
37 #include "got_commit_graph.h"
38 #include "got_opentemp.h"
39 #include "got_diff.h"
40 #include "got_blame.h"
41 
42 #include "got_lib_inflate.h"
43 #include "got_lib_delta.h"
44 #include "got_lib_object.h"
45 #include "got_lib_diff.h"
46 
47 #ifndef MAX
48 #define	MAX(_a,_b) ((_a) > (_b) ? (_a) : (_b))
49 #endif
50 
51 struct got_blame_line {
52 	int annotated;
53 	struct got_object_id id;
54 };
55 
56 struct got_blame {
57 	struct diff_config *cfg;
58 	int nlines;	/* number of lines in file being blamed */
59 	int nannotated;	/* number of lines already annotated */
60 	struct got_blame_line *lines; /* one per line */
61 	int ncommits;
62 
63 	/*
64 	 * These change with every traversed commit. After diffing
65 	 * commits N:N-1, in preparation for diffing commits N-1:N-2,
66 	 * data for commit N is retained and flipped into data for N-1.
67 	 *
68 	 */
69 	FILE *f1; /* older version from commit N-1. */
70 	FILE *f2; /* newer version from commit N. */
71 	int fd;
72 	unsigned char *map1;
73 	unsigned char *map2;
74 	off_t size1;
75 	off_t size2;
76 	int nlines1;
77 	int nlines2;
78 	off_t *line_offsets1;
79 	off_t *line_offsets2;
80 
81 	/*
82 	 * Map line numbers of an older version of the file to valid line
83 	 * numbers in the version of the file being blamed. This map is
84 	 * updated with each commit we traverse throughout the file's history.
85 	 * Lines mapped to -1 do not correspond to any line in the version
86 	 * being blamed.
87 	 */
88 	int *linemap1;
89 	int *linemap2;
90 
91 	struct diff_data *data1;
92 	struct diff_data *data2;
93 };
94 
95 static const struct got_error *
96 annotate_line(struct got_blame *blame, int lineno,
97     struct got_commit_object *commit, struct got_object_id *id,
98     got_blame_cb cb, void *arg)
99 {
100 	const struct got_error *err = NULL;
101 	struct got_blame_line *line;
102 
103 	if (lineno < 0 || lineno >= blame->nlines)
104 		return NULL;
105 
106 	line = &blame->lines[lineno];
107 	if (line->annotated)
108 		return NULL;
109 
110 	memcpy(&line->id, id, sizeof(line->id));
111 	line->annotated = 1;
112 	blame->nannotated++;
113 	if (cb)
114 		err = cb(arg, blame->nlines, lineno + 1, commit, id);
115 	return err;
116 }
117 
118 static const struct got_error *
119 blame_changes(struct got_blame *blame, struct diff_result *diff_result,
120     struct got_commit_object *commit, struct got_object_id *commit_id,
121     got_blame_cb cb, void *arg)
122 {
123 	const struct got_error *err = NULL;
124 	int i;
125 	int idx1 = 0, idx2 = 0;
126 
127 	for (i = 0; i < diff_result->chunks.len &&
128 	    blame->nannotated < blame->nlines; i++) {
129 		struct diff_chunk *c = diff_chunk_get(diff_result, i);
130 		unsigned int left_count, right_count;
131 		int j;
132 
133 		/*
134 		 * We do not need to worry about idx1/idx2 growing out
135 		 * of bounds because the diff implementation ensures
136 		 * that chunk ranges never exceed the number of lines
137 		 * in the left/right input files.
138 		 */
139 		left_count = diff_chunk_get_left_count(c);
140 		right_count = diff_chunk_get_right_count(c);
141 
142 		if (left_count == right_count) {
143 			for (j = 0; j < left_count; j++) {
144 				blame->linemap1[idx1++] =
145 				    blame->linemap2[idx2++];
146 			}
147 			continue;
148 		}
149 
150 		if (right_count == 0) {
151 			for (j = 0; j < left_count; j++) {
152 				blame->linemap1[idx1++] = -1;
153 			}
154 			continue;
155 		}
156 
157 		for (j = 0; j < right_count; j++) {
158 			int ln = blame->linemap2[idx2++];
159 			err = annotate_line(blame, ln, commit, commit_id,
160 			    cb, arg);
161 			if (err)
162 				return err;
163 			if (blame->nlines == blame->nannotated)
164 				break;
165 		}
166 	}
167 
168 	return NULL;
169 }
170 
171 static const struct got_error *
172 blame_prepare_file(FILE *f, unsigned char **p, off_t *size,
173     int *nlines, off_t **line_offsets, struct diff_data *diff_data,
174     const struct diff_config *cfg, struct got_blob_object *blob)
175 {
176 	const struct got_error *err = NULL;
177 	int diff_flags = 0, rc;
178 
179 	err = got_object_blob_dump_to_file(size, nlines, line_offsets,
180 	    f, blob);
181 	if (err)
182 		return err;
183 
184 #ifndef GOT_DIFF_NO_MMAP
185 	*p = mmap(NULL, *size, PROT_READ, MAP_PRIVATE, fileno(f), 0);
186 	if (*p == MAP_FAILED)
187 #endif
188 		*p = NULL; /* fall back on file I/O */
189 
190 	/* Allow blaming lines in binary files even though it's useless. */
191 	diff_flags |= DIFF_FLAG_FORCE_TEXT_DATA;
192 
193 	rc = diff_atomize_file(diff_data, cfg, f, *p, *size, diff_flags);
194 	if (rc)
195 		return got_error_set_errno(rc, "diff_atomize_file");
196 
197 	return NULL;
198 }
199 
200 static const struct got_error *
201 blame_commit(struct got_blame *blame, struct got_object_id *id,
202     const char *path, struct got_repository *repo,
203     got_blame_cb cb, void *arg)
204 {
205 	const struct got_error *err = NULL;
206 	struct got_commit_object *commit = NULL, *pcommit = NULL;
207 	struct got_object_qid *pid = NULL;
208 	struct got_object_id *pblob_id = NULL;
209 	struct got_blob_object *pblob = NULL;
210 	struct diff_result *diff_result = NULL;
211 
212 	err = got_object_open_as_commit(&commit, repo, id);
213 	if (err)
214 		return err;
215 
216 	pid = STAILQ_FIRST(got_object_commit_get_parent_ids(commit));
217 	if (pid == NULL) {
218 		got_object_commit_close(commit);
219 		return NULL;
220 	}
221 
222 	err = got_object_open_as_commit(&pcommit, repo, &pid->id);
223 	if (err)
224 		goto done;
225 
226 	err = got_object_id_by_path(&pblob_id, repo, pcommit, path);
227 	if (err) {
228 		if (err->code == GOT_ERR_NO_TREE_ENTRY)
229 			err = NULL;
230 		goto done;
231 	}
232 
233 	err = got_object_open_as_blob(&pblob, repo, pblob_id, 8192, blame->fd);
234 	if (err)
235 		goto done;
236 
237 	err = blame_prepare_file(blame->f1, &blame->map1, &blame->size1,
238 	    &blame->nlines1, &blame->line_offsets1, blame->data1,
239 	    blame->cfg, pblob);
240 	if (err)
241 		goto done;
242 
243 	diff_result = diff_main(blame->cfg, blame->data1, blame->data2);
244 	if (diff_result == NULL) {
245 		err = got_error_set_errno(ENOMEM, "malloc");
246 		goto done;
247 	}
248 	if (diff_result->rc != DIFF_RC_OK) {
249 		err = got_error_set_errno(diff_result->rc, "diff");
250 		goto done;
251 	}
252 	if (diff_result->chunks.len > 0) {
253 		if (blame->nlines1 > 0) {
254 			blame->linemap1 = calloc(blame->nlines1,
255 			    sizeof(*blame->linemap1));
256 			if (blame->linemap1 == NULL) {
257 				err = got_error_from_errno("malloc");
258 				goto done;
259 			}
260 		}
261 		err = blame_changes(blame, diff_result, commit, id, cb, arg);
262 		if (err)
263 			goto done;
264 	} else if (cb)
265 		err = cb(arg, blame->nlines, -1, commit, id);
266 done:
267 	if (diff_result)
268 		diff_result_free(diff_result);
269 	if (commit)
270 		got_object_commit_close(commit);
271 	if (pcommit)
272 		got_object_commit_close(pcommit);
273 	free(pblob_id);
274 	if (pblob)
275 		got_object_blob_close(pblob);
276 	return err;
277 }
278 
279 static const struct got_error *
280 blame_close(struct got_blame *blame)
281 {
282 	const struct got_error *err = NULL;
283 
284 	diff_data_free(blame->data1);
285 	free(blame->data1);
286 	diff_data_free(blame->data2);
287 	free(blame->data2);
288 	if (blame->map1) {
289 		if (munmap(blame->map1, blame->size1) == -1 && err == NULL)
290 			err = got_error_from_errno("munmap");
291 	}
292 	if (blame->map2) {
293 		if (munmap(blame->map2, blame->size2) == -1 && err == NULL)
294 			err = got_error_from_errno("munmap");
295 	}
296 	free(blame->lines);
297 	free(blame->line_offsets1);
298 	free(blame->line_offsets2);
299 	free(blame->linemap1);
300 	free(blame->linemap2);
301 	free(blame->cfg);
302 	free(blame);
303 	return err;
304 }
305 
306 static int
307 atomize_file(struct diff_data *d, FILE *f, off_t filesize, int nlines,
308     off_t *line_offsets)
309 {
310 	int i, rc = DIFF_RC_OK;
311 	int embedded_nul = 0;
312 
313 	ARRAYLIST_INIT(d->atoms, nlines);
314 
315 	for (i = 0; i < nlines; i++) {
316 		struct diff_atom *atom;
317 		off_t len, pos = line_offsets[i];
318 		unsigned int hash = 0;
319 		int j;
320 
321 		ARRAYLIST_ADD(atom, d->atoms);
322 		if (atom == NULL) {
323 			rc = errno;
324 			break;
325 		}
326 
327 		if (i < nlines - 1)
328 			len = line_offsets[i + 1] - pos;
329 		else
330 			len = filesize - pos;
331 
332 		if (fseeko(f, pos, SEEK_SET) == -1) {
333 			rc = errno;
334 			break;
335 		}
336 		for (j = 0; j < len; j++) {
337 			int c = fgetc(f);
338 			if (c == EOF) {
339 				if (feof(f))
340 					rc = EIO; /* unexpected EOF */
341 				else
342 					rc = errno;
343 				goto done;
344 			}
345 
346 			hash = diff_atom_hash_update(hash, (unsigned char)c);
347 
348 			if (c == '\0')
349 				embedded_nul = 1;
350 
351 		}
352 		*atom = (struct diff_atom){
353 			.root = d,
354 			.pos = pos,
355 			.at = NULL,	/* atom data is not memory-mapped */
356 			.len = len,
357 			.hash = hash,
358 		};
359 	}
360 
361 	/* File are considered binary if they contain embedded '\0' bytes. */
362 	if (embedded_nul)
363 		d->atomizer_flags |= DIFF_ATOMIZER_FOUND_BINARY_DATA;
364 done:
365 	if (rc)
366 		ARRAYLIST_FREE(d->atoms);
367 
368 	return rc;
369 }
370 
371 static int
372 atomize_file_mmap(struct diff_data *d, unsigned char *p,
373     off_t filesize, int nlines, off_t *line_offsets)
374 {
375 	int i, rc = DIFF_RC_OK;
376 	int embedded_nul = 0;
377 
378 	ARRAYLIST_INIT(d->atoms, nlines);
379 
380 	for (i = 0; i < nlines; i++) {
381 		struct diff_atom *atom;
382 		off_t len, pos = line_offsets[i];
383 		unsigned int hash = 0;
384 		int j;
385 
386 		ARRAYLIST_ADD(atom, d->atoms);
387 		if (atom == NULL) {
388 			rc = errno;
389 			break;
390 		}
391 
392 		if (i < nlines - 1)
393 			len = line_offsets[i + 1] - pos;
394 		else
395 			len = filesize - pos;
396 
397 		for (j = 0; j < len; j++)
398 			hash = diff_atom_hash_update(hash, p[pos + j]);
399 
400 		if (!embedded_nul && memchr(&p[pos], '\0', len) != NULL)
401 			embedded_nul = 1;
402 
403 		*atom = (struct diff_atom){
404 			.root = d,
405 			.pos = pos,
406 			.at = &p[pos],
407 			.len = len,
408 			.hash = hash,
409 		};
410 	}
411 
412 	/* File are considered binary if they contain embedded '\0' bytes. */
413 	if (embedded_nul)
414 		d->atomizer_flags |= DIFF_ATOMIZER_FOUND_BINARY_DATA;
415 
416 	if (rc)
417 		ARRAYLIST_FREE(d->atoms);
418 
419 	return rc;
420 }
421 
422 /* Implements diff_atomize_func_t */
423 static int
424 blame_atomize_file(void *arg, struct diff_data *d)
425 {
426 	struct got_blame *blame = arg;
427 
428 	if (d->f == blame->f1) {
429 		if (blame->map1)
430 			return atomize_file_mmap(d, blame->map1,
431 			    blame->size1, blame->nlines1,
432 			    blame->line_offsets1);
433 		else
434 			return atomize_file(d, blame->f1, blame->size1,
435 			    blame->nlines1, blame->line_offsets1);
436 	} else if (d->f == blame->f2) {
437 		if (d->atoms.len > 0) {
438 			/* Re-use data from previous commit. */
439 			return DIFF_RC_OK;
440 		}
441 		if (blame->map2)
442 			return atomize_file_mmap(d, blame->map2,
443 			    blame->size2, blame->nlines2,
444 			    blame->line_offsets2);
445 		else
446 			return atomize_file(d, blame->f2, blame->size2,
447 			    blame->nlines2, blame->line_offsets2);
448 	}
449 
450 	return DIFF_RC_OK;
451 }
452 
453 static const struct got_error *
454 flip_files(struct got_blame *blame)
455 {
456 	const struct got_error *err = NULL;
457 	struct diff_data *d;
458 	FILE *tmp;
459 
460 	free(blame->line_offsets2);
461 	blame->line_offsets2 = blame->line_offsets1;
462 	blame->line_offsets1 = NULL;
463 
464 	free(blame->linemap2);
465 	blame->linemap2 = blame->linemap1;
466 	blame->linemap1 = NULL;
467 
468 	if (blame->map2) {
469 		if (munmap(blame->map2, blame->size2) == -1)
470 			return got_error_from_errno("munmap");
471 		blame->map2 = blame->map1;
472 		blame->map1 = NULL;
473 	}
474 	blame->size2 = blame->size1;
475 
476 	err = got_opentemp_truncate(blame->f2);
477 	if (err)
478 		return err;
479 	tmp = blame->f2;
480 	blame->f2 = blame->f1;
481 	blame->f1 = tmp;
482 	blame->size1 = 0;
483 
484 	blame->nlines2 = blame->nlines1;
485 	blame->nlines1 = 0;
486 
487 	diff_data_free(blame->data2); /* does not free pointer itself */
488 	memset(blame->data2, 0, sizeof(*blame->data2));
489 	d = blame->data2;
490 	blame->data2 = blame->data1;
491 	blame->data1 = d;
492 
493 	return NULL;
494 }
495 
496 static const struct got_error *
497 blame_open(struct got_blame **blamep, const char *path,
498     struct got_object_id *start_commit_id, struct got_repository *repo,
499     enum got_diff_algorithm diff_algo, got_blame_cb cb, void *arg,
500     got_cancel_cb cancel_cb, void *cancel_arg,
501     int fd1, int fd2, FILE *f1, FILE *f2)
502 {
503 	const struct got_error *err = NULL;
504 	struct got_commit_object *start_commit = NULL, *last_commit = NULL;
505 	struct got_object_id *obj_id = NULL;
506 	struct got_blob_object *blob = NULL;
507 	struct got_blame *blame = NULL;
508 	struct got_object_id id;
509 	int lineno, have_id = 0;
510 	struct got_commit_graph *graph = NULL;
511 
512 	*blamep = NULL;
513 
514 	err = got_object_open_as_commit(&start_commit, repo, start_commit_id);
515 	if (err)
516 		goto done;
517 
518 	err = got_object_id_by_path(&obj_id, repo, start_commit, path);
519 	if (err)
520 		goto done;
521 
522 	err = got_object_open_as_blob(&blob, repo, obj_id, 8192, fd1);
523 	if (err)
524 		goto done;
525 
526 	blame = calloc(1, sizeof(*blame));
527 	if (blame == NULL) {
528 		err = got_error_from_errno("calloc");
529 		goto done;
530 	}
531 
532 	blame->data1 = calloc(1, sizeof(*blame->data1));
533 	if (blame->data1 == NULL) {
534 		err = got_error_from_errno("calloc");
535 		goto done;
536 	}
537 	blame->data2 = calloc(1, sizeof(*blame->data2));
538 	if (blame->data2 == NULL) {
539 		err = got_error_from_errno("calloc");
540 		goto done;
541 	}
542 
543 	blame->f1 = f1;
544 	blame->f2 = f2;
545 	blame->fd = fd2;
546 
547 	err = got_diff_get_config(&blame->cfg, diff_algo, blame_atomize_file,
548 	    blame);
549 	if (err)
550 		goto done;
551 
552 	err = blame_prepare_file(blame->f2, &blame->map2, &blame->size2,
553 	    &blame->nlines2, &blame->line_offsets2, blame->data2,
554 	    blame->cfg, blob);
555 	blame->nlines = blame->nlines2;
556 	if (err || blame->nlines == 0)
557 		goto done;
558 
559 	got_object_blob_close(blob);
560 	blob = NULL;
561 
562 	/* Don't include \n at EOF in the blame line count. */
563 	if (blame->line_offsets2[blame->nlines - 1] == blame->size2)
564 		blame->nlines--;
565 
566 	blame->lines = calloc(blame->nlines, sizeof(*blame->lines));
567 	if (blame->lines == NULL) {
568 		err = got_error_from_errno("calloc");
569 		goto done;
570 	}
571 
572 	blame->linemap2 = calloc(blame->nlines2, sizeof(*blame->linemap2));
573 	if (blame->linemap2 == NULL) {
574 		err = got_error_from_errno("calloc");
575 		goto done;
576 	}
577 	for (lineno = 0; lineno < blame->nlines2; lineno++)
578 		blame->linemap2[lineno] = lineno;
579 
580 	err = got_commit_graph_open(&graph, path, 1);
581 	if (err)
582 		goto done;
583 
584 	err = got_commit_graph_iter_start(graph, start_commit_id, repo,
585 	    cancel_cb, cancel_arg);
586 	if (err)
587 		goto done;
588 	for (;;) {
589 		err = got_commit_graph_iter_next(&id, graph, repo,
590 		    cancel_cb, cancel_arg);
591 		if (err) {
592 			if (err->code == GOT_ERR_ITER_COMPLETED) {
593 				err = NULL;
594 				break;
595 			}
596 			goto done;
597 		}
598 		have_id = 1;
599 
600 		err = blame_commit(blame, &id, path, repo, cb, arg);
601 		if (err) {
602 			if (err->code == GOT_ERR_ITER_COMPLETED)
603 				err = NULL;
604 			goto done;
605 		}
606 		if (blame->nannotated == blame->nlines)
607 			break;
608 
609 		err = flip_files(blame);
610 		if (err)
611 			goto done;
612 	}
613 
614 	if (have_id && blame->nannotated < blame->nlines) {
615 		/* Annotate remaining non-annotated lines with last commit. */
616 		err = got_object_open_as_commit(&last_commit, repo, &id);
617 		if (err)
618 			goto done;
619 		for (lineno = 0; lineno < blame->nlines; lineno++) {
620 			err = annotate_line(blame, lineno, last_commit, &id,
621 			    cb, arg);
622 			if (err)
623 				goto done;
624 		}
625 	}
626 
627 done:
628 	if (graph)
629 		got_commit_graph_close(graph);
630 	free(obj_id);
631 	if (blob)
632 		got_object_blob_close(blob);
633 	if (start_commit)
634 		got_object_commit_close(start_commit);
635 	if (last_commit)
636 		got_object_commit_close(last_commit);
637 	if (err) {
638 		if (blame)
639 			blame_close(blame);
640 	} else
641 		*blamep = blame;
642 
643 	return err;
644 }
645 
646 const struct got_error *
647 got_blame(const char *path, struct got_object_id *commit_id,
648     struct got_repository *repo, enum got_diff_algorithm diff_algo,
649     got_blame_cb cb, void *arg, got_cancel_cb cancel_cb, void* cancel_arg,
650     int fd1, int fd2, FILE *f1, FILE *f2)
651 {
652 	const struct got_error *err = NULL, *close_err = NULL;
653 	struct got_blame *blame;
654 	char *abspath;
655 
656 	if (asprintf(&abspath, "%s%s", path[0] == '/' ? "" : "/", path) == -1)
657 		return got_error_from_errno2("asprintf", path);
658 
659 	err = blame_open(&blame, abspath, commit_id, repo, diff_algo,
660 	    cb, arg, cancel_cb, cancel_arg, fd1, fd2, f1, f2);
661 	free(abspath);
662 	if (blame)
663 		close_err = blame_close(blame);
664 	return err ? err : close_err;
665 }