commit bcbad685e6f97b8f9112a4c68d1c7937c5f519a3 from: Stefan Sperling via: Thomas Adam date: Sat Jul 26 10:36:08 2025 UTC fix off-by-one during deltification in maximum stretch size calculation Fix an off-by-one which causes invalid deltas to be written when common file sections exceed the maximum size which can be represented in a base-copy delta operation. This bug causes an invalid pack file to be written which neither gotadmin nor git will index successfully. Add a test which triggers the problem: got-index-pack: delta application result size mismatch: \ actual: 65536 expected: 16777216: bad delta ok op@ commit - 873d09e09fdcaceba69c02e8cefdb95cabfc80d9 commit + bcbad685e6f97b8f9112a4c68d1c7937c5f519a3 blob - 9620bdf873585daa098962e7bb8c27a0446cac38 blob + 8af57db82801b5d4479f4e41a2f0f533f9e209b8 --- lib/deltify.c +++ lib/deltify.c @@ -505,7 +505,7 @@ stretchblk(FILE *basefile, off_t base_offset0, struct SEEK_SET) == -1) return got_error_from_errno("fseeko"); - while (buf_equal && *blocklen < (1 << 24) - 1) { + while (buf_equal && *blocklen < GOT_DELTIFY_STRETCHMAX - 1) { base_r = fread(basebuf, 1, sizeof(basebuf), basefile); if (base_r == 0) { if (ferror(basefile)) @@ -523,6 +523,8 @@ stretchblk(FILE *basefile, off_t base_offset0, struct buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -559,6 +561,8 @@ stretchblk_file_mem(uint8_t *basedata, off_t base_offs buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -599,6 +603,8 @@ stretchblk_mem_file(FILE *basefile, off_t base_offset0 buf_equal = 0; break; } + if (*blocklen >= GOT_DELTIFY_STRETCHMAX) + break; (*blocklen)++; } } @@ -630,7 +636,7 @@ stretchblk_mem_mem(uint8_t *basedata, off_t base_offse p = data + fileoffset; q = basedata + base_offset; maxlen = MIN(basefile_size - base_offset, filesize - fileoffset); - for (i = 0; i < maxlen && *blocklen < (1 << 24) - 1; i++) { + for (i = 0; i < maxlen && *blocklen < GOT_DELTIFY_STRETCHMAX - 1; i++) { if (p[i] != q[i]) break; (*blocklen)++; blob - 973f1682b45bdf0b31cfe37ef1473caa9819418d blob + 644fb97f03e7d5c3669e32724188d86ee8f5ca27 --- lib/got_lib_deltify.h +++ lib/got_lib_deltify.h @@ -45,6 +45,7 @@ enum { GOT_DELTIFY_MINCHUNK = 32, GOT_DELTIFY_MAXCHUNK = 8192, GOT_DELTIFY_SPLITMASK = (1 << 8) - 1, + GOT_DELTIFY_STRETCHMAX = (1 << 24) - 1, }; const struct got_error *got_deltify_init(struct got_delta_table **dt, FILE *f, blob - ff0ad628633ae3a3e6c5bd87cc76f95a70977acb blob + 1698972f9fb04c46f3e9026125a1493811eb358d --- regress/gotd/Makefile +++ regress/gotd/Makefile @@ -7,7 +7,8 @@ REGRESS_TARGETS=test_repo_read test_repo_read_group \ test_repo_write_protected test_repo_write_readonly \ test_email_notification test_http_notification \ test_git_interop test_email_and_http_notification \ - test_http_notification_hmac test_connection_limit + test_http_notification_hmac test_connection_limit \ + test_large_loose_objects NOOBJ=Yes CLEANFILES=gotd.conf gotd-secrets.conf @@ -251,6 +252,10 @@ prepare_test_repo: ensure_root prepare_test_repo_empty: ensure_root @chown ${GOTD_USER} "${GOTD_TEST_REPO}" @su -m ${GOTD_USER} -c 'env $(GOTD_TEST_ENV) sh ./prepare_test_repo.sh 1' + +prepare_large_loose_objects: ensure_root prepare_test_repo + @su -m ${GOTD_USER} -c 'env $(GOTD_TEST_ENV) \ + sh ./prepare_large_loose_objects.sh' test_repo_read: prepare_test_repo start_gotd_ro @-$(GOTD_TRAP); su ${GOTD_TEST_USER} -c \ @@ -350,4 +355,9 @@ test_connection_limit: prepare_test_repo start_gotd_co 'env $(GOTD_TEST_ENV) sh ./connection_limit.sh' @$(GOTD_STOP_CMD) 2>/dev/null +test_large_loose_objects: prepare_large_loose_objects start_gotd_ro + @-$(GOTD_TRAP); su ${GOTD_TEST_USER} -c \ + 'env $(GOTD_TEST_ENV) sh ./large_loose_objects.sh' + @$(GOTD_STOP_CMD) 2>/dev/null + .include blob - /dev/null blob + 358b001d6584ee02bae559e83d62199d0b6ea030 (mode 644) --- /dev/null +++ regress/gotd/large_loose_objects.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Copyright (c) 2025 Stefan Sperling +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +. ../cmdline/common.sh +. ./common.sh + +test_clone_basic() { + local testroot=`test_init clone_basic 1` + + got clone -q ${GOTD_TEST_REPO_URL} $testroot/repo-clone + ret=$? + if [ $ret -ne 0 ]; then + echo "got clone failed unexpectedly" >&2 + test_done "$testroot" "1" + return 1 + fi + + # Verify that the clone operation worked fine. + git_fsck "$testroot" "$testroot/repo-clone" + ret=$? + if [ $ret -ne 0 ]; then + test_done "$testroot" "1" + return 1 + fi + + got tree -R -r "$testroot/repo-clone" > $testroot/stdout + cat > $testroot/stdout.expected < +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +test_tree=`mktemp -d "${GOTD_TEST_ROOT}/gotd-test-tree-XXXXXXXXXX"` +trap "rm -r $test_tree" HUP INT QUIT PIPE TERM + +got checkout -q "${GOTD_TEST_REPO}" "$test_tree" > /dev/null + +dd if=/dev/random of="$test_tree/large_file1" count=32768 status=none +(cd "$test_tree" && got add "$test_tree/large_file1" > /dev/null) +for i in 2 3; do + cp "$test_tree/large_file1" "$test_tree/large_file$i" + dd if=/dev/random of="$test_tree/large_file$i" seek=32768 count=64 \ + status=none + (cd "$test_tree" && got add "$test_tree/large_file$i" > /dev/null) +done + +(cd "$test_tree" && got commit -m "add large objects" "$test_tree" > /dev/null) + +rm -r "$test_tree"