From 9c454b007b57669e7baf2b8b69cf053f32a620a2 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 11 Jan 2013 22:13:02 -0800 Subject: [PATCH 01/14] Initial implementation of similarity scoring algo This adds a new `git_buf_text_hashsig` type and functions to generate these hash signatures and compare them to give a similarity score. This can be plugged into diff similarity scoring. --- src/buf_text.c | 302 +++++++++++++++++++++++++++++++++++++++ src/buf_text.h | 48 +++++++ tests-clar/core/buffer.c | 89 ++++++++++++ 3 files changed, 439 insertions(+) diff --git a/src/buf_text.c b/src/buf_text.c index 3a8f442b4..ab583f830 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -5,6 +5,7 @@ * a Linking Exception. For full terms see the included COPYING file. */ #include "buf_text.h" +#include "fileops.h" int git_buf_text_puts_escaped( git_buf *buf, @@ -212,3 +213,304 @@ bool git_buf_text_gather_stats( return (stats->nul > 0 || ((stats->printable >> 7) < stats->nonprintable)); } + +#define SIMILARITY_MAXRUN 256 +#define SIMILARITY_HASH_START 5381 +#define SIMILARITY_HASH_UPDATE(S,N) (((S) << 5) + (S) + (uint32_t)(N)) + +enum { + SIMILARITY_FORMAT_UNKNOWN = 0, + SIMILARITY_FORMAT_TEXT = 1, + SIMILARITY_FORMAT_BINARY = 2 +}; + +struct git_buf_text_hashsig { + uint32_t *hashes; + size_t size; + size_t asize; + unsigned int format : 2; + unsigned int pairs : 1; +}; + +static int similarity_advance(git_buf_text_hashsig *sig, uint32_t hash) +{ + if (sig->size >= sig->asize) { + size_t new_asize = sig->asize + 512; + uint32_t *new_hashes = + git__realloc(sig->hashes, new_asize * sizeof(uint32_t)); + GITERR_CHECK_ALLOC(new_hashes); + + sig->hashes = new_hashes; + sig->asize = new_asize; + } + + sig->hashes[sig->size++] = hash; + return 0; +} + +static int similarity_add_hashes( + git_buf_text_hashsig *sig, + uint32_t *hash_start, + size_t *hashlen_start, + const char *ptr, + size_t len) +{ + int error = 0; + const char *scan = ptr, *scan_end = ptr + len; + char term = (sig->format == SIMILARITY_FORMAT_TEXT) ? '\n' : '\0'; + uint32_t hash = hash_start ? *hash_start : SIMILARITY_HASH_START; + size_t hashlen = hashlen_start ? *hashlen_start : 0; + + while (scan < scan_end) { + char ch = *scan++; + + if (ch == term || hashlen >= SIMILARITY_MAXRUN) { + if ((error = similarity_advance(sig, hash)) < 0) + break; + + hash = SIMILARITY_HASH_START; + hashlen = 0; + + /* skip run of terminators */ + while (scan < scan_end && *scan == term) + scan++; + } else { + hash = SIMILARITY_HASH_UPDATE(hash, ch); + hashlen++; + } + } + + if (hash_start) + *hash_start = hash; + if (hashlen_start) + *hashlen_start = hashlen; + + /* if we're not saving intermediate state, add final hash as needed */ + if (!error && !hash_start && hashlen > 0) + error = similarity_advance(sig, hash); + + return error; +} + +/* + * Decide if \0 or \n terminated runs are a better choice for hashes + */ +static void similarity_guess_format( + git_buf_text_hashsig *sig, const char *ptr, size_t len) +{ + size_t lines = 0, line_length = 0, max_line_length = 0; + size_t runs = 0, run_length = 0, max_run_length = 0; + + /* don't process more than 4k of data for this */ + if (len > 4096) + len = 4096; + + /* gather some stats */ + while (len--) { + char ch = *ptr++; + + if (ch == '\0') { + runs++; + if (max_run_length < run_length) + max_run_length = run_length; + run_length = 0; + } else if (ch == '\n') { + lines++; + if (max_line_length < line_length) + max_line_length = line_length; + line_length = 0; + } else { + run_length++; + line_length++; + } + } + + /* the following heuristic could probably be improved */ + if (lines > runs) + sig->format = SIMILARITY_FORMAT_TEXT; + else if (runs > 0) + sig->format = SIMILARITY_FORMAT_BINARY; + else + sig->format = SIMILARITY_FORMAT_UNKNOWN; +} + +static int similarity_compare_score(const void *a, const void *b) +{ + uint32_t av = *(uint32_t *)a, bv = *(uint32_t *)b; + return (av < bv) ? -1 : (av > bv) ? 1 : 0; +} + +static int similarity_finalize_hashes( + git_buf_text_hashsig *sig, bool generate_pairs) +{ + if (!sig->size) + return 0; + + /* create pairwise hashes if requested */ + + if (generate_pairs) { + size_t i, needed_size = sig->size * 2 - 1; + + if (needed_size > sig->asize) { + uint32_t *new_hashes = + git__realloc(sig->hashes, needed_size * sizeof(uint32_t)); + GITERR_CHECK_ALLOC(new_hashes); + + sig->hashes = new_hashes; + sig->asize = needed_size; + } + + for (i = 1; i < sig->size; ++i) + sig->hashes[sig->size + i - 1] = + SIMILARITY_HASH_UPDATE(sig->hashes[i - 1], sig->hashes[i]); + + sig->pairs = 1; + } + + /* sort all hashes */ + + qsort(sig->hashes, sig->size, sizeof(uint32_t), similarity_compare_score); + + if (generate_pairs) + qsort(&sig->hashes[sig->size], sig->size - 1, sizeof(uint32_t), + similarity_compare_score); + + return 0; +} + +int git_buf_text_hashsig_create( + git_buf_text_hashsig **out, + const git_buf *buf, + bool generate_pairs) +{ + int error; + git_buf_text_hashsig *sig = git__calloc(1, sizeof(git_buf_text_hashsig)); + GITERR_CHECK_ALLOC(sig); + + similarity_guess_format(sig, buf->ptr, buf->size); + + error = similarity_add_hashes(sig, NULL, NULL, buf->ptr, buf->size); + + if (!error) + error = similarity_finalize_hashes(sig, generate_pairs); + + if (!error) + *out = sig; + else + git_buf_text_hashsig_free(sig); + + return error; +} + +int git_buf_text_hashsig_create_fromfile( + git_buf_text_hashsig **out, + const char *path, + bool generate_pairs) +{ + char buf[4096]; + ssize_t buflen = 0; + uint32_t hash = SIMILARITY_HASH_START; + size_t hashlen = 0; + int error = 0, fd; + git_buf_text_hashsig *sig = git__calloc(1, sizeof(git_buf_text_hashsig)); + GITERR_CHECK_ALLOC(sig); + + if ((fd = git_futils_open_ro(path)) < 0) { + git__free(sig); + return fd; + } + + while (!error && (buflen = p_read(fd, buf, sizeof(buf))) > 0) { + if (sig->format == SIMILARITY_FORMAT_UNKNOWN) + similarity_guess_format(sig, buf, buflen); + + error = similarity_add_hashes(sig, &hash, &hashlen, buf, buflen); + } + + if (buflen < 0) { + giterr_set(GITERR_OS, + "Read error on '%s' while calculating similarity hashes", path); + error = (int)buflen; + } + + p_close(fd); + + if (!error && hashlen > 0) + error = similarity_advance(sig, hash); + + if (!error) + error = similarity_finalize_hashes(sig, generate_pairs); + + if (!error) + *out = sig; + else + git_buf_text_hashsig_free(sig); + + return error; +} + +void git_buf_text_hashsig_free(git_buf_text_hashsig *sig) +{ + if (!sig) + return; + + if (sig->hashes) { + git__free(sig->hashes); + sig->hashes = NULL; + } + + git__free(sig); +} + +int git_buf_text_hashsig_compare( + const git_buf_text_hashsig *a, + const git_buf_text_hashsig *b, + int scale) +{ + size_t matches = 0, pairs = 0, total = 0, i, j; + + if (a->format != b->format || !a->size || !b->size) + return 0; + + if (scale <= 0) + scale = 100; + + /* hash lists are sorted - just look for overlap vs total */ + + for (i = 0, j = 0; i < a->size && j < b->size; ) { + uint32_t av = a->hashes[i]; + uint32_t bv = b->hashes[j]; + + if (av < bv) + ++i; + else if (av > bv) + ++j; + else { + ++i; ++j; + ++matches; + } + } + + total = (a->size + b->size); + + if (a->pairs && b->pairs) { + for (i = 0, j = 0; i < a->size - 1 && j < b->size - 1; ) { + uint32_t av = a->hashes[i + a->size]; + uint32_t bv = b->hashes[j + b->size]; + + if (av < bv) + ++i; + else if (av > bv) + ++j; + else { + ++i; ++j; + ++pairs; + } + } + + total += (a->size + b->size - 2); + } + + return (int)(scale * 2 * (matches + pairs) / total); +} + diff --git a/src/buf_text.h b/src/buf_text.h index 458ee33c9..c48c010d3 100644 --- a/src/buf_text.h +++ b/src/buf_text.h @@ -105,4 +105,52 @@ extern int git_buf_text_detect_bom( extern bool git_buf_text_gather_stats( git_buf_text_stats *stats, const git_buf *buf, bool skip_bom); +/** + * Similarity signature of line hashes for a buffer + */ +typedef struct git_buf_text_hashsig git_buf_text_hashsig; + +/** + * Build a similarity signature for a buffer + * + * This can either generate a simple array of hashed lines/runs in the + * file, or it can also keep hashes of pairs of runs in sequence. Adding + * the pairwise runs means the final score will be sensitive to line + * ordering changes as well as individual line contents. + * + * @param out The array of hashed runs representing the file content + * @param buf The contents of the file to hash + * @param generate_pairwise_hashes Should pairwise runs be hashed + */ +extern int git_buf_text_hashsig_create( + git_buf_text_hashsig **out, + const git_buf *buf, + bool generate_pairwise_hashes); + +/** + * Build a similarity signature from a file + * + * This walks through the file, only loading a maximum of 4K of file data at + * a time. Otherwise, it acts just like `git_buf_text_hashsig_create`. + */ +extern int git_buf_text_hashsig_create_fromfile( + git_buf_text_hashsig **out, + const char *path, + bool generate_pairwise_hashes); + +/** + * Release memory for a content similarity signature + */ +extern void git_buf_text_hashsig_free(git_buf_text_hashsig *sig); + +/** + * Measure similarity between two files + * + * @return <0 for error, [0 to scale] as similarity score + */ +extern int git_buf_text_hashsig_compare( + const git_buf_text_hashsig *a, + const git_buf_text_hashsig *b, + int scale); + #endif diff --git a/tests-clar/core/buffer.c b/tests-clar/core/buffer.c index 49ab41f71..63753bb67 100644 --- a/tests-clar/core/buffer.c +++ b/tests-clar/core/buffer.c @@ -1,6 +1,7 @@ #include "clar_libgit2.h" #include "buffer.h" #include "buf_text.h" +#include "fileops.h" #define TESTSTR "Have you seen that? Have you seeeen that??" const char *test_string = TESTSTR; @@ -730,3 +731,91 @@ void test_core_buffer__classify_with_utf8(void) cl_assert(git_buf_text_is_binary(&b)); cl_assert(git_buf_text_contains_nul(&b)); } + +void test_core_buffer__similarity_metric(void) +{ + git_buf_text_hashsig *a, *b; + git_buf buf = GIT_BUF_INIT; + int sim; + + /* in the first case, we compare data to itself and expect 100% match */ + + cl_git_pass(git_buf_sets(&buf, "test data\nright here\ninline\ntada")); + cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + + cl_assert_equal_i(100, git_buf_text_hashsig_compare(a, b, 100)); + + git_buf_text_hashsig_free(a); + git_buf_text_hashsig_free(b); + + /* in the second case, half of a is matched and all of b is matched, so + * we'll expect a score of around 66% to be the similarity score + */ + + cl_git_pass( + git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); + cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + + cl_git_pass(git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh")); + cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + + sim = git_buf_text_hashsig_compare(a, b, 100); + cl_assert(sim > 60 && sim < 70); + + git_buf_text_hashsig_free(a); + git_buf_text_hashsig_free(b); + + /* in the reversed case, 100% of line hashes match, but no pairwise hashes + * match, so we'll expect about a 50% match for a reversed file + */ + + cl_git_pass( + git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); + cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + cl_git_pass( + git_buf_sets(&buf, "p\no\nn\nm\nl\nk\nj\ni\nh\ng\nf\ne\nd\nc\nb\na\n")); + cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + + sim = git_buf_text_hashsig_compare(a, b, 100); + cl_assert(sim > 45 && sim < 55); + + git_buf_text_hashsig_free(a); + git_buf_text_hashsig_free(b); + + /* if we don't use pairwise signatures, then a reversed file should + * match 100% + */ + + cl_git_pass( + git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); + cl_git_pass(git_buf_text_hashsig_create(&a, &buf, false)); + cl_git_pass( + git_buf_sets(&buf, "p\no\nn\nm\nl\nk\nj\ni\nh\ng\nf\ne\nd\nc\nb\na\n")); + cl_git_pass(git_buf_text_hashsig_create(&b, &buf, false)); + + sim = git_buf_text_hashsig_compare(a, b, 100); + cl_assert_equal_i(100, sim); + + git_buf_text_hashsig_free(a); + git_buf_text_hashsig_free(b); + + /* lastly, let's check that we can hash file content as well */ + + cl_git_pass( + git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); + cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + + cl_git_pass(git_futils_mkdir("scratch", NULL, 0755, GIT_MKDIR_PATH)); + cl_git_mkfile("scratch/testdata", + "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n"); + cl_git_pass(git_buf_text_hashsig_create_fromfile(&b, "scratch/testdata", true)); + + cl_assert_equal_i(100, git_buf_text_hashsig_compare(a, b, 100)); + + git_buf_text_hashsig_free(a); + git_buf_text_hashsig_free(b); + + git_buf_free(&buf); + git_futils_rmdir_r("scratch", NULL, GIT_RMDIR_REMOVE_FILES); +} From f3327cac1d5cc47d37eb79eb4235f40e447f3ef9 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Sun, 13 Jan 2013 10:06:09 -0800 Subject: [PATCH 02/14] Some similarity metric adjustments This makes the text similarity metric treat \r as equivalent to \n and makes it skip whitespace immediately following a line terminator, so line indentation will have less effect on the difference measurement (and so \r\n will be treated as just a single line terminator). This also separates the text and binary hash calculators into two separate functions instead of have more if statements inside the loop. This should make it easier to have more differentiated heuristics in the future if we so wish. --- src/buf_text.c | 104 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/src/buf_text.c b/src/buf_text.c index ab583f830..49ec16aaf 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -232,7 +232,7 @@ struct git_buf_text_hashsig { unsigned int pairs : 1; }; -static int similarity_advance(git_buf_text_hashsig *sig, uint32_t hash) +static int similarity_record_hash(git_buf_text_hashsig *sig, uint32_t hash) { if (sig->size >= sig->asize) { size_t new_asize = sig->asize + 512; @@ -248,6 +248,80 @@ static int similarity_advance(git_buf_text_hashsig *sig, uint32_t hash) return 0; } +static int similarity_add_hashes_text( + git_buf_text_hashsig *sig, + uint32_t *hash_start, + size_t *hashlen_start, + const char *ptr, + size_t len) +{ + int error; + const char *scan = ptr, *scan_end = ptr + len; + uint32_t hash = *hash_start; + size_t hashlen = *hashlen_start; + + while (scan < scan_end) { + char ch = *scan++; + + if (ch == '\r' || ch == '\n' || hashlen >= SIMILARITY_MAXRUN) { + if ((error = similarity_record_hash(sig, hash)) < 0) + break; + + hash = SIMILARITY_HASH_START; + hashlen = 0; + + /* skip all whitespace immediately after line ending */ + while (scan < scan_end && git__isspace(*scan)) + scan++; + } else { + hash = SIMILARITY_HASH_UPDATE(hash, ch); + hashlen++; + } + } + + *hash_start = hash; + *hashlen_start = hashlen; + + return error; +} + +static int similarity_add_hashes_binary( + git_buf_text_hashsig *sig, + uint32_t *hash_start, + size_t *hashlen_start, + const char *ptr, + size_t len) +{ + int error; + const char *scan = ptr, *scan_end = ptr + len; + uint32_t hash = *hash_start; + size_t hashlen = *hashlen_start; + + while (scan < scan_end) { + char ch = *scan++; + + if (!ch || hashlen >= SIMILARITY_MAXRUN) { + if ((error = similarity_record_hash(sig, hash)) < 0) + break; + + hash = SIMILARITY_HASH_START; + hashlen = 0; + + /* skip run of terminators */ + while (scan < scan_end && !*scan) + scan++; + } else { + hash = SIMILARITY_HASH_UPDATE(hash, ch); + hashlen++; + } + } + + *hash_start = hash; + *hashlen_start = hashlen; + + return error; +} + static int similarity_add_hashes( git_buf_text_hashsig *sig, uint32_t *hash_start, @@ -256,29 +330,13 @@ static int similarity_add_hashes( size_t len) { int error = 0; - const char *scan = ptr, *scan_end = ptr + len; - char term = (sig->format == SIMILARITY_FORMAT_TEXT) ? '\n' : '\0'; uint32_t hash = hash_start ? *hash_start : SIMILARITY_HASH_START; size_t hashlen = hashlen_start ? *hashlen_start : 0; - while (scan < scan_end) { - char ch = *scan++; - - if (ch == term || hashlen >= SIMILARITY_MAXRUN) { - if ((error = similarity_advance(sig, hash)) < 0) - break; - - hash = SIMILARITY_HASH_START; - hashlen = 0; - - /* skip run of terminators */ - while (scan < scan_end && *scan == term) - scan++; - } else { - hash = SIMILARITY_HASH_UPDATE(hash, ch); - hashlen++; - } - } + if (sig->format == SIMILARITY_FORMAT_TEXT) + error = similarity_add_hashes_text(sig, &hash, &hashlen, ptr, len); + else + error = similarity_add_hashes_binary(sig, &hash, &hashlen, ptr, len); if (hash_start) *hash_start = hash; @@ -287,7 +345,7 @@ static int similarity_add_hashes( /* if we're not saving intermediate state, add final hash as needed */ if (!error && !hash_start && hashlen > 0) - error = similarity_advance(sig, hash); + error = similarity_record_hash(sig, hash); return error; } @@ -436,7 +494,7 @@ int git_buf_text_hashsig_create_fromfile( p_close(fd); if (!error && hashlen > 0) - error = similarity_advance(sig, hash); + error = similarity_record_hash(sig, hash); if (!error) error = similarity_finalize_hashes(sig, generate_pairs); From 99ba8f2322eaa2df51ace9782b8eadc8c5a6e8b8 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Tue, 22 Jan 2013 15:27:08 -0800 Subject: [PATCH 03/14] wip: adding metric to diff --- src/diff_tform.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index 2c2e1fb19..dbcdc277f 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -7,6 +7,7 @@ #include "common.h" #include "diff.h" #include "git2/config.h" +#include "buf_text.h" static git_diff_delta *diff_delta__dup( const git_diff_delta *d, git_pool *pool) @@ -297,10 +298,15 @@ on_error: return -1; } +typedef struct { + /* array of delta index * 2 + (old_file/new_file) -> file hashes */ + git_buf_text_hashsig *sigs; +} diff_similarity_cache; + static unsigned int calc_similarity( - void *cache, git_diff_file *old_file, git_diff_file *new_file) + void *ref, git_diff_file *old_file, git_diff_file *new_file) { - GIT_UNUSED(cache); + diff_similarity_cache *cache = ref; if (git_oid_cmp(&old_file->oid, &new_file->oid) == 0) return 100; From 5e5848eb15cc0dd8476d1c6882a9f770e6556586 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 14 Feb 2013 17:25:10 -0800 Subject: [PATCH 04/14] Change similarity metric to sampled hashes This moves the similarity metric code out of buf_text and into a new file. Also, this implements a different approach to similarity measurement based on a Rabin-Karp rolling hash where we only keep the top 100 and bottom 100 hashes. In theory, that should be sufficient samples to given a fairly accurate measurement while limiting the amount of data we keep for file signatures no matter how large the file is. --- src/buf_text.c | 360 -------------------------------------- src/buf_text.h | 48 ------ src/diff_tform.c | 6 +- src/hashsig.c | 364 +++++++++++++++++++++++++++++++++++++++ src/hashsig.h | 70 ++++++++ tests-clar/core/buffer.c | 114 ++++++------ 6 files changed, 498 insertions(+), 464 deletions(-) create mode 100644 src/hashsig.c create mode 100644 src/hashsig.h diff --git a/src/buf_text.c b/src/buf_text.c index 49ec16aaf..3a8f442b4 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -5,7 +5,6 @@ * a Linking Exception. For full terms see the included COPYING file. */ #include "buf_text.h" -#include "fileops.h" int git_buf_text_puts_escaped( git_buf *buf, @@ -213,362 +212,3 @@ bool git_buf_text_gather_stats( return (stats->nul > 0 || ((stats->printable >> 7) < stats->nonprintable)); } - -#define SIMILARITY_MAXRUN 256 -#define SIMILARITY_HASH_START 5381 -#define SIMILARITY_HASH_UPDATE(S,N) (((S) << 5) + (S) + (uint32_t)(N)) - -enum { - SIMILARITY_FORMAT_UNKNOWN = 0, - SIMILARITY_FORMAT_TEXT = 1, - SIMILARITY_FORMAT_BINARY = 2 -}; - -struct git_buf_text_hashsig { - uint32_t *hashes; - size_t size; - size_t asize; - unsigned int format : 2; - unsigned int pairs : 1; -}; - -static int similarity_record_hash(git_buf_text_hashsig *sig, uint32_t hash) -{ - if (sig->size >= sig->asize) { - size_t new_asize = sig->asize + 512; - uint32_t *new_hashes = - git__realloc(sig->hashes, new_asize * sizeof(uint32_t)); - GITERR_CHECK_ALLOC(new_hashes); - - sig->hashes = new_hashes; - sig->asize = new_asize; - } - - sig->hashes[sig->size++] = hash; - return 0; -} - -static int similarity_add_hashes_text( - git_buf_text_hashsig *sig, - uint32_t *hash_start, - size_t *hashlen_start, - const char *ptr, - size_t len) -{ - int error; - const char *scan = ptr, *scan_end = ptr + len; - uint32_t hash = *hash_start; - size_t hashlen = *hashlen_start; - - while (scan < scan_end) { - char ch = *scan++; - - if (ch == '\r' || ch == '\n' || hashlen >= SIMILARITY_MAXRUN) { - if ((error = similarity_record_hash(sig, hash)) < 0) - break; - - hash = SIMILARITY_HASH_START; - hashlen = 0; - - /* skip all whitespace immediately after line ending */ - while (scan < scan_end && git__isspace(*scan)) - scan++; - } else { - hash = SIMILARITY_HASH_UPDATE(hash, ch); - hashlen++; - } - } - - *hash_start = hash; - *hashlen_start = hashlen; - - return error; -} - -static int similarity_add_hashes_binary( - git_buf_text_hashsig *sig, - uint32_t *hash_start, - size_t *hashlen_start, - const char *ptr, - size_t len) -{ - int error; - const char *scan = ptr, *scan_end = ptr + len; - uint32_t hash = *hash_start; - size_t hashlen = *hashlen_start; - - while (scan < scan_end) { - char ch = *scan++; - - if (!ch || hashlen >= SIMILARITY_MAXRUN) { - if ((error = similarity_record_hash(sig, hash)) < 0) - break; - - hash = SIMILARITY_HASH_START; - hashlen = 0; - - /* skip run of terminators */ - while (scan < scan_end && !*scan) - scan++; - } else { - hash = SIMILARITY_HASH_UPDATE(hash, ch); - hashlen++; - } - } - - *hash_start = hash; - *hashlen_start = hashlen; - - return error; -} - -static int similarity_add_hashes( - git_buf_text_hashsig *sig, - uint32_t *hash_start, - size_t *hashlen_start, - const char *ptr, - size_t len) -{ - int error = 0; - uint32_t hash = hash_start ? *hash_start : SIMILARITY_HASH_START; - size_t hashlen = hashlen_start ? *hashlen_start : 0; - - if (sig->format == SIMILARITY_FORMAT_TEXT) - error = similarity_add_hashes_text(sig, &hash, &hashlen, ptr, len); - else - error = similarity_add_hashes_binary(sig, &hash, &hashlen, ptr, len); - - if (hash_start) - *hash_start = hash; - if (hashlen_start) - *hashlen_start = hashlen; - - /* if we're not saving intermediate state, add final hash as needed */ - if (!error && !hash_start && hashlen > 0) - error = similarity_record_hash(sig, hash); - - return error; -} - -/* - * Decide if \0 or \n terminated runs are a better choice for hashes - */ -static void similarity_guess_format( - git_buf_text_hashsig *sig, const char *ptr, size_t len) -{ - size_t lines = 0, line_length = 0, max_line_length = 0; - size_t runs = 0, run_length = 0, max_run_length = 0; - - /* don't process more than 4k of data for this */ - if (len > 4096) - len = 4096; - - /* gather some stats */ - while (len--) { - char ch = *ptr++; - - if (ch == '\0') { - runs++; - if (max_run_length < run_length) - max_run_length = run_length; - run_length = 0; - } else if (ch == '\n') { - lines++; - if (max_line_length < line_length) - max_line_length = line_length; - line_length = 0; - } else { - run_length++; - line_length++; - } - } - - /* the following heuristic could probably be improved */ - if (lines > runs) - sig->format = SIMILARITY_FORMAT_TEXT; - else if (runs > 0) - sig->format = SIMILARITY_FORMAT_BINARY; - else - sig->format = SIMILARITY_FORMAT_UNKNOWN; -} - -static int similarity_compare_score(const void *a, const void *b) -{ - uint32_t av = *(uint32_t *)a, bv = *(uint32_t *)b; - return (av < bv) ? -1 : (av > bv) ? 1 : 0; -} - -static int similarity_finalize_hashes( - git_buf_text_hashsig *sig, bool generate_pairs) -{ - if (!sig->size) - return 0; - - /* create pairwise hashes if requested */ - - if (generate_pairs) { - size_t i, needed_size = sig->size * 2 - 1; - - if (needed_size > sig->asize) { - uint32_t *new_hashes = - git__realloc(sig->hashes, needed_size * sizeof(uint32_t)); - GITERR_CHECK_ALLOC(new_hashes); - - sig->hashes = new_hashes; - sig->asize = needed_size; - } - - for (i = 1; i < sig->size; ++i) - sig->hashes[sig->size + i - 1] = - SIMILARITY_HASH_UPDATE(sig->hashes[i - 1], sig->hashes[i]); - - sig->pairs = 1; - } - - /* sort all hashes */ - - qsort(sig->hashes, sig->size, sizeof(uint32_t), similarity_compare_score); - - if (generate_pairs) - qsort(&sig->hashes[sig->size], sig->size - 1, sizeof(uint32_t), - similarity_compare_score); - - return 0; -} - -int git_buf_text_hashsig_create( - git_buf_text_hashsig **out, - const git_buf *buf, - bool generate_pairs) -{ - int error; - git_buf_text_hashsig *sig = git__calloc(1, sizeof(git_buf_text_hashsig)); - GITERR_CHECK_ALLOC(sig); - - similarity_guess_format(sig, buf->ptr, buf->size); - - error = similarity_add_hashes(sig, NULL, NULL, buf->ptr, buf->size); - - if (!error) - error = similarity_finalize_hashes(sig, generate_pairs); - - if (!error) - *out = sig; - else - git_buf_text_hashsig_free(sig); - - return error; -} - -int git_buf_text_hashsig_create_fromfile( - git_buf_text_hashsig **out, - const char *path, - bool generate_pairs) -{ - char buf[4096]; - ssize_t buflen = 0; - uint32_t hash = SIMILARITY_HASH_START; - size_t hashlen = 0; - int error = 0, fd; - git_buf_text_hashsig *sig = git__calloc(1, sizeof(git_buf_text_hashsig)); - GITERR_CHECK_ALLOC(sig); - - if ((fd = git_futils_open_ro(path)) < 0) { - git__free(sig); - return fd; - } - - while (!error && (buflen = p_read(fd, buf, sizeof(buf))) > 0) { - if (sig->format == SIMILARITY_FORMAT_UNKNOWN) - similarity_guess_format(sig, buf, buflen); - - error = similarity_add_hashes(sig, &hash, &hashlen, buf, buflen); - } - - if (buflen < 0) { - giterr_set(GITERR_OS, - "Read error on '%s' while calculating similarity hashes", path); - error = (int)buflen; - } - - p_close(fd); - - if (!error && hashlen > 0) - error = similarity_record_hash(sig, hash); - - if (!error) - error = similarity_finalize_hashes(sig, generate_pairs); - - if (!error) - *out = sig; - else - git_buf_text_hashsig_free(sig); - - return error; -} - -void git_buf_text_hashsig_free(git_buf_text_hashsig *sig) -{ - if (!sig) - return; - - if (sig->hashes) { - git__free(sig->hashes); - sig->hashes = NULL; - } - - git__free(sig); -} - -int git_buf_text_hashsig_compare( - const git_buf_text_hashsig *a, - const git_buf_text_hashsig *b, - int scale) -{ - size_t matches = 0, pairs = 0, total = 0, i, j; - - if (a->format != b->format || !a->size || !b->size) - return 0; - - if (scale <= 0) - scale = 100; - - /* hash lists are sorted - just look for overlap vs total */ - - for (i = 0, j = 0; i < a->size && j < b->size; ) { - uint32_t av = a->hashes[i]; - uint32_t bv = b->hashes[j]; - - if (av < bv) - ++i; - else if (av > bv) - ++j; - else { - ++i; ++j; - ++matches; - } - } - - total = (a->size + b->size); - - if (a->pairs && b->pairs) { - for (i = 0, j = 0; i < a->size - 1 && j < b->size - 1; ) { - uint32_t av = a->hashes[i + a->size]; - uint32_t bv = b->hashes[j + b->size]; - - if (av < bv) - ++i; - else if (av > bv) - ++j; - else { - ++i; ++j; - ++pairs; - } - } - - total += (a->size + b->size - 2); - } - - return (int)(scale * 2 * (matches + pairs) / total); -} - diff --git a/src/buf_text.h b/src/buf_text.h index c48c010d3..458ee33c9 100644 --- a/src/buf_text.h +++ b/src/buf_text.h @@ -105,52 +105,4 @@ extern int git_buf_text_detect_bom( extern bool git_buf_text_gather_stats( git_buf_text_stats *stats, const git_buf *buf, bool skip_bom); -/** - * Similarity signature of line hashes for a buffer - */ -typedef struct git_buf_text_hashsig git_buf_text_hashsig; - -/** - * Build a similarity signature for a buffer - * - * This can either generate a simple array of hashed lines/runs in the - * file, or it can also keep hashes of pairs of runs in sequence. Adding - * the pairwise runs means the final score will be sensitive to line - * ordering changes as well as individual line contents. - * - * @param out The array of hashed runs representing the file content - * @param buf The contents of the file to hash - * @param generate_pairwise_hashes Should pairwise runs be hashed - */ -extern int git_buf_text_hashsig_create( - git_buf_text_hashsig **out, - const git_buf *buf, - bool generate_pairwise_hashes); - -/** - * Build a similarity signature from a file - * - * This walks through the file, only loading a maximum of 4K of file data at - * a time. Otherwise, it acts just like `git_buf_text_hashsig_create`. - */ -extern int git_buf_text_hashsig_create_fromfile( - git_buf_text_hashsig **out, - const char *path, - bool generate_pairwise_hashes); - -/** - * Release memory for a content similarity signature - */ -extern void git_buf_text_hashsig_free(git_buf_text_hashsig *sig); - -/** - * Measure similarity between two files - * - * @return <0 for error, [0 to scale] as similarity score - */ -extern int git_buf_text_hashsig_compare( - const git_buf_text_hashsig *a, - const git_buf_text_hashsig *b, - int scale); - #endif diff --git a/src/diff_tform.c b/src/diff_tform.c index dbcdc277f..3939230b7 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -7,7 +7,7 @@ #include "common.h" #include "diff.h" #include "git2/config.h" -#include "buf_text.h" +#include "hashsig.h" static git_diff_delta *diff_delta__dup( const git_diff_delta *d, git_pool *pool) @@ -300,7 +300,7 @@ on_error: typedef struct { /* array of delta index * 2 + (old_file/new_file) -> file hashes */ - git_buf_text_hashsig *sigs; + git_hashsig *sigs; } diff_similarity_cache; static unsigned int calc_similarity( @@ -308,6 +308,8 @@ static unsigned int calc_similarity( { diff_similarity_cache *cache = ref; + GIT_UNUSED(cache); + if (git_oid_cmp(&old_file->oid, &new_file->oid) == 0) return 100; diff --git a/src/hashsig.c b/src/hashsig.c new file mode 100644 index 000000000..a9cd8fa53 --- /dev/null +++ b/src/hashsig.c @@ -0,0 +1,364 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#include "hashsig.h" +#include "fileops.h" + +typedef uint32_t hashsig_t; +typedef uint64_t hashsig_state; + +#define HASHSIG_SCALE 100 + +#define HASHSIG_HASH_WINDOW 32 +#define HASHSIG_HASH_START 0 +#define HASHSIG_HASH_SHIFT 5 +#define HASHSIG_HASH_MASK 0x7FFFFFFF + +#define HASHSIG_HEAP_SIZE ((1 << 7) - 1) + +typedef int (*hashsig_cmp)(const void *a, const void *b); + +typedef struct { + int size, asize; + hashsig_cmp cmp; + hashsig_t values[HASHSIG_HEAP_SIZE]; +} hashsig_heap; + +typedef struct { + hashsig_state state, shift_n; + char window[HASHSIG_HASH_WINDOW]; + int win_len, win_pos, saw_lf; +} hashsig_in_progress; + +#define HASHSIG_IN_PROGRESS_INIT { HASHSIG_HASH_START, 1, {0}, 0, 0, 1 } + +struct git_hashsig { + hashsig_heap mins; + hashsig_heap maxs; + git_hashsig_option_t opt; + int considered; +}; + +#define HEAP_LCHILD_OF(I) (((I)*2)+1) +#define HEAP_RCHILD_OF(I) (((I)*2)+2) +#define HEAP_PARENT_OF(I) (((I)-1)>>1) + +static void hashsig_heap_init(hashsig_heap *h, hashsig_cmp cmp) +{ + h->size = 0; + h->asize = HASHSIG_HEAP_SIZE; + h->cmp = cmp; +} + +static int hashsig_cmp_max(const void *a, const void *b) +{ + hashsig_t av = *(const hashsig_t *)a, bv = *(const hashsig_t *)b; + return (av < bv) ? -1 : (av > bv) ? 1 : 0; +} + +static int hashsig_cmp_min(const void *a, const void *b) +{ + hashsig_t av = *(const hashsig_t *)a, bv = *(const hashsig_t *)b; + return (av > bv) ? -1 : (av < bv) ? 1 : 0; +} + +static void hashsig_heap_up(hashsig_heap *h, int el) +{ + int parent_el = HEAP_PARENT_OF(el); + + while (el > 0 && h->cmp(&h->values[parent_el], &h->values[el]) > 0) { + hashsig_t t = h->values[el]; + h->values[el] = h->values[parent_el]; + h->values[parent_el] = t; + + el = parent_el; + parent_el = HEAP_PARENT_OF(el); + } +} + +static void hashsig_heap_down(hashsig_heap *h, int el) +{ + hashsig_t v, lv, rv; + + /* 'el < h->size / 2' tests if el is bottom row of heap */ + + while (el < h->size / 2) { + int lel = HEAP_LCHILD_OF(el), rel = HEAP_RCHILD_OF(el), swapel; + + v = h->values[el]; + lv = h->values[lel]; + rv = h->values[rel]; + + if (h->cmp(&v, &lv) < 0 && h->cmp(&v, &rv) < 0) + break; + + swapel = (h->cmp(&lv, &rv) < 0) ? lel : rel; + + h->values[el] = h->values[swapel]; + h->values[swapel] = v; + + el = swapel; + } +} + +static void hashsig_heap_sort(hashsig_heap *h) +{ + /* only need to do this at the end for signature comparison */ + qsort(h->values, h->size, sizeof(hashsig_t), h->cmp); +} + +static void hashsig_heap_insert(hashsig_heap *h, hashsig_t val) +{ + /* if heap is full, pop top if new element should replace it */ + if (h->size == h->asize && h->cmp(&val, &h->values[0]) > 0) { + h->size--; + h->values[0] = h->values[h->size]; + hashsig_heap_down(h, 0); + } + + /* if heap is not full, insert new element */ + if (h->size < h->asize) { + h->values[h->size++] = val; + hashsig_heap_up(h, h->size - 1); + } +} + +GIT_INLINE(bool) hashsig_include_char( + char ch, git_hashsig_option_t opt, int *saw_lf) +{ + if ((opt & GIT_HASHSIG_IGNORE_WHITESPACE) && git__isspace(ch)) + return false; + + if (opt & GIT_HASHSIG_SMART_WHITESPACE) { + if (ch == '\r' || (*saw_lf && git__isspace(ch))) + return false; + + *saw_lf = (ch == '\n'); + } + + return true; +} + +static void hashsig_initial_window( + git_hashsig *sig, + const char **data, + size_t size, + hashsig_in_progress *prog) +{ + hashsig_state state, shift_n; + int win_len; + const char *scan, *end; + + /* init until we have processed at least HASHSIG_HASH_WINDOW data */ + + if (prog->win_len >= HASHSIG_HASH_WINDOW) + return; + + state = prog->state; + win_len = prog->win_len; + shift_n = prog->shift_n; + + scan = *data; + end = scan + size; + + while (scan < end && win_len < HASHSIG_HASH_WINDOW) { + char ch = *scan++; + + if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf)) + continue; + + state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK; + + if (!win_len) + shift_n = 1; + else + shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; + + prog->window[win_len++] = ch; + } + + /* insert initial hash if we just finished */ + + if (win_len == HASHSIG_HASH_WINDOW) { + hashsig_heap_insert(&sig->mins, state); + hashsig_heap_insert(&sig->maxs, state); + sig->considered = 1; + } + + prog->state = state; + prog->win_len = win_len; + prog->shift_n = shift_n; + + *data = scan; +} + +static int hashsig_add_hashes( + git_hashsig *sig, + const char *data, + size_t size, + hashsig_in_progress *prog) +{ + const char *scan = data, *end = data + size; + hashsig_state state, shift_n, rmv; + + if (prog->win_len < HASHSIG_HASH_WINDOW) + hashsig_initial_window(sig, &scan, size, prog); + + state = prog->state; + shift_n = prog->shift_n; + + /* advance window, adding new chars and removing old */ + + for (; scan < end; ++scan) { + char ch = *scan; + + if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf)) + continue; + + rmv = shift_n * prog->window[prog->win_pos]; + + state = (state - rmv) & HASHSIG_HASH_MASK; + state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK; + state = (state + ch) & HASHSIG_HASH_MASK; + + hashsig_heap_insert(&sig->mins, state); + hashsig_heap_insert(&sig->maxs, state); + sig->considered++; + + prog->window[prog->win_pos] = ch; + prog->win_pos = (prog->win_pos + 1) % HASHSIG_HASH_WINDOW; + } + + prog->state = state; + + return 0; +} + +static int hashsig_finalize_hashes(git_hashsig *sig) +{ + if (sig->mins.size < HASHSIG_HEAP_SIZE) { + giterr_set(GITERR_INVALID, + "File too small for similarity signature calculation"); + return GIT_EBUFS; + } + + hashsig_heap_sort(&sig->mins); + hashsig_heap_sort(&sig->maxs); + + return 0; +} + +static git_hashsig *hashsig_alloc(git_hashsig_option_t opts) +{ + git_hashsig *sig = git__calloc(1, sizeof(git_hashsig)); + if (!sig) + return NULL; + + hashsig_heap_init(&sig->mins, hashsig_cmp_min); + hashsig_heap_init(&sig->maxs, hashsig_cmp_max); + sig->opt = opts; + + return sig; +} + +int git_hashsig_create( + git_hashsig **out, + const git_buf *buf, + git_hashsig_option_t opts) +{ + int error; + hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT; + git_hashsig *sig = hashsig_alloc(opts); + GITERR_CHECK_ALLOC(sig); + + error = hashsig_add_hashes(sig, buf->ptr, buf->size, &prog); + + if (!error) + error = hashsig_finalize_hashes(sig); + + if (!error) + *out = sig; + else + git_hashsig_free(sig); + + return error; +} + +int git_hashsig_create_fromfile( + git_hashsig **out, + const char *path, + git_hashsig_option_t opts) +{ + char buf[4096]; + ssize_t buflen = 0; + int error = 0, fd; + hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT; + git_hashsig *sig = hashsig_alloc(opts); + GITERR_CHECK_ALLOC(sig); + + if ((fd = git_futils_open_ro(path)) < 0) { + git__free(sig); + return fd; + } + + while (!error) { + if ((buflen = p_read(fd, buf, sizeof(buf))) <= 0) { + if ((error = buflen) < 0) + giterr_set(GITERR_OS, + "Read error on '%s' calculating similarity hashes", path); + break; + } + + error = hashsig_add_hashes(sig, buf, buflen, &prog); + } + + p_close(fd); + + if (!error) + error = hashsig_finalize_hashes(sig); + + if (!error) + *out = sig; + else + git_hashsig_free(sig); + + return error; +} + +void git_hashsig_free(git_hashsig *sig) +{ + git__free(sig); +} + +static int hashsig_heap_compare(const hashsig_heap *a, const hashsig_heap *b) +{ + int matches = 0, i, j, cmp; + + assert(a->cmp == b->cmp); + + /* hash heaps are sorted - just look for overlap vs total */ + + for (i = 0, j = 0; i < a->size && j < b->size; ) { + cmp = a->cmp(&a->values[i], &b->values[j]); + + if (cmp < 0) + ++i; + else if (cmp > 0) + ++j; + else { + ++i; ++j; ++matches; + } + } + + return HASHSIG_SCALE * (matches * 2) / (a->size + b->size); +} + +int git_hashsig_compare(const git_hashsig *a, const git_hashsig *b) +{ + return (hashsig_heap_compare(&a->mins, &b->mins) + + hashsig_heap_compare(&a->maxs, &b->maxs)) / 2; +} + diff --git a/src/hashsig.h b/src/hashsig.h new file mode 100644 index 000000000..70b47f5f3 --- /dev/null +++ b/src/hashsig.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#ifndef INCLUDE_hashsig_h__ +#define INCLUDE_hashsig_h__ + +#include "buffer.h" + +/** + * Similarity signature of line hashes for a buffer + */ +typedef struct git_hashsig git_hashsig; + +typedef enum { + GIT_HASHSIG_NORMAL = 0, /* use all data */ + GIT_HASHSIG_IGNORE_WHITESPACE = 1, /* ignore whitespace */ + GIT_HASHSIG_SMART_WHITESPACE = 2, /* ignore \r and all space after \n */ +} git_hashsig_option_t; + +/** + * Build a similarity signature for a buffer + * + * If you have passed a whitespace-ignoring buffer, then the whitespace + * will be removed from the buffer while it is being processed, modifying + * the buffer in place. Sorry about that! + * + * This will return an error if the buffer doesn't contain enough data to + * compute a valid signature. + * + * @param out The array of hashed runs representing the file content + * @param buf The contents of the file to hash + * @param generate_pairwise_hashes Should pairwise runs be hashed + */ +extern int git_hashsig_create( + git_hashsig **out, + const git_buf *buf, + git_hashsig_option_t opts); + +/** + * Build a similarity signature from a file + * + * This walks through the file, only loading a maximum of 4K of file data at + * a time. Otherwise, it acts just like `git_hashsig_create`. + * + * This will return an error if the file doesn't contain enough data to + * compute a valid signature. + */ +extern int git_hashsig_create_fromfile( + git_hashsig **out, + const char *path, + git_hashsig_option_t opts); + +/** + * Release memory for a content similarity signature + */ +extern void git_hashsig_free(git_hashsig *sig); + +/** + * Measure similarity between two files + * + * @return <0 for error, [0 to 100] as similarity score + */ +extern int git_hashsig_compare( + const git_hashsig *a, + const git_hashsig *b); + +#endif diff --git a/tests-clar/core/buffer.c b/tests-clar/core/buffer.c index 63753bb67..6cad05c00 100644 --- a/tests-clar/core/buffer.c +++ b/tests-clar/core/buffer.c @@ -1,6 +1,7 @@ #include "clar_libgit2.h" #include "buffer.h" #include "buf_text.h" +#include "hashsig.h" #include "fileops.h" #define TESTSTR "Have you seen that? Have you seeeen that??" @@ -732,89 +733,94 @@ void test_core_buffer__classify_with_utf8(void) cl_assert(git_buf_text_contains_nul(&b)); } +#define SIMILARITY_TEST_DATA_1 \ + "test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" \ + "is this enough?\nthere has to be enough data to fill the hash array!\n" \ + "Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" \ + "Let's make sure we've got plenty to go with here.\n smile \n" + void test_core_buffer__similarity_metric(void) { - git_buf_text_hashsig *a, *b; + git_hashsig *a, *b; git_buf buf = GIT_BUF_INIT; int sim; /* in the first case, we compare data to itself and expect 100% match */ - cl_git_pass(git_buf_sets(&buf, "test data\nright here\ninline\ntada")); - cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); - cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); + cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); - cl_assert_equal_i(100, git_buf_text_hashsig_compare(a, b, 100)); + cl_assert_equal_i(100, git_hashsig_compare(a, b)); - git_buf_text_hashsig_free(a); - git_buf_text_hashsig_free(b); + git_hashsig_free(a); + git_hashsig_free(b); - /* in the second case, half of a is matched and all of b is matched, so - * we'll expect a score of around 66% to be the similarity score - */ + /* if we change just a single byte, how much does that change magnify? */ - cl_git_pass( - git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); - cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); + cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_buf_sets(&buf, + "Test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" + "is this enough?\nthere has to be enough data to fill the hash array!\n" + "Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" + "Let's make sure we've got plenty to go with here.\n smile \n")); + cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); - cl_git_pass(git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh")); - cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + sim = git_hashsig_compare(a, b); - sim = git_buf_text_hashsig_compare(a, b, 100); - cl_assert(sim > 60 && sim < 70); + cl_assert(95 < sim && sim < 100); /* expect >95% similarity */ - git_buf_text_hashsig_free(a); - git_buf_text_hashsig_free(b); + git_hashsig_free(a); + git_hashsig_free(b); - /* in the reversed case, 100% of line hashes match, but no pairwise hashes - * match, so we'll expect about a 50% match for a reversed file - */ + /* let's try comparing data to a superset of itself */ - cl_git_pass( - git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); - cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); - cl_git_pass( - git_buf_sets(&buf, "p\no\nn\nm\nl\nk\nj\ni\nh\ng\nf\ne\nd\nc\nb\na\n")); - cl_git_pass(git_buf_text_hashsig_create(&b, &buf, true)); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); + cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1 + "and if I add some more, it should still be pretty similar, yes?\n")); + cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); - sim = git_buf_text_hashsig_compare(a, b, 100); - cl_assert(sim > 45 && sim < 55); + sim = git_hashsig_compare(a, b); - git_buf_text_hashsig_free(a); - git_buf_text_hashsig_free(b); + cl_assert(70 < sim && sim < 80); /* expect in the 70-80% similarity range */ - /* if we don't use pairwise signatures, then a reversed file should - * match 100% - */ + git_hashsig_free(a); + git_hashsig_free(b); - cl_git_pass( - git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); - cl_git_pass(git_buf_text_hashsig_create(&a, &buf, false)); - cl_git_pass( - git_buf_sets(&buf, "p\no\nn\nm\nl\nk\nj\ni\nh\ng\nf\ne\nd\nc\nb\na\n")); - cl_git_pass(git_buf_text_hashsig_create(&b, &buf, false)); + /* what if we keep about half the original data and add half new */ - sim = git_buf_text_hashsig_compare(a, b, 100); - cl_assert_equal_i(100, sim); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); + cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_buf_sets(&buf, + "test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" + "is this enough?\nthere has to be enough data to fill the hash array!\n" + "okay, that's half the original\nwhat else can we add?\nmore data\n" + "one more line will complete this\nshort\nlines\ndon't\nmatter\n")); + cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); - git_buf_text_hashsig_free(a); - git_buf_text_hashsig_free(b); + sim = git_hashsig_compare(a, b); + + cl_assert(40 < sim && sim < 60); /* expect in the 40-60% similarity range */ + + git_hashsig_free(a); + git_hashsig_free(b); /* lastly, let's check that we can hash file content as well */ - cl_git_pass( - git_buf_sets(&buf, "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n")); - cl_git_pass(git_buf_text_hashsig_create(&a, &buf, true)); + cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); + cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); cl_git_pass(git_futils_mkdir("scratch", NULL, 0755, GIT_MKDIR_PATH)); - cl_git_mkfile("scratch/testdata", - "a\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\n"); - cl_git_pass(git_buf_text_hashsig_create_fromfile(&b, "scratch/testdata", true)); + cl_git_mkfile("scratch/testdata", SIMILARITY_TEST_DATA_1); + cl_git_pass(git_hashsig_create_fromfile( + &b, "scratch/testdata", GIT_HASHSIG_NORMAL)); - cl_assert_equal_i(100, git_buf_text_hashsig_compare(a, b, 100)); + cl_assert_equal_i(100, git_hashsig_compare(a, b)); - git_buf_text_hashsig_free(a); - git_buf_text_hashsig_free(b); + git_hashsig_free(a); + git_hashsig_free(b); git_buf_free(&buf); git_futils_rmdir_r("scratch", NULL, GIT_RMDIR_REMOVE_FILES); From aa6432604e09de40ee20ee1f6631a2f717111863 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 15 Feb 2013 11:08:02 -0800 Subject: [PATCH 05/14] More tests of file signatures with whitespace opts Seems to be working pretty well... --- tests-clar/core/buffer.c | 79 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests-clar/core/buffer.c b/tests-clar/core/buffer.c index 6cad05c00..90733815c 100644 --- a/tests-clar/core/buffer.c +++ b/tests-clar/core/buffer.c @@ -825,3 +825,82 @@ void test_core_buffer__similarity_metric(void) git_buf_free(&buf); git_futils_rmdir_r("scratch", NULL, GIT_RMDIR_REMOVE_FILES); } + + +void test_core_buffer__similarity_metric_whitespace(void) +{ + git_hashsig *a, *b; + git_buf buf = GIT_BUF_INIT; + int sim, i, j; + git_hashsig_option_t opt; + const char *tabbed = + " for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\n" + " separator = sep[s];\n" + " expect = expect_values[s];\n" + "\n" + " for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\n" + " for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\n" + " git_buf_join(&buf, separator, a[i], b[j]);\n" + " cl_assert_equal_s(*expect, buf.ptr);\n" + " expect++;\n" + " }\n" + " }\n" + " }\n"; + const char *spaced = + " for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\n" + " separator = sep[s];\n" + " expect = expect_values[s];\n" + "\n" + " for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\n" + " for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\n" + " git_buf_join(&buf, separator, a[i], b[j]);\n" + " cl_assert_equal_s(*expect, buf.ptr);\n" + " expect++;\n" + " }\n" + " }\n" + " }\n"; + const char *crlf_spaced2 = + " for (s = 0; s < sizeof(sep) / sizeof(char); ++s) {\r\n" + " separator = sep[s];\r\n" + " expect = expect_values[s];\r\n" + "\r\n" + " for (j = 0; j < sizeof(b) / sizeof(char*); ++j) {\r\n" + " for (i = 0; i < sizeof(a) / sizeof(char*); ++i) {\r\n" + " git_buf_join(&buf, separator, a[i], b[j]);\r\n" + " cl_assert_equal_s(*expect, buf.ptr);\r\n" + " expect++;\r\n" + " }\r\n" + " }\r\n" + " }\r\n"; + const char *text[3] = { tabbed, spaced, crlf_spaced2 }; + + /* let's try variations of our own code with whitespace changes */ + + for (opt = GIT_HASHSIG_NORMAL; opt <= GIT_HASHSIG_SMART_WHITESPACE; ++opt) { + for (i = 0; i < 3; ++i) { + for (j = 0; j < 3; ++j) { + cl_git_pass(git_buf_sets(&buf, text[i])); + cl_git_pass(git_hashsig_create(&a, &buf, opt)); + + cl_git_pass(git_buf_sets(&buf, text[j])); + cl_git_pass(git_hashsig_create(&b, &buf, opt)); + + sim = git_hashsig_compare(a, b); + + if (opt == GIT_HASHSIG_NORMAL) { + if (i == j) + cl_assert_equal_i(100, sim); + else + cl_assert(sim < 30); /* expect pretty different */ + } else { + cl_assert_equal_i(100, sim); + } + + git_hashsig_free(a); + git_hashsig_free(b); + } + } + } + + git_buf_free(&buf); +} From a235e9d355c2188eb35efeac8147b2e8b626caa3 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 15 Feb 2013 14:12:43 -0800 Subject: [PATCH 06/14] Pluggable similarity metric API --- include/git2/diff.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/git2/diff.h b/include/git2/diff.h index 3a88902ad..d90fedfbd 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -389,6 +389,16 @@ typedef enum { GIT_DIFF_FIND_AND_BREAK_REWRITES = (1 << 4), } git_diff_find_t; +/** + * Pluggable similarity metric + */ +typedef struct { + int (*calc_signature)(void **out, const git_diff_file *file, void *payload); + void (*free_signature)(void *sig, void *payload); + int (*calc_similarity)(int *score, void *siga, void *sigb, void *payload); + void *payload; +} git_diff_similarity_metric; + /** * Control behavior of rename and copy detection */ @@ -411,6 +421,9 @@ typedef struct { * the `diff.renameLimit` config) (default 200) */ unsigned int target_limit; + + /** Pluggable similarity metric; pass NULL to use internal metric */ + git_diff_similarity_metric *metric; } git_diff_find_options; #define GIT_DIFF_FIND_OPTIONS_VERSION 1 From 9bc8be3d7e5134de1d912c7ef08d6207079bd8c1 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Tue, 19 Feb 2013 10:25:41 -0800 Subject: [PATCH 07/14] Refine pluggable similarity API This plugs in the three basic similarity strategies for handling whitespace via internal use of the pluggable API. In so doing, I realized that the use of git_buf in the hashsig API was not needed and actually just made it harder to use, so I tweaked that API as well. Note that the similarity metric is still not hooked up in the find_similarity code - this is just setting out the function that will be used. --- include/git2/diff.h | 30 +++++++++++++++++-- src/diff_tform.c | 64 ++++++++++++++++++++++++++++++++++++++++ src/hashsig.c | 5 ++-- src/hashsig.h | 6 ++-- tests-clar/core/buffer.c | 22 +++++++------- 5 files changed, 110 insertions(+), 17 deletions(-) diff --git a/include/git2/diff.h b/include/git2/diff.h index d90fedfbd..c0f48368e 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -387,20 +387,46 @@ typedef enum { /** split large rewrites into delete/add pairs (`--break-rewrites=/M`) */ GIT_DIFF_FIND_AND_BREAK_REWRITES = (1 << 4), + + /** measure similarity ignoring leading whitespace (default) */ + GIT_DIFF_FIND_IGNORE_LEADING_WHITESPACE = 0, + /** measure similarity ignoring all whitespace */ + GIT_DIFF_FIND_IGNORE_WHITESPACE = (1 << 6), + /** measure similarity including all data */ + GIT_DIFF_FIND_DONT_IGNORE_WHITESPACE = (1 << 7), } git_diff_find_t; /** * Pluggable similarity metric */ typedef struct { - int (*calc_signature)(void **out, const git_diff_file *file, void *payload); + int (*file_signature)( + void **out, const git_diff_file *file, + const char *fullpath, void *payload); + int (*buffer_signature)( + void **out, const git_diff_file *file, + const char *buf, size_t buflen, void *payload); void (*free_signature)(void *sig, void *payload); - int (*calc_similarity)(int *score, void *siga, void *sigb, void *payload); + int (*similarity)(int *score, void *siga, void *sigb, void *payload); void *payload; } git_diff_similarity_metric; /** * Control behavior of rename and copy detection + * + * These options mostly mimic parameters that can be passed to git-diff. + * + * - `rename_threshold` is the same as the -M option with a value + * - `copy_threshold` is the same as the -C option with a value + * - `rename_from_rewrite_threshold` matches the top of the -B option + * - `break_rewrite_threshold` matches the bottom of the -B option + * - `target_limit` matches the -l option + * + * The `metric` option allows you to plug in a custom similarity metric. + * Set it to NULL for the default internal metric which is based on sampling + * hashes of ranges of data in the file. The default metric is a pretty + * good similarity approximation that should work fairly well for both text + * and binary data, and is pretty fast with fixed memory overhead. */ typedef struct { unsigned int version; diff --git a/src/diff_tform.c b/src/diff_tform.c index 3939230b7..61cf5b392 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -169,6 +169,60 @@ int git_diff_merge( return error; } +#define FIND_SIMILAR_HASHSIG(NAME,OPT) \ + static int find_similar__hashsig_for_file ## NAME( \ + void **out, const git_diff_file *f, const char *path, void *p) { \ + GIT_UNUSED(f); GIT_UNUSED(p); \ + return git_hashsig_create_fromfile((git_hashsig **)out, path, OPT); \ + } \ + static int find_similar__hashsig_for_buf ## NAME( \ + void **out, const git_diff_file *f, const char *buf, size_t len, void *p) { \ + GIT_UNUSED(f); GIT_UNUSED(p); \ + return git_hashsig_create((git_hashsig **)out, buf, len, OPT); \ + } + +FIND_SIMILAR_HASHSIG(_default, GIT_HASHSIG_SMART_WHITESPACE); +FIND_SIMILAR_HASHSIG(_ignore_whitespace, GIT_HASHSIG_IGNORE_WHITESPACE); +FIND_SIMILAR_HASHSIG(_include_whitespace, GIT_HASHSIG_NORMAL); + +static void find_similar__hashsig_free(void *sig, void *payload) +{ + GIT_UNUSED(payload); + git_hashsig_free(sig); +} + +static int find_similar__calc_similarity( + int *score, void *siga, void *sigb, void *payload) +{ + GIT_UNUSED(payload); + *score = git_hashsig_compare(siga, sigb); + return 0; +} + +static git_diff_similarity_metric find_similar__internal_metrics[3] = { + { + find_similar__hashsig_for_file_default, + find_similar__hashsig_for_buf_default, + find_similar__hashsig_free, + find_similar__calc_similarity, + NULL + }, + { + find_similar__hashsig_for_file_ignore_whitespace, + find_similar__hashsig_for_buf_ignore_whitespace, + find_similar__hashsig_free, + find_similar__calc_similarity, + NULL + }, + { + find_similar__hashsig_for_file_include_whitespace, + find_similar__hashsig_for_buf_include_whitespace, + find_similar__hashsig_free, + find_similar__calc_similarity, + NULL + } +}; + #define DEFAULT_THRESHOLD 50 #define DEFAULT_BREAK_REWRITE_THRESHOLD 60 #define DEFAULT_TARGET_LIMIT 200 @@ -237,6 +291,16 @@ static int normalize_find_opts( opts->target_limit = limit; } + /* for now, always assign the same internal metric */ + if (!opts->metric) { + if (opts->flags & GIT_DIFF_FIND_IGNORE_WHITESPACE) + opts->metric = &find_similar__internal_metrics[1]; + else if (opts->flags & GIT_DIFF_FIND_DONT_IGNORE_WHITESPACE) + opts->metric = &find_similar__internal_metrics[2]; + else + opts->metric = &find_similar__internal_metrics[0]; + } + return 0; } diff --git a/src/hashsig.c b/src/hashsig.c index a9cd8fa53..60649fd11 100644 --- a/src/hashsig.c +++ b/src/hashsig.c @@ -266,7 +266,8 @@ static git_hashsig *hashsig_alloc(git_hashsig_option_t opts) int git_hashsig_create( git_hashsig **out, - const git_buf *buf, + const char *buf, + size_t buflen, git_hashsig_option_t opts) { int error; @@ -274,7 +275,7 @@ int git_hashsig_create( git_hashsig *sig = hashsig_alloc(opts); GITERR_CHECK_ALLOC(sig); - error = hashsig_add_hashes(sig, buf->ptr, buf->size, &prog); + error = hashsig_add_hashes(sig, buf, buflen, &prog); if (!error) error = hashsig_finalize_hashes(sig); diff --git a/src/hashsig.h b/src/hashsig.h index 70b47f5f3..8c920cbf1 100644 --- a/src/hashsig.h +++ b/src/hashsig.h @@ -7,7 +7,7 @@ #ifndef INCLUDE_hashsig_h__ #define INCLUDE_hashsig_h__ -#include "buffer.h" +#include "common.h" /** * Similarity signature of line hashes for a buffer @@ -32,11 +32,13 @@ typedef enum { * * @param out The array of hashed runs representing the file content * @param buf The contents of the file to hash + * @param buflen The length of the data at `buf` * @param generate_pairwise_hashes Should pairwise runs be hashed */ extern int git_hashsig_create( git_hashsig **out, - const git_buf *buf, + const char *buf, + size_t buflen, git_hashsig_option_t opts); /** diff --git a/tests-clar/core/buffer.c b/tests-clar/core/buffer.c index 90733815c..cb5f0161b 100644 --- a/tests-clar/core/buffer.c +++ b/tests-clar/core/buffer.c @@ -748,8 +748,8 @@ void test_core_buffer__similarity_metric(void) /* in the first case, we compare data to itself and expect 100% match */ cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); - cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); - cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_assert_equal_i(100, git_hashsig_compare(a, b)); @@ -759,13 +759,13 @@ void test_core_buffer__similarity_metric(void) /* if we change just a single byte, how much does that change magnify? */ cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); - cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, "Test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" "is this enough?\nthere has to be enough data to fill the hash array!\n" "Apparently 191 bytes is the minimum amount of data needed.\nHere goes!\n" "Let's make sure we've got plenty to go with here.\n smile \n")); - cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); @@ -777,10 +777,10 @@ void test_core_buffer__similarity_metric(void) /* let's try comparing data to a superset of itself */ cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); - cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1 "and if I add some more, it should still be pretty similar, yes?\n")); - cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); @@ -792,13 +792,13 @@ void test_core_buffer__similarity_metric(void) /* what if we keep about half the original data and add half new */ cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); - cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_buf_sets(&buf, "test data\nright here\ninline\ntada\nneeds more data\nlots of data\n" "is this enough?\nthere has to be enough data to fill the hash array!\n" "okay, that's half the original\nwhat else can we add?\nmore data\n" "one more line will complete this\nshort\nlines\ndon't\nmatter\n")); - cl_git_pass(git_hashsig_create(&b, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); sim = git_hashsig_compare(a, b); @@ -810,7 +810,7 @@ void test_core_buffer__similarity_metric(void) /* lastly, let's check that we can hash file content as well */ cl_git_pass(git_buf_sets(&buf, SIMILARITY_TEST_DATA_1)); - cl_git_pass(git_hashsig_create(&a, &buf, GIT_HASHSIG_NORMAL)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, GIT_HASHSIG_NORMAL)); cl_git_pass(git_futils_mkdir("scratch", NULL, 0755, GIT_MKDIR_PATH)); cl_git_mkfile("scratch/testdata", SIMILARITY_TEST_DATA_1); @@ -880,10 +880,10 @@ void test_core_buffer__similarity_metric_whitespace(void) for (i = 0; i < 3; ++i) { for (j = 0; j < 3; ++j) { cl_git_pass(git_buf_sets(&buf, text[i])); - cl_git_pass(git_hashsig_create(&a, &buf, opt)); + cl_git_pass(git_hashsig_create(&a, buf.ptr, buf.size, opt)); cl_git_pass(git_buf_sets(&buf, text[j])); - cl_git_pass(git_hashsig_create(&b, &buf, opt)); + cl_git_pass(git_hashsig_create(&b, buf.ptr, buf.size, opt)); sim = git_hashsig_compare(a, b); From 71a3d27ea686845811f04314d02798b4f1745046 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 8 Feb 2013 10:06:47 -0800 Subject: [PATCH 08/14] Replace diff delta binary with flags Previously the git_diff_delta recorded if the delta was binary. This replaces that (with no net change in structure size) with a full set of flags. The flag values that were already in use for individual git_diff_file objects are reused for the delta flags, too (along with renaming those flags to make it clear that they are used more generally). This (a) makes things somewhat more consistent (because I was using a -1 value in the "boolean" binary field to indicate unset, whereas now I can just use the flags that are easier to understand), and (b) will make it easier for me to add some additional flags to the delta object in the future, such as marking the results of a copy/rename detection or other deltas that might want a special indicator. While making this change, I officially moved some of the flags that were internal only into the private diff header. This also allowed me to remove a gross hack in rename/copy detect code where I was overwriting the status field with an internal value. --- include/git2/diff.h | 54 ++++++------ src/checkout.c | 2 +- src/diff.c | 8 +- src/diff.h | 10 ++- src/diff_output.c | 147 ++++++++++++++++----------------- src/diff_tform.c | 16 ++-- tests-clar/diff/diff_helpers.c | 5 +- tests-clar/diff/diffiter.c | 8 +- 8 files changed, 128 insertions(+), 122 deletions(-) diff --git a/include/git2/diff.h b/include/git2/diff.h index c0f48368e..7f165041a 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -88,10 +88,9 @@ typedef enum { GIT_DIFF_INCLUDE_UNTRACKED = (1 << 8), /** Include unmodified files in the diff list */ GIT_DIFF_INCLUDE_UNMODIFIED = (1 << 9), - /** Even with the GIT_DIFF_INCLUDE_UNTRACKED flag, when an untracked - * directory is found, only a single entry for the directory is added - * to the diff list; with this flag, all files under the directory will - * be included, too. + /** Even with GIT_DIFF_INCLUDE_UNTRACKED, an entire untracked directory + * will be marked with only a single entry in the diff list; this flag + * adds all files under the directory as UNTRACKED entries, too. */ GIT_DIFF_RECURSE_UNTRACKED_DIRS = (1 << 10), /** If the pathspec is set in the diff options, this flags means to @@ -120,6 +119,11 @@ typedef enum { GIT_DIFF_INCLUDE_TYPECHANGE_TREES = (1 << 16), /** Ignore file mode changes */ GIT_DIFF_IGNORE_FILEMODE = (1 << 17), + /** Even with GIT_DIFF_INCLUDE_IGNORED, an entire ignored directory + * will be marked with only a single entry in the diff list; this flag + * adds all files under the directory as IGNORED entries, too. + */ + GIT_DIFF_RECURSE_IGNORED_DIRS = (1 << 10), } git_diff_option_t; /** @@ -133,20 +137,18 @@ typedef enum { typedef struct git_diff_list git_diff_list; /** - * Flags for the file object on each side of a diff. + * Flags for the delta object and the file objects on each side. * - * Note: most of these flags are just for **internal** consumption by - * libgit2, but some of them may be interesting to external users. + * These flags are used for both the `flags` value of the `git_diff_delta` + * and the flags for the `git_diff_file` objects representing the old and + * new sides of the delta. Values outside of this public range should be + * considered reserved for internal or future use. */ typedef enum { - GIT_DIFF_FILE_VALID_OID = (1 << 0), /** `oid` value is known correct */ - GIT_DIFF_FILE_FREE_PATH = (1 << 1), /** `path` is allocated memory */ - GIT_DIFF_FILE_BINARY = (1 << 2), /** should be considered binary data */ - GIT_DIFF_FILE_NOT_BINARY = (1 << 3), /** should be considered text data */ - GIT_DIFF_FILE_FREE_DATA = (1 << 4), /** internal file data is allocated */ - GIT_DIFF_FILE_UNMAP_DATA = (1 << 5), /** internal file data is mmap'ed */ - GIT_DIFF_FILE_NO_DATA = (1 << 6), /** file data should not be loaded */ -} git_diff_file_flag_t; + GIT_DIFF_FLAG_BINARY = (1 << 0), /** file(s) treated as binary data */ + GIT_DIFF_FLAG_NOT_BINARY = (1 << 1), /** file(s) treated as text data */ + GIT_DIFF_FLAG_VALID_OID = (1 << 2), /** `oid` value is known correct */ +} git_diff_flag_t; /** * What type of change is described by a git_diff_delta? @@ -186,18 +188,17 @@ typedef enum { * * `size` is the size of the entry in bytes. * - * `flags` is a combination of the `git_diff_file_flag_t` types, but those - * are largely internal values. + * `flags` is a combination of the `git_diff_flag_t` types * * `mode` is, roughly, the stat() `st_mode` value for the item. This will * be restricted to one of the `git_filemode_t` values. */ typedef struct { - git_oid oid; + git_oid oid; const char *path; - git_off_t size; - unsigned int flags; - uint16_t mode; + git_off_t size; + uint32_t flags; + uint16_t mode; } git_diff_file; /** @@ -219,16 +220,17 @@ typedef struct { * * Under some circumstances, in the name of efficiency, not all fields will * be filled in, but we generally try to fill in as much as possible. One - * example is that the "binary" field will not examine file contents if you - * do not pass in hunk and/or line callbacks to the diff foreach iteration - * function. It will just use the git attributes for those files. + * example is that the "flags" field may not have either the `BINARY` or the + * `NOT_BINARY` flag set to avoid examining file contents if you do not pass + * in hunk and/or line callbacks to the diff foreach iteration function. It + * will just use the git attributes for those files. */ typedef struct { git_diff_file old_file; git_diff_file new_file; git_delta_t status; - unsigned int similarity; /**< for RENAMED and COPIED, value 0-100 */ - int binary; + uint32_t similarity; /**< for RENAMED and COPIED, value 0-100 */ + uint32_t flags; } git_diff_delta; /** diff --git a/src/checkout.c b/src/checkout.c index 59cd218a9..52a4a4e80 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -78,7 +78,7 @@ static int checkout_notify( git_oid_cpy(&wdfile.oid, &wditem->oid); wdfile.path = wditem->path; wdfile.size = wditem->file_size; - wdfile.flags = GIT_DIFF_FILE_VALID_OID; + wdfile.flags = GIT_DIFF_FLAG_VALID_OID; wdfile.mode = wditem->mode; workdir = &wdfile; diff --git a/src/diff.c b/src/diff.c index d9bc32a37..0861b13eb 100644 --- a/src/diff.c +++ b/src/diff.c @@ -92,11 +92,11 @@ static int diff_delta__from_one( git_oid_cpy(&delta->new_file.oid, &entry->oid); } - delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID; + delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; if (delta->status == GIT_DELTA_DELETED || !git_oid_iszero(&delta->new_file.oid)) - delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID; + delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; notify_res = diff_notify(diff, delta, matched_pathspec); @@ -142,7 +142,7 @@ static int diff_delta__from_two( git_oid_cpy(&delta->old_file.oid, &old_entry->oid); delta->old_file.size = old_entry->file_size; delta->old_file.mode = old_mode; - delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID; + delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; git_oid_cpy(&delta->new_file.oid, &new_entry->oid); delta->new_file.size = new_entry->file_size; @@ -156,7 +156,7 @@ static int diff_delta__from_two( } if (new_oid || !git_oid_iszero(&new_entry->oid)) - delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID; + delta->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; notify_res = diff_notify(diff, delta, matched_pathspec); diff --git a/src/diff.h b/src/diff.h index 16fbf71e6..8e3cbcd46 100644 --- a/src/diff.h +++ b/src/diff.h @@ -28,8 +28,14 @@ enum { GIT_DIFFCAPS_USE_DEV = (1 << 4), /* use st_dev? */ }; -#define GIT_DELTA__TO_DELETE 10 -#define GIT_DELTA__TO_SPLIT 11 +enum { + GIT_DIFF_FLAG__FREE_PATH = (1 << 7), /* `path` is allocated memory */ + GIT_DIFF_FLAG__FREE_DATA = (1 << 8), /* internal file data is allocated */ + GIT_DIFF_FLAG__UNMAP_DATA = (1 << 9), /* internal file data is mmap'ed */ + GIT_DIFF_FLAG__NO_DATA = (1 << 10), /* file data should not be loaded */ + GIT_DIFF_FLAG__TO_DELETE = (1 << 11), /* delete entry during rename det. */ + GIT_DIFF_FLAG__TO_SPLIT = (1 << 12), /* split entry during rename det. */ +}; struct git_diff_list { git_refcount rc; diff --git a/src/diff_output.c b/src/diff_output.c index 88ccc9d45..5e9d5fe76 100644 --- a/src/diff_output.c +++ b/src/diff_output.c @@ -52,8 +52,8 @@ static int parse_hunk_header(git_diff_range *range, const char *header) return 0; } -#define KNOWN_BINARY_FLAGS (GIT_DIFF_FILE_BINARY|GIT_DIFF_FILE_NOT_BINARY) -#define NOT_BINARY_FLAGS (GIT_DIFF_FILE_NOT_BINARY|GIT_DIFF_FILE_NO_DATA) +#define KNOWN_BINARY_FLAGS (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY) +#define NOT_BINARY_FLAGS (GIT_DIFF_FLAG_NOT_BINARY|GIT_DIFF_FLAG__NO_DATA) static int update_file_is_binary_by_attr( git_repository *repo, git_diff_file *file) @@ -68,9 +68,9 @@ static int update_file_is_binary_by_attr( return -1; if (GIT_ATTR_FALSE(value)) - file->flags |= GIT_DIFF_FILE_BINARY; + file->flags |= GIT_DIFF_FLAG_BINARY; else if (GIT_ATTR_TRUE(value)) - file->flags |= GIT_DIFF_FILE_NOT_BINARY; + file->flags |= GIT_DIFF_FLAG_NOT_BINARY; /* otherwise leave file->flags alone */ return 0; @@ -78,15 +78,15 @@ static int update_file_is_binary_by_attr( static void update_delta_is_binary(git_diff_delta *delta) { - if ((delta->old_file.flags & GIT_DIFF_FILE_BINARY) != 0 || - (delta->new_file.flags & GIT_DIFF_FILE_BINARY) != 0) - delta->binary = 1; + if ((delta->old_file.flags & GIT_DIFF_FLAG_BINARY) != 0 || + (delta->new_file.flags & GIT_DIFF_FLAG_BINARY) != 0) + delta->flags |= GIT_DIFF_FLAG_BINARY; else if ((delta->old_file.flags & NOT_BINARY_FLAGS) != 0 && (delta->new_file.flags & NOT_BINARY_FLAGS) != 0) - delta->binary = 0; + delta->flags |= GIT_DIFF_FLAG_NOT_BINARY; - /* otherwise leave delta->binary value untouched */ + /* otherwise leave delta->flags binary value untouched */ } /* returns if we forced binary setting (and no further checks needed) */ @@ -95,24 +95,24 @@ static bool diff_delta_is_binary_forced( git_diff_delta *delta) { /* return true if binary-ness has already been settled */ - if (delta->binary != -1) + if ((delta->flags & KNOWN_BINARY_FLAGS) != 0) return true; /* make sure files are conceivably mmap-able */ if ((git_off_t)((size_t)delta->old_file.size) != delta->old_file.size || (git_off_t)((size_t)delta->new_file.size) != delta->new_file.size) { - delta->old_file.flags |= GIT_DIFF_FILE_BINARY; - delta->new_file.flags |= GIT_DIFF_FILE_BINARY; - delta->binary = 1; + delta->old_file.flags |= GIT_DIFF_FLAG_BINARY; + delta->new_file.flags |= GIT_DIFF_FLAG_BINARY; + delta->flags |= GIT_DIFF_FLAG_BINARY; return true; } /* check if user is forcing us to text diff these files */ if (ctxt->opts && (ctxt->opts->flags & GIT_DIFF_FORCE_TEXT) != 0) { - delta->old_file.flags |= GIT_DIFF_FILE_NOT_BINARY; - delta->new_file.flags |= GIT_DIFF_FILE_NOT_BINARY; - delta->binary = 0; + delta->old_file.flags |= GIT_DIFF_FLAG_NOT_BINARY; + delta->new_file.flags |= GIT_DIFF_FLAG_NOT_BINARY; + delta->flags |= GIT_DIFF_FLAG_NOT_BINARY; return true; } @@ -125,8 +125,6 @@ static int diff_delta_is_binary_by_attr( int error = 0, mirror_new; git_diff_delta *delta = patch->delta; - delta->binary = -1; - if (diff_delta_is_binary_forced(ctxt, delta)) return 0; @@ -152,23 +150,21 @@ static int diff_delta_is_binary_by_content( git_diff_file *file, const git_map *map) { + const git_buf search = { map->data, 0, min(map->len, 4000) }; + if (diff_delta_is_binary_forced(ctxt, delta)) return 0; - if ((file->flags & KNOWN_BINARY_FLAGS) == 0) { - const git_buf search = { map->data, 0, min(map->len, 4000) }; + /* TODO: provide encoding / binary detection callbacks that can + * be UTF-8 aware, etc. For now, instead of trying to be smart, + * let's just use the simple NUL-byte detection that core git uses. + */ - /* TODO: provide encoding / binary detection callbacks that can - * be UTF-8 aware, etc. For now, instead of trying to be smart, - * let's just use the simple NUL-byte detection that core git uses. - */ - - /* previously was: if (git_buf_text_is_binary(&search)) */ - if (git_buf_text_contains_nul(&search)) - file->flags |= GIT_DIFF_FILE_BINARY; - else - file->flags |= GIT_DIFF_FILE_NOT_BINARY; - } + /* previously was: if (git_buf_text_is_binary(&search)) */ + if (git_buf_text_contains_nul(&search)) + file->flags |= GIT_DIFF_FLAG_BINARY; + else + file->flags |= GIT_DIFF_FLAG_NOT_BINARY; update_delta_is_binary(delta); @@ -192,7 +188,7 @@ static int diff_delta_is_binary_by_size( } if (file->size > threshold) - file->flags |= GIT_DIFF_FILE_BINARY; + file->flags |= GIT_DIFF_FLAG_BINARY; update_delta_is_binary(delta); @@ -247,7 +243,7 @@ static int get_blob_content( map->data = git_buf_detach(&content); map->len = strlen(map->data); - file->flags |= GIT_DIFF_FILE_FREE_DATA; + file->flags |= GIT_DIFF_FLAG__FREE_DATA; return 0; } @@ -270,7 +266,7 @@ static int get_blob_content( /* if blob is too large to diff, mark as binary */ if ((error = diff_delta_is_binary_by_size(ctxt, delta, file)) < 0) return error; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) return 0; if (odb_obj != NULL) { @@ -306,14 +302,14 @@ static int get_workdir_sm_content( return error; /* update OID if we didn't have it previously */ - if ((file->flags & GIT_DIFF_FILE_VALID_OID) == 0) { + if ((file->flags & GIT_DIFF_FLAG_VALID_OID) == 0) { const git_oid* sm_head; if ((sm_head = git_submodule_wd_id(sm)) != NULL || (sm_head = git_submodule_head_id(sm)) != NULL) { git_oid_cpy(&file->oid, sm_head); - file->flags |= GIT_DIFF_FILE_VALID_OID; + file->flags |= GIT_DIFF_FLAG_VALID_OID; } } @@ -329,7 +325,7 @@ static int get_workdir_sm_content( map->data = git_buf_detach(&content); map->len = strlen(map->data); - file->flags |= GIT_DIFF_FILE_FREE_DATA; + file->flags |= GIT_DIFF_FLAG__FREE_DATA; return 0; } @@ -356,8 +352,8 @@ static int get_workdir_content( if (S_ISLNK(file->mode)) { ssize_t alloc_len, read_len; - file->flags |= GIT_DIFF_FILE_FREE_DATA; - file->flags |= GIT_DIFF_FILE_BINARY; + file->flags |= GIT_DIFF_FLAG__FREE_DATA; + file->flags |= GIT_DIFF_FLAG_BINARY; /* link path on disk could be UTF-16, so prepare a buffer that is * big enough to handle some UTF-8 data expansion @@ -389,7 +385,7 @@ static int get_workdir_content( file->size = git_futils_filesize(fd); if ((error = diff_delta_is_binary_by_size(ctxt, delta, file)) < 0 || - delta->binary == 1) + (delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto close_and_cleanup; error = git_filters_load( @@ -402,7 +398,7 @@ static int get_workdir_content( goto close_and_cleanup; error = git_futils_mmap_ro(map, fd, 0, (size_t)file->size); - file->flags |= GIT_DIFF_FILE_UNMAP_DATA; + file->flags |= GIT_DIFF_FLAG__UNMAP_DATA; } else { git_buf raw = GIT_BUF_INIT, filtered = GIT_BUF_INIT; @@ -412,7 +408,7 @@ static int get_workdir_content( map->len = git_buf_len(&filtered); map->data = git_buf_detach(&filtered); - file->flags |= GIT_DIFF_FILE_FREE_DATA; + file->flags |= GIT_DIFF_FLAG__FREE_DATA; } git_buf_free(&raw); @@ -425,11 +421,11 @@ close_and_cleanup: } /* once data is loaded, update OID if we didn't have it previously */ - if (!error && (file->flags & GIT_DIFF_FILE_VALID_OID) == 0) { + if (!error && (file->flags & GIT_DIFF_FLAG_VALID_OID) == 0) { error = git_odb_hash( &file->oid, map->data, map->len, GIT_OBJ_BLOB); if (!error) - file->flags |= GIT_DIFF_FILE_VALID_OID; + file->flags |= GIT_DIFF_FLAG_VALID_OID; } if (!error) @@ -445,17 +441,17 @@ static void release_content(git_diff_file *file, git_map *map, git_blob *blob) if (blob != NULL) git_blob_free(blob); - if (file->flags & GIT_DIFF_FILE_FREE_DATA) { + if (file->flags & GIT_DIFF_FLAG__FREE_DATA) { git__free(map->data); map->data = ""; map->len = 0; - file->flags &= ~GIT_DIFF_FILE_FREE_DATA; + file->flags &= ~GIT_DIFF_FLAG__FREE_DATA; } - else if (file->flags & GIT_DIFF_FILE_UNMAP_DATA) { + else if (file->flags & GIT_DIFF_FLAG__UNMAP_DATA) { git_futils_mmap_free(map); map->data = ""; map->len = 0; - file->flags &= ~GIT_DIFF_FILE_UNMAP_DATA; + file->flags &= ~GIT_DIFF_FLAG__UNMAP_DATA; } } @@ -555,7 +551,7 @@ static int diff_patch_load( patch->new_data.len = 0; patch->new_blob = NULL; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto cleanup; if (!ctxt->hunk_cb && @@ -565,25 +561,25 @@ static int diff_patch_load( switch (delta->status) { case GIT_DELTA_ADDED: - delta->old_file.flags |= GIT_DIFF_FILE_NO_DATA; + delta->old_file.flags |= GIT_DIFF_FLAG__NO_DATA; break; case GIT_DELTA_DELETED: - delta->new_file.flags |= GIT_DIFF_FILE_NO_DATA; + delta->new_file.flags |= GIT_DIFF_FLAG__NO_DATA; break; case GIT_DELTA_MODIFIED: break; case GIT_DELTA_UNTRACKED: - delta->old_file.flags |= GIT_DIFF_FILE_NO_DATA; + delta->old_file.flags |= GIT_DIFF_FLAG__NO_DATA; if ((ctxt->opts->flags & GIT_DIFF_INCLUDE_UNTRACKED_CONTENT) == 0) - delta->new_file.flags |= GIT_DIFF_FILE_NO_DATA; + delta->new_file.flags |= GIT_DIFF_FLAG__NO_DATA; break; default: - delta->new_file.flags |= GIT_DIFF_FILE_NO_DATA; - delta->old_file.flags |= GIT_DIFF_FILE_NO_DATA; + delta->new_file.flags |= GIT_DIFF_FLAG__NO_DATA; + delta->old_file.flags |= GIT_DIFF_FLAG__NO_DATA; break; } -#define CHECK_UNMODIFIED (GIT_DIFF_FILE_NO_DATA | GIT_DIFF_FILE_VALID_OID) +#define CHECK_UNMODIFIED (GIT_DIFF_FLAG__NO_DATA | GIT_DIFF_FLAG_VALID_OID) check_if_unmodified = (delta->old_file.flags & CHECK_UNMODIFIED) == 0 && @@ -594,41 +590,41 @@ static int diff_patch_load( * memory footprint during diff. */ - if ((delta->old_file.flags & GIT_DIFF_FILE_NO_DATA) == 0 && + if ((delta->old_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0 && patch->old_src == GIT_ITERATOR_TYPE_WORKDIR) { if ((error = get_workdir_content( ctxt, delta, &delta->old_file, &patch->old_data)) < 0) goto cleanup; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto cleanup; } - if ((delta->new_file.flags & GIT_DIFF_FILE_NO_DATA) == 0 && + if ((delta->new_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0 && patch->new_src == GIT_ITERATOR_TYPE_WORKDIR) { if ((error = get_workdir_content( ctxt, delta, &delta->new_file, &patch->new_data)) < 0) goto cleanup; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto cleanup; } - if ((delta->old_file.flags & GIT_DIFF_FILE_NO_DATA) == 0 && + if ((delta->old_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0 && patch->old_src != GIT_ITERATOR_TYPE_WORKDIR) { if ((error = get_blob_content( ctxt, delta, &delta->old_file, &patch->old_data, &patch->old_blob)) < 0) goto cleanup; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto cleanup; } - if ((delta->new_file.flags & GIT_DIFF_FILE_NO_DATA) == 0 && + if ((delta->new_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0 && patch->new_src != GIT_ITERATOR_TYPE_WORKDIR) { if ((error = get_blob_content( ctxt, delta, &delta->new_file, &patch->new_data, &patch->new_blob)) < 0) goto cleanup; - if (delta->binary == 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) goto cleanup; } @@ -646,13 +642,13 @@ static int diff_patch_load( } cleanup: - if (delta->binary == -1) + if ((delta->flags & KNOWN_BINARY_FLAGS) == 0) update_delta_is_binary(delta); if (!error) { patch->flags |= GIT_DIFF_PATCH_LOADED; - if (delta->binary != 1 && + if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0 && delta->status != GIT_DELTA_UNMODIFIED && (patch->old_data.len || patch->new_data.len) && !git_oid_equal(&delta->old_file.oid, &delta->new_file.oid)) @@ -1138,7 +1134,7 @@ static int print_patch_file( newpath = "/dev/null"; } - if (delta->binary != 1) { + if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0) { git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath); git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath); } @@ -1153,7 +1149,7 @@ static int print_patch_file( return GIT_EUSER; } - if (delta->binary != 1) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0) return 0; git_buf_clear(pi->buf); @@ -1268,7 +1264,7 @@ static void set_data_from_blob( map->data = (char *)git_blob_rawcontent(blob); } else { file->size = 0; - file->flags |= GIT_DIFF_FILE_NO_DATA; + file->flags |= GIT_DIFF_FLAG__NO_DATA; map->len = 0; map->data = ""; @@ -1283,7 +1279,7 @@ static void set_data_from_buffer( map->len = buffer_len; if (!buffer) { - file->flags |= GIT_DIFF_FILE_NO_DATA; + file->flags |= GIT_DIFF_FLAG__NO_DATA; map->data = NULL; } else { map->data = (char *)buffer; @@ -1322,13 +1318,13 @@ static int diff_single_apply(diff_single_data *data) { int error; git_diff_delta *delta = &data->delta; - bool has_old = ((delta->old_file.flags & GIT_DIFF_FILE_NO_DATA) == 0); - bool has_new = ((delta->new_file.flags & GIT_DIFF_FILE_NO_DATA) == 0); + bool has_old = ((delta->old_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0); + bool has_new = ((delta->new_file.flags & GIT_DIFF_FLAG__NO_DATA) == 0); /* finish setting up fake git_diff_delta record and loaded data */ data->patch.delta = delta; - delta->binary = -1; + delta->flags = delta->flags & ~KNOWN_BINARY_FLAGS; delta->status = has_new ? (has_old ? GIT_DELTA_MODIFIED : GIT_DELTA_ADDED) : @@ -1345,7 +1341,8 @@ static int diff_single_apply(diff_single_data *data) data->patch.flags |= GIT_DIFF_PATCH_LOADED; - if (delta->binary != 1 && delta->status != GIT_DELTA_UNMODIFIED) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0 && + delta->status != GIT_DELTA_UNMODIFIED) data->patch.flags |= GIT_DIFF_PATCH_DIFFABLE; /* do diffs */ @@ -1469,7 +1466,7 @@ int git_diff_get_patch( *delta_ptr = delta; if (!patch_ptr && - (delta->binary != -1 || + ((delta->flags & KNOWN_BINARY_FLAGS) != 0 || (diff->opts.flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0)) return 0; diff --git a/src/diff_tform.c b/src/diff_tform.c index 61cf5b392..e051732c5 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -315,10 +315,10 @@ static int apply_splits_and_deletes(git_diff_list *diff, size_t expected_size) /* build new delta list without TO_DELETE and splitting TO_SPLIT */ git_vector_foreach(&diff->deltas, i, delta) { - if (delta->status == GIT_DELTA__TO_DELETE) + if ((delta->flags & GIT_DIFF_FLAG__TO_DELETE) != 0) continue; - if (delta->status == GIT_DELTA__TO_SPLIT) { + if ((delta->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0) { git_diff_delta *deleted = diff_delta__dup(delta, &diff->pool); if (!deleted) goto on_error; @@ -326,7 +326,7 @@ static int apply_splits_and_deletes(git_diff_list *diff, size_t expected_size) deleted->status = GIT_DELTA_DELETED; memset(&deleted->new_file, 0, sizeof(deleted->new_file)); deleted->new_file.path = deleted->old_file.path; - deleted->new_file.flags |= GIT_DIFF_FILE_VALID_OID; + deleted->new_file.flags |= GIT_DIFF_FLAG_VALID_OID; if (git_vector_insert(&onto, deleted) < 0) goto on_error; @@ -334,7 +334,7 @@ static int apply_splits_and_deletes(git_diff_list *diff, size_t expected_size) delta->status = GIT_DELTA_ADDED; memset(&delta->old_file, 0, sizeof(delta->old_file)); delta->old_file.path = delta->new_file.path; - delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID; + delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; } if (git_vector_insert(&onto, delta) < 0) @@ -343,7 +343,7 @@ static int apply_splits_and_deletes(git_diff_list *diff, size_t expected_size) /* cannot return an error past this point */ git_vector_foreach(&diff->deltas, i, delta) - if (delta->status == GIT_DELTA__TO_DELETE) + if ((delta->flags & GIT_DIFF_FLAG__TO_DELETE) != 0) git__free(delta); /* swap new delta list into place */ @@ -411,7 +411,7 @@ int git_diff_find_similar( /* calc_similarity(NULL, &from->old_file, from->new_file); */ if (similarity < opts.break_rewrite_threshold) { - from->status = GIT_DELTA__TO_SPLIT; + from->flags |= GIT_DIFF_FLAG__TO_SPLIT; num_changes++; } } @@ -502,7 +502,7 @@ int git_diff_find_similar( to->status = GIT_DELTA_RENAMED; memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); - from->status = GIT_DELTA__TO_DELETE; + from->flags |= GIT_DIFF_FLAG__TO_DELETE; num_changes++; continue; @@ -522,7 +522,7 @@ int git_diff_find_similar( from->status = GIT_DELTA_ADDED; memset(&from->old_file, 0, sizeof(from->old_file)); from->old_file.path = to->old_file.path; - from->old_file.flags |= GIT_DIFF_FILE_VALID_OID; + from->old_file.flags |= GIT_DIFF_FLAG_VALID_OID; continue; } diff --git a/tests-clar/diff/diff_helpers.c b/tests-clar/diff/diff_helpers.c index 1436ada03..a1f75ce39 100644 --- a/tests-clar/diff/diff_helpers.c +++ b/tests-clar/diff/diff_helpers.c @@ -32,7 +32,7 @@ int diff_file_cb( e->files++; - if (delta->binary) + if ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0) e->files_binary++; cl_assert(delta->status <= GIT_DELTA_TYPECHANGE); @@ -126,7 +126,8 @@ int diff_foreach_via_iterator( /* if there are no changes, then the patch will be NULL */ if (!patch) { - cl_assert(delta->status == GIT_DELTA_UNMODIFIED || delta->binary == 1); + cl_assert(delta->status == GIT_DELTA_UNMODIFIED || + (delta->flags & GIT_DIFF_FLAG_BINARY) != 0); continue; } diff --git a/tests-clar/diff/diffiter.c b/tests-clar/diff/diffiter.c index 8d550ec0f..932d720f2 100644 --- a/tests-clar/diff/diffiter.c +++ b/tests-clar/diff/diffiter.c @@ -152,8 +152,8 @@ void test_diff_diffiter__max_size_threshold(void) file_count++; hunk_count += (int)git_diff_patch_num_hunks(patch); - assert(delta->binary == 0 || delta->binary == 1); - binary_count += delta->binary; + assert((delta->flags & (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY)) != 0); + binary_count += ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0); git_diff_patch_free(patch); } @@ -185,8 +185,8 @@ void test_diff_diffiter__max_size_threshold(void) file_count++; hunk_count += (int)git_diff_patch_num_hunks(patch); - assert(delta->binary == 0 || delta->binary == 1); - binary_count += delta->binary; + assert((delta->flags & (GIT_DIFF_FLAG_BINARY|GIT_DIFF_FLAG_NOT_BINARY)) != 0); + binary_count += ((delta->flags & GIT_DIFF_FLAG_BINARY) != 0); git_diff_patch_free(patch); } From 960a04dd56d89e94b5092be19ba9704b2d292dba Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 21 Feb 2013 12:40:33 -0800 Subject: [PATCH 09/14] Initial integration of similarity metric to diff This is the initial integration of the similarity metric into the `git_diff_find_similar()` code path. The existing tests all pass, but the new functionality isn't currently well tested. The integration does go through the pluggable metric interface, so it should be possible to drop in an alternative to the internal metric that libgit2 implements. This comes along with a behavior change for an existing interface; namely, passing two NULLs to git_diff_blobs (or passing NULLs to git_diff_blob_to_buffer) will now call the file_cb parameter zero times instead of one time. I know it's strange that that change is paired with this other change, but it emerged from some initialization changes that I ended up making. --- include/git2/diff.h | 16 ++- src/diff_output.c | 56 +++++++--- src/diff_tform.c | 239 +++++++++++++++++++++++++++++++---------- tests-clar/diff/blob.c | 4 +- 4 files changed, 237 insertions(+), 78 deletions(-) diff --git a/include/git2/diff.h b/include/git2/diff.h index 7f165041a..379f4694e 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -379,7 +379,7 @@ typedef struct git_diff_patch git_diff_patch; typedef enum { /** look for renames? (`--find-renames`) */ GIT_DIFF_FIND_RENAMES = (1 << 0), - /** consider old size of modified for renames? (`--break-rewrites=N`) */ + /** consider old side of modified for renames? (`--break-rewrites=N`) */ GIT_DIFF_FIND_RENAMES_FROM_REWRITES = (1 << 1), /** look for copies? (a la `--find-copies`) */ @@ -897,11 +897,12 @@ GIT_EXTERN(int) git_diff_patch_to_str( * * NULL is allowed for either `old_blob` or `new_blob` and will be treated * as an empty blob, with the `oid` set to NULL in the `git_diff_file` data. + * Passing NULL for both blobs is a noop; no callbacks will be made at all. * - * We do run a binary content check on the two blobs and if either of the - * blobs looks like binary data, the `git_diff_delta` binary attribute will - * be set to 1 and no call to the hunk_cb nor line_cb will be made (unless - * you pass `GIT_DIFF_FORCE_TEXT` of course). + * We do run a binary content check on the blob content and if either blob + * looks like binary data, the `git_diff_delta` binary attribute will be set + * to 1 and no call to the hunk_cb nor line_cb will be made (unless you pass + * `GIT_DIFF_FORCE_TEXT` of course). * * @return 0 on success, GIT_EUSER on non-zero callback, or error code */ @@ -921,6 +922,11 @@ GIT_EXTERN(int) git_diff_blobs( * so the `git_diff_file` parameters to the callbacks will be faked a la the * rules for `git_diff_blobs()`. * + * Passing NULL for `old_blob` will be treated as an empty blob (i.e. the + * `file_cb` will be invoked with GIT_DELTA_ADDED and the diff will be the + * entire content of the buffer added). Passing NULL to the buffer will do + * the reverse, with GIT_DELTA_REMOVED and blob content removed. + * * @return 0 on success, GIT_EUSER on non-zero callback, or error code */ GIT_EXTERN(int) git_diff_blob_to_buffer( diff --git a/src/diff_output.c b/src/diff_output.c index 5e9d5fe76..13434beb9 100644 --- a/src/diff_output.c +++ b/src/diff_output.c @@ -456,7 +456,7 @@ static void release_content(git_diff_file *file, git_map *map, git_blob *blob) } -static void diff_context_init( +static int diff_context_init( diff_context *ctxt, git_diff_list *diff, git_repository *repo, @@ -468,6 +468,12 @@ static void diff_context_init( { memset(ctxt, 0, sizeof(diff_context)); + if (!repo && diff) + repo = diff->repo; + + if (!opts && diff) + opts = &diff->opts; + ctxt->repo = repo; ctxt->diff = diff; ctxt->opts = opts; @@ -478,6 +484,8 @@ static void diff_context_init( ctxt->error = 0; setup_xdiff_options(ctxt->opts, &ctxt->xdiff_config, &ctxt->xdiff_params); + + return 0; } static int diff_delta_file_callback( @@ -922,6 +930,15 @@ static int diff_patch_line_cb( return 0; } +static int diff_required(git_diff_list *diff, const char *action) +{ + if (!diff) { + giterr_set(GITERR_INVALID, "Must provide valid diff to %s", action); + return -1; + } + + return 0; +} int git_diff_foreach( git_diff_list *diff, @@ -935,9 +952,12 @@ int git_diff_foreach( size_t idx; git_diff_patch patch; - diff_context_init( - &ctxt, diff, diff->repo, &diff->opts, - file_cb, hunk_cb, data_cb, payload); + if (diff_required(diff, "git_diff_foreach") < 0) + return -1; + + if (diff_context_init( + &ctxt, diff, NULL, NULL, file_cb, hunk_cb, data_cb, payload) < 0) + return -1; diff_patch_init(&ctxt, &patch); @@ -1306,8 +1326,10 @@ static int diff_single_init( memset(data, 0, sizeof(*data)); - diff_context_init( - &data->ctxt, NULL, repo, opts, file_cb, hunk_cb, data_cb, payload); + if (diff_context_init( + &data->ctxt, NULL, repo, opts, + file_cb, hunk_cb, data_cb, payload) < 0) + return -1; diff_patch_init(&data->ctxt, &data->patch); @@ -1374,6 +1396,9 @@ int git_diff_blobs( new_blob ? git_object_owner((const git_object *)new_blob) : old_blob ? git_object_owner((const git_object *)old_blob) : NULL; + if (!repo) /* Hmm, given two NULL blobs, silently do no callbacks? */ + return 0; + if ((error = diff_single_init( &d, repo, options, file_cb, hunk_cb, data_cb, payload)) < 0) return error; @@ -1405,6 +1430,9 @@ int git_diff_blob_to_buffer( git_repository *repo = old_blob ? git_object_owner((const git_object *)old_blob) : NULL; + if (!repo && !buf) /* Hmm, given NULLs, silently do no callbacks? */ + return 0; + if ((error = diff_single_init( &d, repo, options, file_cb, hunk_cb, data_cb, payload)) < 0) return error; @@ -1453,11 +1481,19 @@ int git_diff_get_patch( if (patch_ptr) *patch_ptr = NULL; + if (delta_ptr) + *delta_ptr = NULL; + + if (diff_required(diff, "git_diff_get_patch") < 0) + return -1; + + if (diff_context_init( + &ctxt, diff, NULL, NULL, + NULL, diff_patch_hunk_cb, diff_patch_line_cb, NULL) < 0) + return -1; delta = git_vector_get(&diff->deltas, idx); if (!delta) { - if (delta_ptr) - *delta_ptr = NULL; giterr_set(GITERR_INVALID, "Index out of range for delta in diff"); return GIT_ENOTFOUND; } @@ -1470,10 +1506,6 @@ int git_diff_get_patch( (diff->opts.flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0)) return 0; - diff_context_init( - &ctxt, diff, diff->repo, &diff->opts, - NULL, diff_patch_hunk_cb, diff_patch_line_cb, NULL); - if (git_diff_delta__should_skip(ctxt.opts, delta)) return 0; diff --git a/src/diff_tform.c b/src/diff_tform.c index e051732c5..48332d3e5 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -7,6 +7,7 @@ #include "common.h" #include "diff.h" #include "git2/config.h" +#include "git2/blob.h" #include "hashsig.h" static git_diff_delta *diff_delta__dup( @@ -362,24 +363,86 @@ on_error: return -1; } -typedef struct { - /* array of delta index * 2 + (old_file/new_file) -> file hashes */ - git_hashsig *sigs; -} diff_similarity_cache; - -static unsigned int calc_similarity( - void *ref, git_diff_file *old_file, git_diff_file *new_file) +GIT_INLINE(git_diff_file *) similarity_get_file(git_diff_list *diff, size_t idx) { - diff_similarity_cache *cache = ref; + git_diff_delta *delta = git_vector_get(&diff->deltas, idx / 2); + return (idx & 1) ? &delta->new_file : &delta->old_file; +} - GIT_UNUSED(cache); +static int similarity_calc( + git_diff_list *diff, + git_diff_find_options *opts, + size_t file_idx, + void **cache) +{ + int error = 0; + git_diff_file *file = similarity_get_file(diff, file_idx); + git_iterator_type_t src = (file_idx & 1) ? diff->old_src : diff->new_src; - if (git_oid_cmp(&old_file->oid, &new_file->oid) == 0) + if (src == GIT_ITERATOR_TYPE_WORKDIR) { /* compute hashsig from file */ + git_buf path = GIT_BUF_INIT; + + /* TODO: apply wd-to-odb filters to file data if necessary */ + + if (!(error = git_buf_joinpath( + &path, git_repository_workdir(diff->repo), file->path))) + error = opts->metric->file_signature( + &cache[file_idx], file, path.ptr, opts->metric->payload); + + git_buf_free(&path); + } else { /* compute hashsig from blob buffer */ + git_blob *blob = NULL; + + /* TODO: add max size threshold a la diff? */ + + if ((error = git_blob_lookup(&blob, diff->repo, &file->oid)) < 0) + return error; + + error = opts->metric->buffer_signature( + &cache[file_idx], file, git_blob_rawcontent(blob), + git_blob_rawsize(blob), opts->metric->payload); + + git_blob_free(blob); + } + + return error; +} + +static int similarity_measure( + git_diff_list *diff, + git_diff_find_options *opts, + void **cache, + size_t a_idx, + size_t b_idx) +{ + int score = 0; + git_diff_file *a_file = similarity_get_file(diff, a_idx); + git_diff_file *b_file = similarity_get_file(diff, b_idx); + + if (GIT_MODE_TYPE(a_file->mode) != GIT_MODE_TYPE(b_file->mode)) + return 0; + + if (git_oid_cmp(&a_file->oid, &b_file->oid) == 0) return 100; - /* TODO: insert actual similarity algo here */ + /* update signature cache if needed */ + if (!cache[a_idx] && similarity_calc(diff, opts, a_idx, cache) < 0) + return -1; + if (!cache[b_idx] && similarity_calc(diff, opts, b_idx, cache) < 0) + return -1; - return 0; + /* compare signatures */ + if (opts->metric->similarity( + &score, cache[a_idx], cache[b_idx], opts->metric->payload) < 0) + return -1; + + /* clip score */ + if (score < 0) + score = 0; + else if (score > 100) + score = 100; + + return score; } #define FLAG_SET(opts,flag_name) ((opts.flags & flag_name) != 0) @@ -388,67 +451,85 @@ int git_diff_find_similar( git_diff_list *diff, git_diff_find_options *given_opts) { - unsigned int i, j, similarity; + size_t i, j, cache_size, *matches; + int error = 0, similarity; git_diff_delta *from, *to; git_diff_find_options opts; - unsigned int tried_targets, num_changes = 0; - git_vector matches = GIT_VECTOR_INIT; + size_t tried_targets, num_rewrites = 0; + void **cache; - if (normalize_find_opts(diff, &opts, given_opts) < 0) - return -1; + if ((error = normalize_find_opts(diff, &opts, given_opts)) < 0) + return error; - /* first do splits if requested */ + /* TODO: maybe abort if deltas.length > target_limit ??? */ + + cache_size = diff->deltas.length * 2; /* must store b/c length may change */ + cache = git__calloc(cache_size, sizeof(void *)); + GITERR_CHECK_ALLOC(cache); + + matches = git__calloc(diff->deltas.length, sizeof(size_t)); + GITERR_CHECK_ALLOC(matches); + + /* first break MODIFIED records that are too different (if requested) */ if (FLAG_SET(opts, GIT_DIFF_FIND_AND_BREAK_REWRITES)) { git_vector_foreach(&diff->deltas, i, from) { if (from->status != GIT_DELTA_MODIFIED) continue; - /* Right now, this doesn't work right because the similarity - * algorithm isn't actually implemented... - */ - similarity = 100; - /* calc_similarity(NULL, &from->old_file, from->new_file); */ + similarity = similarity_measure( + diff, &opts, cache, 2 * i, 2 * i + 1); - if (similarity < opts.break_rewrite_threshold) { + if (similarity < 0) { + error = similarity; + goto cleanup; + } + + if ((unsigned int)similarity < opts.break_rewrite_threshold) { from->flags |= GIT_DIFF_FLAG__TO_SPLIT; - num_changes++; + num_rewrites++; } } - - /* apply splits as needed */ - if (num_changes > 0 && - apply_splits_and_deletes( - diff, diff->deltas.length + num_changes) < 0) - return -1; } /* next find the most similar delta for each rename / copy candidate */ - if (git_vector_init(&matches, diff->deltas.length, git_diff_delta__cmp) < 0) - return -1; - git_vector_foreach(&diff->deltas, i, from) { tried_targets = 0; + /* skip things that aren't blobs */ + if (GIT_MODE_TYPE(from->old_file.mode) != + GIT_MODE_TYPE(GIT_FILEMODE_BLOB)) + continue; + git_vector_foreach(&diff->deltas, j, to) { if (i == j) continue; + /* skip things that aren't blobs */ + if (GIT_MODE_TYPE(to->new_file.mode) != + GIT_MODE_TYPE(GIT_FILEMODE_BLOB)) + continue; + switch (to->status) { case GIT_DELTA_ADDED: case GIT_DELTA_UNTRACKED: case GIT_DELTA_RENAMED: case GIT_DELTA_COPIED: break; + case GIT_DELTA_MODIFIED: + if ((to->flags & GIT_DIFF_FLAG__TO_SPLIT) == 0) + continue; + break; default: /* only the above status values should be checked */ continue; } /* skip all but DELETED files unless copy detection is on */ - if (from->status != GIT_DELTA_DELETED && - !FLAG_SET(opts, GIT_DIFF_FIND_COPIES)) + if (!FLAG_SET(opts, GIT_DIFF_FIND_COPIES) && + from->status != GIT_DELTA_DELETED && + (from->flags & GIT_DIFF_FLAG__TO_SPLIT) == 0) continue; /* don't check UNMODIFIED files as source unless given option */ @@ -463,34 +544,44 @@ int git_diff_find_similar( /* calculate similarity and see if this pair beats the * similarity score of the current best pair. */ - similarity = calc_similarity(NULL, &from->old_file, &to->new_file); + similarity = similarity_measure( + diff, &opts, cache, 2 * i, 2 * j + 1); - if (to->similarity < similarity) { - to->similarity = similarity; - if (git_vector_set(NULL, &matches, j, from) < 0) - return -1; + if (similarity < 0) { + error = similarity; + goto cleanup; + } + + if (to->similarity < (unsigned int)similarity) { + to->similarity = (unsigned int)similarity; + matches[j] = i + 1; } } } /* next rewrite the diffs with renames / copies */ - num_changes = 0; + num_rewrites = 0; git_vector_foreach(&diff->deltas, j, to) { - from = GIT_VECTOR_GET(&matches, j); - if (!from) { + if (!matches[j]) { assert(to->similarity == 0); continue; } - /* three possible outcomes here: + i = matches[j] - 1; + from = GIT_VECTOR_GET(&diff->deltas, i); + assert(from); + + /* four possible outcomes here: * 1. old DELETED and if over rename threshold, * new becomes RENAMED and old goes away - * 2. old was MODIFIED but FIND_RENAMES_FROM_REWRITES is on and + * 2. old SPLIT and if over rename threshold, + * new becomes RENAMED and old becomes ADDED (clear SPLIT) + * 3. old was MODIFIED but FIND_RENAMES_FROM_REWRITES is on and * old is more similar to new than it is to itself, in which * case, new becomes RENAMED and old becomed ADDED - * 3. otherwise if over copy threshold, new becomes COPIED + * 4. otherwise if over copy threshold, new becomes COPIED */ if (from->status == GIT_DELTA_DELETED) { @@ -503,7 +594,26 @@ int git_diff_find_similar( memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); from->flags |= GIT_DIFF_FLAG__TO_DELETE; - num_changes++; + num_rewrites++; + + continue; + } + + if (from->status == GIT_DELTA_MODIFIED && + (from->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0) + { + if (to->similarity < opts.rename_threshold) { + to->similarity = 0; + continue; + } + + to->status = GIT_DELTA_RENAMED; + memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); + + from->status = GIT_DELTA_ADDED; + from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT; + memset(&from->old_file, 0, sizeof(from->old_file)); + num_rewrites--; continue; } @@ -512,10 +622,15 @@ int git_diff_find_similar( FLAG_SET(opts, GIT_DIFF_FIND_RENAMES_FROM_REWRITES) && to->similarity > opts.rename_threshold) { - similarity = 100; - /* calc_similarity(NULL, &from->old_file, from->new_file); */ + similarity = similarity_measure( + diff, &opts, cache, 2 * i, 2 * i + 1); - if (similarity < opts.rename_from_rewrite_threshold) { + if (similarity < 0) { + error = similarity; + goto cleanup; + } + + if ((unsigned int)similarity < opts.rename_from_rewrite_threshold) { to->status = GIT_DELTA_RENAMED; memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); @@ -538,17 +653,23 @@ int git_diff_find_similar( memcpy(&to->old_file, &from->old_file, sizeof(to->old_file)); } - git_vector_free(&matches); + if (num_rewrites > 0) { + assert(num_rewrites < diff->deltas.length); - if (num_changes > 0) { - assert(num_changes < diff->deltas.length); - - if (apply_splits_and_deletes( - diff, diff->deltas.length - num_changes) < 0) - return -1; + error = apply_splits_and_deletes( + diff, diff->deltas.length - num_rewrites); } - return 0; +cleanup: + git__free(matches); + + for (i = 0; i < cache_size; ++i) { + if (cache[i] != NULL) + opts.metric->free_signature(cache[i], opts.metric->payload); + } + git__free(cache); + + return error; } #undef FLAG_SET diff --git a/tests-clar/diff/blob.c b/tests-clar/diff/blob.c index a6950b871..2ac8dbc51 100644 --- a/tests-clar/diff/blob.c +++ b/tests-clar/diff/blob.c @@ -196,7 +196,7 @@ void test_diff_blob__can_compare_identical_blobs(void) NULL, NULL, &opts, diff_file_cb, diff_hunk_cb, diff_line_cb, &expected)); cl_assert_equal_i(0, expected.files_binary); - assert_identical_blobs_comparison(&expected); + cl_assert_equal_i(0, expected.files); /* NULLs mean no callbacks, period */ memset(&expected, 0, sizeof(expected)); cl_git_pass(git_diff_blobs( @@ -399,7 +399,7 @@ void test_diff_blob__can_compare_blob_to_buffer(void) assert_identical_blobs_comparison(&expected); - /* diff from NULL blob to content of b */ + /* diff from NULL blob to content of a */ memset(&expected, 0, sizeof(expected)); cl_git_pass(git_diff_blob_to_buffer( NULL, a_content, strlen(a_content), From d4b747c1cb72119d783154f88640920d3a3fdb3d Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Thu, 21 Feb 2013 16:44:44 -0800 Subject: [PATCH 10/14] Add diff rename tests with partial similarity This adds some new tests that actually exercise the similarity metric between files to detect renames, copies, and split modified files that are too heavily modified. There is still more testing to do - these tests are just partially covering the cases. There is also one bug fix in this where a change set with only MODIFY being broken into ADD/DELETE (due to low self-similarity) without any additional RENAMED entries would end up not processing the split requests (because the num_rewrites counter got reset). --- include/git2/diff.h | 3 + src/diff_tform.c | 2 - tests-clar/diff/rename.c | 117 +++++++++++++++++- tests-clar/resources/renames/.gitted/index | Bin 272 -> 352 bytes .../resources/renames/.gitted/logs/HEAD | 2 + .../renames/.gitted/logs/refs/heads/master | 2 + .../19/dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 | 1 + .../1c/068dee5790ef1580cfc4cd670915b48d790084 | Bin 0 -> 176 bytes .../36/020db6cdacaa93497f31edcd8f242ff9bc366d | Bin 0 -> 431 bytes .../3c/04741dd4b96c4ae4b00ec0f6e10c816a30aad2 | Bin 0 -> 159 bytes .../42/10ffd5c390b21dd5483375e75288dea9ede512 | Bin 0 -> 1145 bytes .../4e/4cae3e7dd56ed74bff39526d0469e554432953 | Bin 0 -> 452 bytes .../5e/26abc56a5a84d89790f45416648899cbe13109 | Bin 0 -> 163 bytes .../9a/69d960ae94b060f56c2a8702545e2bb1abb935 | Bin 0 -> 464 bytes .../d7/9b202de198fa61b02424b9e25e840dc75e1323 | Bin 0 -> 421 bytes .../ea/f4a3e3bfe68585e90cada20736ace491cd100b | 5 + .../f9/0d4fc20ecddf21eebe6a37e9225d244339d2b5 | Bin 0 -> 441 bytes .../renames/.gitted/refs/heads/master | 2 +- tests-clar/resources/renames/ikeepsix.txt | 27 ++++ tests-clar/resources/renames/sixserving.txt | 43 +++---- .../{sevencities.txt => songof7cities.txt} | 18 +-- tests-clar/resources/renames/songofseven.txt | 49 -------- tests-clar/resources/renames/untimely.txt | 24 ++++ 23 files changed, 211 insertions(+), 84 deletions(-) create mode 100644 tests-clar/resources/renames/.gitted/objects/19/dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 create mode 100644 tests-clar/resources/renames/.gitted/objects/1c/068dee5790ef1580cfc4cd670915b48d790084 create mode 100644 tests-clar/resources/renames/.gitted/objects/36/020db6cdacaa93497f31edcd8f242ff9bc366d create mode 100644 tests-clar/resources/renames/.gitted/objects/3c/04741dd4b96c4ae4b00ec0f6e10c816a30aad2 create mode 100644 tests-clar/resources/renames/.gitted/objects/42/10ffd5c390b21dd5483375e75288dea9ede512 create mode 100644 tests-clar/resources/renames/.gitted/objects/4e/4cae3e7dd56ed74bff39526d0469e554432953 create mode 100644 tests-clar/resources/renames/.gitted/objects/5e/26abc56a5a84d89790f45416648899cbe13109 create mode 100644 tests-clar/resources/renames/.gitted/objects/9a/69d960ae94b060f56c2a8702545e2bb1abb935 create mode 100644 tests-clar/resources/renames/.gitted/objects/d7/9b202de198fa61b02424b9e25e840dc75e1323 create mode 100644 tests-clar/resources/renames/.gitted/objects/ea/f4a3e3bfe68585e90cada20736ace491cd100b create mode 100644 tests-clar/resources/renames/.gitted/objects/f9/0d4fc20ecddf21eebe6a37e9225d244339d2b5 create mode 100644 tests-clar/resources/renames/ikeepsix.txt rename tests-clar/resources/renames/{sevencities.txt => songof7cities.txt} (82%) delete mode 100644 tests-clar/resources/renames/songofseven.txt create mode 100644 tests-clar/resources/renames/untimely.txt diff --git a/include/git2/diff.h b/include/git2/diff.h index 379f4694e..ca3484332 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -390,6 +390,9 @@ typedef enum { /** split large rewrites into delete/add pairs (`--break-rewrites=/M`) */ GIT_DIFF_FIND_AND_BREAK_REWRITES = (1 << 4), + /** turn on all finding features */ + GIT_DIFF_FIND_ALL = (0x1f), + /** measure similarity ignoring leading whitespace (default) */ GIT_DIFF_FIND_IGNORE_LEADING_WHITESPACE = 0, /** measure similarity ignoring all whitespace */ diff --git a/src/diff_tform.c b/src/diff_tform.c index 48332d3e5..ae0fd36d6 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -561,8 +561,6 @@ int git_diff_find_similar( /* next rewrite the diffs with renames / copies */ - num_rewrites = 0; - git_vector_foreach(&diff->deltas, j, to) { if (!matches[j]) { assert(to->similarity == 0); diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index 2995b4ef5..ec99e4dbe 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -23,8 +23,16 @@ void test_diff_rename__cleanup(void) * serving.txt -> sixserving.txt (rename, no change, 100% match) * sevencities.txt -> sevencities.txt (no change) * sevencities.txt -> songofseven.txt (copy, no change, 100% match) - * - * TODO: add commits with various % changes of copy / rename + * commit 1c068dee5790ef1580cfc4cd670915b48d790084 + * songofseven.txt -> songofseven.txt (major rewrite, <20% match - split) + * sixserving.txt -> sixserving.txt (indentation change) + * sixserving.txt -> ikeepsix.txt (copy, add title, >80% match) + * sevencities.txt (no change) + * commit 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 + * songofseven.txt -> untimely.txt (rename, convert to crlf) + * ikeepsix.txt -> ikeepsix.txt (reorder sections in file) + * sixserving.txt -> sixserving.txt (whitespace change - not just indent) + * sevencities.txt -> songof7cities.txt (rename, small text changes) */ void test_diff_rename__match_oid(void) @@ -133,3 +141,108 @@ void test_diff_rename__checks_options_version(void) git_tree_free(old_tree); git_tree_free(new_tree); } + +void test_diff_rename__not_exact_match(void) +{ + const char *sha0 = "2bc7f351d20b53f1c72c16c4b036e491c478c49a"; + const char *sha1 = "1c068dee5790ef1580cfc4cd670915b48d790084"; + const char *sha2 = "19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13"; + git_tree *old_tree, *new_tree; + git_diff_list *diff; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; + diff_expects exp; + + /* Changes: + * songofseven.txt -> songofseven.txt (major rewrite, <20% match - split) + * sixserving.txt -> sixserving.txt (indentation change) + * sixserving.txt -> ikeepsix.txt (copy, add title, >80% match) + * sevencities.txt (no change) + */ + + old_tree = resolve_commit_oid_to_tree(g_repo, sha0); + new_tree = resolve_commit_oid_to_tree(g_repo, sha1); + + /* Must pass GIT_DIFF_INCLUDE_UNMODIFIED if you expect to emulate + * --find-copies-harder during rename transformion... + */ + diffopts.flags |= GIT_DIFF_INCLUDE_UNMODIFIED; + + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + /* git diff --no-renames \ + * 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 + */ + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(4, exp.files); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]); + + /* git diff 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 \ + * 2bc7f351d20b53f1c72c16c4b036e491c478c49a + */ + cl_git_pass(git_diff_find_similar(diff, NULL)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(4, exp.files); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]); + + git_diff_list_free(diff); + + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + /* git diff --find-copies-harder --break-rewrites \ + * 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 + */ + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(5, exp.files); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_COPIED]); + + git_diff_list_free(diff); + + /* Changes: + * songofseven.txt -> untimely.txt (rename, convert to crlf) + * ikeepsix.txt -> ikeepsix.txt (reorder sections in file) + * sixserving.txt -> sixserving.txt (whitespace - not just indent) + * sevencities.txt -> songof7cities.txt (rename, small text changes) + */ + + git_tree_free(old_tree); + old_tree = new_tree; + new_tree = resolve_commit_oid_to_tree(g_repo, sha2); + + /* moar tests needed */ + + git_tree_free(old_tree); + git_tree_free(new_tree); +} + +void test_diff_rename__working_directory_changes(void) +{ + /* let's rewrite some files in the working directory on demand */ + + /* and with / without CRLF changes */ +} diff --git a/tests-clar/resources/renames/.gitted/index b/tests-clar/resources/renames/.gitted/index index 1fc69fcbefd9ccf13194b86b1493359f0f5962e6..72363c0f52fba5560db5c0a127ddebfc47c0ba62 100644 GIT binary patch literal 352 zcmZ?q402{*U|<4bmO!-yRX~~nM)Lv1m^6Z}F)%bPVPIhV3X~E7Dn0k=%i_oTpS8BW zj%!(pCmOpi#xCMP-?J=`a%m)fU=8%!ANS^Eyz?C5dTJALN$-+%dE_s3G1e8 zNcftg)y@_!TH6!oa|>D$P(n=J)gF z9Q8mh>qTGeH1u@3cQNo6rB>Rurcem1X9oLycnyXlL*Tn+K(#?gLsyl>74Y((}{6 du7nzB78M<(+dawb5?_5t|BE2K6H6U-004~dPl*5k diff --git a/tests-clar/resources/renames/.gitted/logs/HEAD b/tests-clar/resources/renames/.gitted/logs/HEAD index 34222ed7d..e69792263 100644 --- a/tests-clar/resources/renames/.gitted/logs/HEAD +++ b/tests-clar/resources/renames/.gitted/logs/HEAD @@ -1,2 +1,4 @@ 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer 1351024687 -0700 commit (initial): Initial commit 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer 1351024817 -0700 commit: copy and rename with no change +2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer 1361485758 -0800 commit: rewrites, copies with changes, etc. +1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer 1361486360 -0800 commit: more renames and smallish modifications diff --git a/tests-clar/resources/renames/.gitted/logs/refs/heads/master b/tests-clar/resources/renames/.gitted/logs/refs/heads/master index 34222ed7d..e69792263 100644 --- a/tests-clar/resources/renames/.gitted/logs/refs/heads/master +++ b/tests-clar/resources/renames/.gitted/logs/refs/heads/master @@ -1,2 +1,4 @@ 0000000000000000000000000000000000000000 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 Russell Belfer 1351024687 -0700 commit (initial): Initial commit 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 2bc7f351d20b53f1c72c16c4b036e491c478c49a Russell Belfer 1351024817 -0700 commit: copy and rename with no change +2bc7f351d20b53f1c72c16c4b036e491c478c49a 1c068dee5790ef1580cfc4cd670915b48d790084 Russell Belfer 1361485758 -0800 commit: rewrites, copies with changes, etc. +1c068dee5790ef1580cfc4cd670915b48d790084 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 Russell Belfer 1361486360 -0800 commit: more renames and smallish modifications diff --git a/tests-clar/resources/renames/.gitted/objects/19/dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 b/tests-clar/resources/renames/.gitted/objects/19/dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 new file mode 100644 index 000000000..4be4c6952 --- /dev/null +++ b/tests-clar/resources/renames/.gitted/objects/19/dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 @@ -0,0 +1 @@ +xM!Ei@3ސc,\X K {Nrbo,xzYC<h[&?=fcvyCW#K@M{9s$^8%(VoNPHty#e~)= zcO>kCa2HM7%eZEE?AxaE40oLe&2Y)-_MFPNr?aff705c`(E2vO2?L_3wEin~`mbrS ee)3dwJA$n5NjLb^5Xcx`&mYtxXY&O)1x?P|R9TS# literal 0 HcmV?d00001 diff --git a/tests-clar/resources/renames/.gitted/objects/36/020db6cdacaa93497f31edcd8f242ff9bc366d b/tests-clar/resources/renames/.gitted/objects/36/020db6cdacaa93497f31edcd8f242ff9bc366d new file mode 100644 index 0000000000000000000000000000000000000000..f4f9303edb4fdc8d0c08efa43034260d0cfebf50 GIT binary patch literal 431 zcmV;g0Z{&U0ew=zuG26OIs1;QqS(oCbr_(mF*?n=CgS9;eU@(1mE3# zMIzW-!RZM+5{%d1jjYIe>P^Nd(9Yq2!OQYyr(jUN*fVk; zmgfiyuQg%I=}DN)Mn;5LB3wz8H;HZ(N8R$GsjQ?HO5CZB6aTtQ4ACo$o&`&YgS6;E zB-aY>EyJ~aVXhI0-4@kO@-oninqb5?YM(g2v`RQf z(L+a84cz1(O;g$BO1wZTui}fDd>>a*^4m(gwOuDsJm&^CT&dq;vy~<7NOADRG+ZR6 z;E_fu`2F*jIvT;ig(qb37|GX`^lfX6I+=i;0)420QP$$zOI5MO(e#Rj>vIgc1;K_c zhQ-s*PTQD@^kN7Fu)7vtTeZFfcPQQOL|rO)V(Stk5f|C}A*T;@x(3&8o?s^@eZH z_N(as++&stRacx^mYSEGS(2Gr3|5+EC?E6td2^0>AeZ%`FLoMwI^DZqN;4~pQ;W(n z^U}d8ug_M{eK_M+;szC!osZ&Lc#p>kE5lUg=cVVTfsF<$_48S0S9>+@y7zy}pj?*B NryP$2*S literal 0 HcmV?d00001 diff --git a/tests-clar/resources/renames/.gitted/objects/42/10ffd5c390b21dd5483375e75288dea9ede512 b/tests-clar/resources/renames/.gitted/objects/42/10ffd5c390b21dd5483375e75288dea9ede512 new file mode 100644 index 0000000000000000000000000000000000000000..d351a6d13b620ff34fa29c0d283f243946fb7de0 GIT binary patch literal 1145 zcmV-<1cv)~0d-YPZ`?);lOo`LmILPT@aa~M#srSLy2^%&c36;gkYn1?r5=Tr1NC{5g*C=tM#pI z9OID|4OE?-QwOw2aJbGsQItLb{4#*D33Lwl zeC5A!9S<|h>^PE1dnQS&JHZM((I_EuF=|J!B8hC!Fz_Py-h377FnUDAG|p3YaPe2> znCzA&u(n`w%N{V!h=?SAH6CR+(PXSc47?F2lEEO`!g|aej4@@O1Qqm=(=XLC z($dV95muY;Xj%nn(`kE0VIt$V(^qoEal-#F@LlyMgFGd1!rvkp{01a|o6~L{INF#L zSwTLTjtPb0RLVG84(;wbPV$CtJvAmJpec1VFoVz#wT`9w@kaO)2M9)h?a5J5%2h?Ij!J4hYaMkZ8abb6;2G8@M z=B;Bk`DlWt>c#HO#8V?^e9fa*fRUW_DjXBgvPkBHa}!5Z7rF{X!U-CBQ47U`=DOcs z!O`tZI;g;4IQpw(j$Tp_AFAq;EMz7fXG)L(j-c_2f?VSOHs*V|5G@5ZF{7+|1#y%;n%mPXb9P7g zdp61d6d7dIelWOG8bj?m?n~bmO-y-%6A={ID)4)gk-7%nrXhVX&R`rYBL-@L*g~fRG0!;!V~?Pbb2hDJ>)Inu4W4 z5aTp67GWHfR#<}m$nc|(zFHX7D;t+aOcaz*`?p@$%(^Oc)Phmbbukqi%%KakKcZv5 z*@gpl!S}oHeg51qdft^%w%WV?%Q(hri(rwtG`p^N0LPv$Vns}7n5eF?)4CON0LxUH LY^eSP6%Gp}TSqL) literal 0 HcmV?d00001 diff --git a/tests-clar/resources/renames/.gitted/objects/4e/4cae3e7dd56ed74bff39526d0469e554432953 b/tests-clar/resources/renames/.gitted/objects/4e/4cae3e7dd56ed74bff39526d0469e554432953 new file mode 100644 index 0000000000000000000000000000000000000000..5e6ebd5e026aaa6b177e86c219ecf9258c821116 GIT binary patch literal 452 zcmV;#0XzP90bNqP&eSjvr1n#6M{}Wof{FrYPWnDh2!yCN@#bvdc%!wG=-!^$92IGt zoM&g}=fVX-~P{DOW0a7E0V`6sfdu6dE5mE6r+tzHPKM zfWw!qQ|@#C+map_RZ!q+66$)6fRFFe!mGAU!1W|lc@fh2d|&vFn1AGc3BQw)$s~D~ zNV={jtz`v6jacAQt{~-xl^`94aOEx+;$3rOY{jcXC$;KM92q&62Swom#XjQZ3qi&z zmRpi|p(AMlXwo|CR-?oazNwuaM&NSK(9;tVz!>`l&LcMoKXofmlk${Yn>0SoOWCml zu8QiJ)JxT)NW8CA7GiNX~FfcPQQOL|rO)V(Stk5f|C}DW@W%1+v&stkw@~mCN zZnoyh#Ipk2P<22h#i>PQnR)48l|Om?5AmJ7ulR0XmibGiSQTf>OIu+o^Yha4)6A1I zOEOc7!HS&({$D*jVUz4t598A3K^^y2zI`eLRa}}^l9`*DQwdf#EAwW;x+xnHzUFAP RGlj%yZ(P086aZGANWbzjPe1?w literal 0 HcmV?d00001 diff --git a/tests-clar/resources/renames/.gitted/objects/9a/69d960ae94b060f56c2a8702545e2bb1abb935 b/tests-clar/resources/renames/.gitted/objects/9a/69d960ae94b060f56c2a8702545e2bb1abb935 new file mode 100644 index 0000000000000000000000000000000000000000..f75178c59abb114a2724eb88d32f32ce03aef3be GIT binary patch literal 464 zcmV;>0Wbb|0bNqDuGBCPq_*UL*pB9f4yY)A=A`#{LLfxFi8p5ppEt6dMECd1`lxVY zC!U?1nf07?rjPGGy!lmnx=Fr!_vzpA&bwFbF_kN&LOHEMksHk-g>D>#<~#O6t5)0J zHrgA&-hJ6SrA{ZnE$WF{1qbdHp{}C_gnX6e-HWzP0QM+U_$H+5IN$h>RQpK%$-CcC zsmdbhkU+X^Ra#35juOzq%1lH|4NXBiP2OF&%MBv>rpv<@dTkf|qV>JdSpknIBdnVNX_Q}+x>F)h&q zO5=0f%7HfUmQ^2#Jr_N*ge&Y<`dm9gHayu(9?E&v+O7Mm&RL-PNQ<;#Du`^;S#`h0 z4h;`G2<=4>t*)z-CN_vY2k3*acjnadg-cP>;~nA8 z2?%0=3`y`=iNj&>A}N5)`Bov!%rTu>(Xzl2Q$u@{(NLUvqwiI>DBZiSdV`T3hk-#c zBNvSM@en|N-MI$KQH>x}!N9{bbyd)6nI=58$e8Y9Lc<>Yx2B1H?BNa~dQO{7+`GTo G-~OIXpz5sv literal 0 HcmV?d00001 diff --git a/tests-clar/resources/renames/.gitted/objects/d7/9b202de198fa61b02424b9e25e840dc75e1323 b/tests-clar/resources/renames/.gitted/objects/d7/9b202de198fa61b02424b9e25e840dc75e1323 new file mode 100644 index 0000000000000000000000000000000000000000..daa2b3997579e55909bdbc77202824fb145fb6ef GIT binary patch literal 421 zcmV;W0b2fe0Zo!kj?^#^g<1D0-mCzrd&CMBtQu+1v}OSUq+UCdN^HfkTeg#s9f#nA zoP<~I4#OgfQ$H_Xy>jDYgAecCzuEyOJUu*oMRq@7V~|{sCCx7MhfxCaz?&adRunV! zwqO((@7Xm3S>AqR0kbY4m{BrJ5}d6mFrnpPy}fTjM+P6`vISXtEsOV+BMbAz%7{2i zg4d)zSqiBE|PK#$Rp?Z^XoTL->qN} zxnvXx7)kH6;AOAMd$tih2M)1X0NtARcj|=ptJV26mtmR|O$!QGnj4oun>*uDzL}ve zaT+!}N8J{W@ggDDa#3a!,R!-EaI>yo*4戁Rf m&eIA*!;dݬ +Ho +OUDyTVHpwqH7Ʒ.ts6{Z+X\)C5Q9 +%t*&&v;|'4 Du[7he!NK*"C-=`#؎$Ee2T|@NBsslW|/0¬aȥJNv)-ڡiۤ3bbO:uWMNX7T \ No newline at end of file diff --git a/tests-clar/resources/renames/.gitted/objects/f9/0d4fc20ecddf21eebe6a37e9225d244339d2b5 b/tests-clar/resources/renames/.gitted/objects/f9/0d4fc20ecddf21eebe6a37e9225d244339d2b5 new file mode 100644 index 0000000000000000000000000000000000000000..f6d933be9929e07a1e66031bc36906d0ae277e8f GIT binary patch literal 441 zcmV;q0Y?6K0bNqdP9rf8mf`?0Jyee#Cwu zUqZDf3kx|++Wjb3RlD-BqV@Xb;uq1w-Q6QmcCSP>2FX1Rl13MrWhcQ9UA2m) z06v7h#ed=6&*BQo%)0n#htMQoVI`|s*}>JdBne6;`u3l?|I(wADQ}d18;aXRM@cyb z_}f9_+n>MY*+M`oas;PdQ0XAXm6p#(urNCti5fK_Aeb#E3- Date: Fri, 22 Feb 2013 10:17:08 -0800 Subject: [PATCH 11/14] Fix tests for find_similar and related This fixes both a test that I broke in diff::patch where I was relying on the current state of the working directory for the renames test data and fixes an unstable test in diff::rename where the environment setting for the "diff.renames" config was being allowed to influence the test results. --- tests-clar/diff/patch.c | 18 +++++++++--------- tests-clar/diff/rename.c | 41 +++++++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/tests-clar/diff/patch.c b/tests-clar/diff/patch.c index 18e90f92d..5cb97fb2d 100644 --- a/tests-clar/diff/patch.c +++ b/tests-clar/diff/patch.c @@ -144,11 +144,11 @@ void test_diff_patch__hunks_have_correct_line_numbers(void) size_t hdrlen, hunklen, textlen; char origin; int oldno, newno; - const char *new_content = "The Song of Seven Cities\n========================\n\nI WAS Lord of Cities very sumptuously builded.\nSeven roaring Cities paid me tribute from afar.\nIvory their outposts were—the guardrooms of them gilded,\nAnd garrisoned with Amazons invincible in war.\n\nThis is some new text;\nNot as good as the old text;\nBut here it is.\n\nSo they warred and trafficked only yesterday, my Cities.\nTo-day there is no mark or mound of where my Cities stood.\nFor the River rose at midnight and it washed away my Cities.\nThey are evened with Atlantis and the towns before the Flood.\n\nRain on rain-gorged channels raised the water-levels round them,\nFreshet backed on freshet swelled and swept their world from sight,\nTill the emboldened floods linked arms and, flashing forward, drowned them—\nDrowned my Seven Cities and their peoples in one night!\n\nLow among the alders lie their derelict foundations,\nThe beams wherein they trusted and the plinths whereon they built—\nMy rulers and their treasure and their unborn populations,\nDead, destroyed, aborted, and defiled with mud and silt!\n\nAnother replacement;\nBreaking up the poem;\nGenerating some hunks.\n\nTo the sound of trumpets shall their seed restore my Cities\nWealthy and well-weaponed, that once more may I behold\nAll the world go softly when it walks before my Cities,\nAnd the horses and the chariots fleeing from them as of old!\n\n -- Rudyard Kipling\n"; + const char *new_content = "The Song of Seven Cities\n------------------------\n\nI WAS Lord of Cities very sumptuously builded.\nSeven roaring Cities paid me tribute from afar.\nIvory their outposts were--the guardrooms of them gilded,\nAnd garrisoned with Amazons invincible in war.\n\nThis is some new text;\nNot as good as the old text;\nBut here it is.\n\nSo they warred and trafficked only yesterday, my Cities.\nTo-day there is no mark or mound of where my Cities stood.\nFor the River rose at midnight and it washed away my Cities.\nThey are evened with Atlantis and the towns before the Flood.\n\nRain on rain-gorged channels raised the water-levels round them,\nFreshet backed on freshet swelled and swept their world from sight,\nTill the emboldened floods linked arms and, flashing forward, drowned them--\nDrowned my Seven Cities and their peoples in one night!\n\nLow among the alders lie their derelict foundations,\nThe beams wherein they trusted and the plinths whereon they built--\nMy rulers and their treasure and their unborn populations,\nDead, destroyed, aborted, and defiled with mud and silt!\n\nAnother replacement;\nBreaking up the poem;\nGenerating some hunks.\n\nTo the sound of trumpets shall their seed restore my Cities\nWealthy and well-weaponed, that once more may I behold\nAll the world go softly when it walks before my Cities,\nAnd the horses and the chariots fleeing from them as of old!\n\n -- Rudyard Kipling\n"; g_repo = cl_git_sandbox_init("renames"); - cl_git_rewritefile("renames/songofseven.txt", new_content); + cl_git_rewritefile("renames/songof7cities.txt", new_content); cl_git_pass(git_repository_head_tree(&head, g_repo)); @@ -178,14 +178,14 @@ void test_diff_patch__hunks_have_correct_line_numbers(void) cl_git_pass(git_diff_patch_get_line_in_hunk( &origin, &text, &textlen, &oldno, &newno, patch, 0, 0)); cl_assert_equal_i(GIT_DIFF_LINE_CONTEXT, (int)origin); - cl_assert(strncmp("Ivory their outposts were—the guardrooms of them gilded,\n", text, textlen) == 0); + cl_assert(strncmp("Ivory their outposts were--the guardrooms of them gilded,\n", text, textlen) == 0); cl_assert_equal_i(6, oldno); cl_assert_equal_i(6, newno); cl_git_pass(git_diff_patch_get_line_in_hunk( &origin, &text, &textlen, &oldno, &newno, patch, 0, 3)); cl_assert_equal_i(GIT_DIFF_LINE_DELETION, (int)origin); - cl_assert(strncmp("All the world went softly when it walked before my Cities—\n", text, textlen) == 0); + cl_assert(strncmp("All the world went softly when it walked before my Cities--\n", text, textlen) == 0); cl_assert_equal_i(9, oldno); cl_assert_equal_i(-1, newno); @@ -271,32 +271,32 @@ void test_diff_patch__line_counts_with_eofnl(void) g_repo = cl_git_sandbox_init("renames"); - cl_git_pass(git_futils_readbuffer(&content, "renames/songofseven.txt")); + cl_git_pass(git_futils_readbuffer(&content, "renames/songof7cities.txt")); /* remove first line */ end = git_buf_cstr(&content) + git_buf_find(&content, '\n') + 1; git_buf_consume(&content, end); - cl_git_rewritefile("renames/songofseven.txt", content.ptr); + cl_git_rewritefile("renames/songof7cities.txt", content.ptr); check_single_patch_stats(g_repo, 1, 0, 1); /* remove trailing whitespace */ git_buf_rtrim(&content); - cl_git_rewritefile("renames/songofseven.txt", content.ptr); + cl_git_rewritefile("renames/songof7cities.txt", content.ptr); check_single_patch_stats(g_repo, 2, 1, 2); /* add trailing whitespace */ cl_git_pass(git_repository_index(&index, g_repo)); - cl_git_pass(git_index_add_bypath(index, "songofseven.txt")); + cl_git_pass(git_index_add_bypath(index, "songof7cities.txt")); cl_git_pass(git_index_write(index)); git_index_free(index); cl_git_pass(git_buf_putc(&content, '\n')); - cl_git_rewritefile("renames/songofseven.txt", content.ptr); + cl_git_rewritefile("renames/songof7cities.txt", content.ptr); check_single_patch_stats(g_repo, 1, 1, 1); diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index ec99e4dbe..f27fd7d82 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -184,10 +184,37 @@ void test_diff_rename__not_exact_match(void) cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]); - /* git diff 31e47d8c1fa36d7f8d537b96158e3f024de0a9f2 \ - * 2bc7f351d20b53f1c72c16c4b036e491c478c49a + /* git diff -M 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 + * + * must not pass NULL for opts because it will pick up environment + * values for "diff.renames" and test won't be consistent. */ - cl_git_pass(git_diff_find_similar(diff, NULL)); + opts.flags = GIT_DIFF_FIND_RENAMES; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(4, exp.files); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_UNMODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]); + + git_diff_list_free(diff); + + /* git diff -M -C 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 + * + * must not pass NULL for opts because it will pick up environment + * values for "diff.renames" and test won't be consistent. + */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + opts.flags = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES; + cl_git_pass(git_diff_find_similar(diff, &opts)); memset(&exp, 0, sizeof(exp)); cl_git_pass(git_diff_foreach( @@ -200,13 +227,13 @@ void test_diff_rename__not_exact_match(void) git_diff_list_free(diff); - cl_git_pass(git_diff_tree_to_tree( - &diff, g_repo, old_tree, new_tree, &diffopts)); - - /* git diff --find-copies-harder --break-rewrites \ + /* git diff -M -C --find-copies-harder --break-rewrites \ * 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ * 1c068dee5790ef1580cfc4cd670915b48d790084 */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + opts.flags = GIT_DIFF_FIND_ALL; cl_git_pass(git_diff_find_similar(diff, &opts)); From f8275890676a56735b1af1abde859722f040a61c Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 22 Feb 2013 10:19:50 -0800 Subject: [PATCH 12/14] Replace static data with configured metric Instead of creating three git_diff_similarity_metric statically for the various config options, just create the metric structure on demand and populate it, using the payload to specific the extra flags that should be passed to the hashsig. This removes a level of obfuscation from the code, I think. --- src/diff_tform.c | 71 ++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index ae0fd36d6..66a4702f2 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -170,21 +170,21 @@ int git_diff_merge( return error; } -#define FIND_SIMILAR_HASHSIG(NAME,OPT) \ - static int find_similar__hashsig_for_file ## NAME( \ - void **out, const git_diff_file *f, const char *path, void *p) { \ - GIT_UNUSED(f); GIT_UNUSED(p); \ - return git_hashsig_create_fromfile((git_hashsig **)out, path, OPT); \ - } \ - static int find_similar__hashsig_for_buf ## NAME( \ - void **out, const git_diff_file *f, const char *buf, size_t len, void *p) { \ - GIT_UNUSED(f); GIT_UNUSED(p); \ - return git_hashsig_create((git_hashsig **)out, buf, len, OPT); \ - } +static int find_similar__hashsig_for_file( + void **out, const git_diff_file *f, const char *path, void *p) +{ + git_hashsig_option_t opt = (git_hashsig_option_t)p; + GIT_UNUSED(f); + return git_hashsig_create_fromfile((git_hashsig **)out, path, opt); +} -FIND_SIMILAR_HASHSIG(_default, GIT_HASHSIG_SMART_WHITESPACE); -FIND_SIMILAR_HASHSIG(_ignore_whitespace, GIT_HASHSIG_IGNORE_WHITESPACE); -FIND_SIMILAR_HASHSIG(_include_whitespace, GIT_HASHSIG_NORMAL); +static int find_similar__hashsig_for_buf( + void **out, const git_diff_file *f, const char *buf, size_t len, void *p) +{ + git_hashsig_option_t opt = (git_hashsig_option_t)p; + GIT_UNUSED(f); + return git_hashsig_create((git_hashsig **)out, buf, len, opt); +} static void find_similar__hashsig_free(void *sig, void *payload) { @@ -200,30 +200,6 @@ static int find_similar__calc_similarity( return 0; } -static git_diff_similarity_metric find_similar__internal_metrics[3] = { - { - find_similar__hashsig_for_file_default, - find_similar__hashsig_for_buf_default, - find_similar__hashsig_free, - find_similar__calc_similarity, - NULL - }, - { - find_similar__hashsig_for_file_ignore_whitespace, - find_similar__hashsig_for_buf_ignore_whitespace, - find_similar__hashsig_free, - find_similar__calc_similarity, - NULL - }, - { - find_similar__hashsig_for_file_include_whitespace, - find_similar__hashsig_for_buf_include_whitespace, - find_similar__hashsig_free, - find_similar__calc_similarity, - NULL - } -}; - #define DEFAULT_THRESHOLD 50 #define DEFAULT_BREAK_REWRITE_THRESHOLD 60 #define DEFAULT_TARGET_LIMIT 200 @@ -292,14 +268,22 @@ static int normalize_find_opts( opts->target_limit = limit; } - /* for now, always assign the same internal metric */ + /* assign the internal metric with whitespace flag as payload */ if (!opts->metric) { + opts->metric = git__malloc(sizeof(git_diff_similarity_metric)); + GITERR_CHECK_ALLOC(opts->metric); + + opts->metric->file_signature = find_similar__hashsig_for_file; + opts->metric->buffer_signature = find_similar__hashsig_for_buf; + opts->metric->free_signature = find_similar__hashsig_free; + opts->metric->similarity = find_similar__calc_similarity; + if (opts->flags & GIT_DIFF_FIND_IGNORE_WHITESPACE) - opts->metric = &find_similar__internal_metrics[1]; + opts->metric->payload = (void *)GIT_HASHSIG_IGNORE_WHITESPACE; else if (opts->flags & GIT_DIFF_FIND_DONT_IGNORE_WHITESPACE) - opts->metric = &find_similar__internal_metrics[2]; + opts->metric->payload = (void *)GIT_HASHSIG_NORMAL; else - opts->metric = &find_similar__internal_metrics[0]; + opts->metric->payload = (void *)GIT_HASHSIG_SMART_WHITESPACE; } return 0; @@ -667,6 +651,9 @@ cleanup: } git__free(cache); + if (!given_opts || !given_opts->metric) + git__free(opts.metric); + return error; } From 0a0089131f6e3c0d6be35c47dd243c87cf8c3fc9 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 22 Feb 2013 10:21:02 -0800 Subject: [PATCH 13/14] Minor improvements to find_similar code This moves a couple of checks outside of the inner loop of the find_similar rename/copy detection phase that are only dependent on the "from" side of a detection. Also, this replaces the inefficient initialization of the options structure when a value is not provided explicitly by the user. --- src/diff_tform.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/diff_tform.c b/src/diff_tform.c index 66a4702f2..958d2bfec 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -210,7 +210,6 @@ static int normalize_find_opts( git_diff_find_options *given) { git_config *cfg = NULL; - const char *val; if (diff->repo != NULL && git_repository_config__weakptr(&cfg, diff->repo) < 0) @@ -219,8 +218,9 @@ static int normalize_find_opts( if (given != NULL) memcpy(opts, given, sizeof(*opts)); else { - git_diff_find_options init = GIT_DIFF_FIND_OPTIONS_INIT; - memmove(opts, &init, sizeof(init)); + const char *val = NULL; + + GIT_INIT_STRUCTURE(opts, GIT_DIFF_FIND_OPTIONS_VERSION); opts->flags = GIT_DIFF_FIND_RENAMES; @@ -486,6 +486,17 @@ int git_diff_find_similar( GIT_MODE_TYPE(GIT_FILEMODE_BLOB)) continue; + /* don't check UNMODIFIED files as source unless given option */ + if (from->status == GIT_DELTA_UNMODIFIED && + !FLAG_SET(opts, GIT_DIFF_FIND_COPIES_FROM_UNMODIFIED)) + continue; + + /* skip all but DELETED files unless copy detection is on */ + if (!FLAG_SET(opts, GIT_DIFF_FIND_COPIES) && + from->status != GIT_DELTA_DELETED && + (from->flags & GIT_DIFF_FLAG__TO_SPLIT) == 0) + continue; + git_vector_foreach(&diff->deltas, j, to) { if (i == j) continue; @@ -510,18 +521,7 @@ int git_diff_find_similar( continue; } - /* skip all but DELETED files unless copy detection is on */ - if (!FLAG_SET(opts, GIT_DIFF_FIND_COPIES) && - from->status != GIT_DELTA_DELETED && - (from->flags & GIT_DIFF_FLAG__TO_SPLIT) == 0) - continue; - - /* don't check UNMODIFIED files as source unless given option */ - if (from->status == GIT_DELTA_UNMODIFIED && - !FLAG_SET(opts, GIT_DIFF_FIND_COPIES_FROM_UNMODIFIED)) - continue; - - /* cap on maximum files we'll examine */ + /* cap on maximum files we'll examine (per "from" file) */ if (++tried_targets > opts.target_limit) break; From 1be4ba984216dfcb1f07945240c2831395fd0460 Mon Sep 17 00:00:00 2001 From: Russell Belfer Date: Fri, 22 Feb 2013 11:13:01 -0800 Subject: [PATCH 14/14] More rename detection tests This includes tests for crlf changes, whitespace changes with the default comparison and with the ignore whitespace comparison, and more sensitivity checking for the comparison code. --- tests-clar/diff/rename.c | 114 +++++++++++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 15 deletions(-) diff --git a/tests-clar/diff/rename.c b/tests-clar/diff/rename.c index f27fd7d82..539512538 100644 --- a/tests-clar/diff/rename.c +++ b/tests-clar/diff/rename.c @@ -153,11 +153,11 @@ void test_diff_rename__not_exact_match(void) git_diff_find_options opts = GIT_DIFF_FIND_OPTIONS_INIT; diff_expects exp; - /* Changes: - * songofseven.txt -> songofseven.txt (major rewrite, <20% match - split) - * sixserving.txt -> sixserving.txt (indentation change) - * sixserving.txt -> ikeepsix.txt (copy, add title, >80% match) - * sevencities.txt (no change) + /* == Changes ===================================================== + * songofseven.txt -> songofseven.txt (major rewrite, <20% match - split) + * sixserving.txt -> sixserving.txt (indentation change) + * sixserving.txt -> ikeepsix.txt (copy, add title, >80% match) + * sevencities.txt (no change) */ old_tree = resolve_commit_oid_to_tree(g_repo, sha0); @@ -204,11 +204,9 @@ void test_diff_rename__not_exact_match(void) git_diff_list_free(diff); - /* git diff -M -C 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ + /* git diff -M -C \ + * 2bc7f351d20b53f1c72c16c4b036e491c478c49a \ * 1c068dee5790ef1580cfc4cd670915b48d790084 - * - * must not pass NULL for opts because it will pick up environment - * values for "diff.renames" and test won't be consistent. */ cl_git_pass(git_diff_tree_to_tree( &diff, g_repo, old_tree, new_tree, &diffopts)); @@ -250,18 +248,104 @@ void test_diff_rename__not_exact_match(void) git_diff_list_free(diff); - /* Changes: - * songofseven.txt -> untimely.txt (rename, convert to crlf) - * ikeepsix.txt -> ikeepsix.txt (reorder sections in file) - * sixserving.txt -> sixserving.txt (whitespace - not just indent) - * sevencities.txt -> songof7cities.txt (rename, small text changes) + /* == Changes ===================================================== + * songofseven.txt -> untimely.txt (rename, convert to crlf) + * ikeepsix.txt -> ikeepsix.txt (reorder sections in file) + * sixserving.txt -> sixserving.txt (whitespace - not just indent) + * sevencities.txt -> songof7cities.txt (rename, small text changes) */ git_tree_free(old_tree); old_tree = new_tree; new_tree = resolve_commit_oid_to_tree(g_repo, sha2); - /* moar tests needed */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + /* git diff --no-renames \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 \ + * 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 + */ + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(6, exp.files); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_DELETED]); + + /* git diff -M -C \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 \ + * 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 + */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + opts.flags = GIT_DIFF_FIND_RENAMES | GIT_DIFF_FIND_COPIES; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(4, exp.files); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]); + + git_diff_list_free(diff); + + /* git diff -M -C --find-copies-harder --break-rewrites \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 \ + * 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 + * with libgit2 default similarity comparison... + */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + opts.flags = GIT_DIFF_FIND_ALL; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + /* the default match algorithm is going to find the internal + * whitespace differences in the lines of sixserving.txt to be + * significant enough that this will decide to split it into + * an ADD and a DELETE + */ + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(5, exp.files); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_ADDED]); + cl_assert_equal_i(1, exp.file_status[GIT_DELTA_DELETED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]); + + git_diff_list_free(diff); + + /* git diff -M -C --find-copies-harder --break-rewrites \ + * 1c068dee5790ef1580cfc4cd670915b48d790084 \ + * 19dd32dfb1520a64e5bbaae8dce6ef423dfa2f13 + * with ignore_space whitespace comparision + */ + cl_git_pass(git_diff_tree_to_tree( + &diff, g_repo, old_tree, new_tree, &diffopts)); + + opts.flags = GIT_DIFF_FIND_ALL | GIT_DIFF_FIND_IGNORE_WHITESPACE; + cl_git_pass(git_diff_find_similar(diff, &opts)); + + /* Ignoring whitespace, this should no longer split sixserver.txt */ + + memset(&exp, 0, sizeof(exp)); + cl_git_pass(git_diff_foreach( + diff, diff_file_cb, diff_hunk_cb, diff_line_cb, &exp)); + + cl_assert_equal_i(4, exp.files); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_MODIFIED]); + cl_assert_equal_i(2, exp.file_status[GIT_DELTA_RENAMED]); + + git_diff_list_free(diff); git_tree_free(old_tree); git_tree_free(new_tree);